List all directories and all files and upload them to my bucket (S3 Amazon) using Node.JS
Code below:
I am using walkit walker, here -> https://github.com/substack/node-findit
In this package I am listing all the directories and files of my application and I am trying to push it to my bucket on Amazon S3 (with my own code).
I'm not sure if the code is correct and I don't know what I need to put in the body, inside the params object.
This part listens to all directories of my application:
finder.on('directory', function (dir, stat, stop) {
var base = path.basename(dir);
if (base === '.git' || base === 'node_modules' || base === 'bower_components') {
stop();
}
else {
console.log(dir + '/');
}
});
And this one listens to all files of my application:
finder.on('file', function (file, stat) {
console.log(file);
});
I updated it to send data to my bucket, for example:
finder.on('file', function (file, stat) {
console.log(file);
var params = {
Bucket: BUCKET_NAME,
Key: file,
//Body:
};
//console.log(params.body);
s3.putObject(params, function(err) {
if(err) {
console.log(err);
}
else {
console.log("Success!");
}
});
});
I really don't know what I need to insert into the Body and I don't know if the code is correct. Can anyone help me?
Thank.
to help, all the code, all the code:
var fs = require('fs');
var finder = require('findit')(process.argv[2] || '.');
var path = require('path');
var aws = require('aws-sdk');
var s3 = new aws.S3();
aws.config.loadFromPath('./AwsConfig.json');
var BUCKET_NAME = 'test-dev-2';
finder.on('directory', function (dir, stat, stop) {
var base = path.basename(dir);
if (base === '.git' || base === 'node_modules' || base === 'bower_components') {
stop();
}
else {
console.log(dir + '/');
}
});
finder.on('file', function (file, stat) {
console.log(file);
var params = {
Bucket: BUCKET_NAME,
Key: file,
//Body:
};
//console.log(params.body);
s3.putObject(params, function(err) {
if(err) {
console.log(err);
}
else {
console.log("Success");
}
});
});
finder.on('error', function (err) {
console.log(err);
});
finder.on('end', function () {
console.log('Done!');
});
Based on the documentation , the parameter Body
s3.putObject
can be Buffer
, Typed Array, Blob,, String
or ReadableStream
. The best one to use in most cases would be ReadableString
. You can create ReadableString
from any file using a createReadStream()
function in a module fs
.
So this part of your code will look something like this:
finder.on('file', function (file, stat) {
console.log(file);
var params = {
Bucket: BUCKET_NAME,
Key: file,
Body: fs.createReadStream(file) // NOTE: You might need to adjust "file" so that it either an absolute path, or relative to your code directory.
};
s3.putObject(params, function(err) {
if(err) {
console.log(err);
}
else {
console.log("Success!");
}
});
});
I also want to point out that you might run into a problem with this code if you pass it a directory with a lot of files. putObject
- this function is asynchronous , which means it will be called and then the code will move on to something else while it does its thing (ok, this is a gross oversimplification, but you can think of it this way). What this means in terms of this code is that you will essentially download all of the files it finds at the same time; this is not good.
I would suggest using something like async
module to queue your file being loaded so that only a few of them happen at the same time.
Essentially, you are moving the code you have in your event handler file
to the queue workflow method:
var async = require('async');
var uploadQueue = async.queue(function(file, callback) {
var params = {
Bucket: BUCKET_NAME,
Key: file,
Body: fs.createReadStream(file) // NOTE: You might need to adjust "file" so that it either an absolute path, or relative to your code directory.
};
s3.putObject(params, function(err) {
if(err) {
console.log(err);
}
else {
console.log("Success!");
}
callback(err); // <-- Don't forget the callback call here so that the queue knows this item is done
});
}, 2); // <-- This "2" is the maximum number of files to upload at once
Pay attention to 2
the end where your concurrency is listed, which in this case is the number of files to download at once.
Then your event handler file
just becomes:
finder.on('file', function (file, stat) {
uploadQueue.push(file);
});
This will queue all the files it finds and download them 2 at a time until they go through everything.
source to share