Node.js - How do you control the Async callback sequence?
In the following example, I traverse a directory tree to get a list of directories and files. It is important that the order of directories and files is the same as you would visually view them in the file explorer. For the sake of brevity, I just write folders and files to console.log in the sequence I would expect ...
var fs = require('fs');
var path = require('path');
var walk = function(dir) {
console.log(dir);
var files = [];
var items = fs.readdirSync(dir).sort(function (a, b) {
return a.toLowerCase().localeCompare(b.toLowerCase());
});
items.forEach(function(item){
if (item.charAt(0) !== '.') {
var itemPath = path.join(dir,item);
if (fs.lstatSync(itemPath).isDirectory()) {
walk(itemPath);
} else {
files.push(itemPath);
}
}
});
files.forEach(function(file){
console.log(file);
});
};
walk('c:\\foo\\bar');
This version of Sync took all 20 minutes to develop and provide the exact sequence I needed.
Turning this into an Async solution is a great example of how to make scalable Node.js "can quickly become verbose, complex, and time consuming compared to other languages.
While I would be wondering how others would solve this in a clean way (forced consistency), I am really interested in how other developers are using the tile table libraries for this type of problem.
There seem to be numerous NPM solutions that have all adopted their own implementation. Without spending time studying them all, I'm not sure which one to choose.
Bottom line question ... How would you turn the above code into an Async solution using a callback sequence. Examples would be helpful. If you are using the boiler plate library why did you choose it. I would be interested to hear other people's comments, in particular on the style and readability of the various solutions.
Update 1
I need to provide a sequence of callbacks for asynchronous events that happen in a recursive function. This should not be confused with doing a sequence of callbacks of nested callbacks, which can be solved with promises, for example. then(). Then(). Then()...
I need the recursive walk () function to fire only one at a time, so that the next walk () actually waits for the previous step () to complete. I say "effectively waits" as an idea of what is about to happen, but obviously the code "really waits" would mean suspending the code, which is also not the right decision.
Although I'm wondering if a solution that somehow implements process.nextTick () to park the next walk () could be in one direction.
source to share
Why you shouldn't use Promises when it comes to asynchronous operations
A simplified, customized version of the bluebird example
var Promise = require("bluebird");
var join = Promise.join;
var fs = Promise.promisifyAll(require("fs"));
var path = require("path")
var walk = function(dir){
return fs.readdirAsync(dir).map(function(fileName) {
var file = path.join(dir, fileName);
var stat = fs.statAsync(file);
return join(stat, function(stat) {
return {
stat: stat,
fileName: file
}
});
}).call("sort", function(a, b) {
return a.fileName.localeCompare(b.fileName);
}).each(function(file) {
if(file.stat.isDirectory()){
walk(file.fileName);
}
console.log(file.fileName + " last modified " + file.stat.mtime)
})
}
walk('./');
source to share
I repeated this problem and came up with the following solution ...
var fs = require('fs');
var path = require('path');
var sorter = function (a, b) {
return a.toLowerCase().localeCompare(b.toLowerCase());
}
var readdirAsync = function(dir) {
return new Promise(function (resolve,reject) {
fs.readdir(dir,function(err,dirs) {
if (err) {
reject(err)
} else {
resolve(dirs)
}
})
})
}
var walker = function(dir) {
return new Promise(function (resolve,reject) {
var depth = 0;
var fileTree = [];
var sortHack=Array(50).join('Z')+':';
var walk = function(dir) {
var files = [];
depth++;
fileTree.push(dir);
readdirAsync(dir).then(function(dirs) {
dirs.forEach(function(item){
if (item.charAt(0) !== '.') {
var itemPath = path.join(dir,item);
if (fs.lstatSync(itemPath).isDirectory()) {
walk(itemPath);
} else {
var parts=itemPath.split('/');
parts.push(sortHack + parts.pop());
files.push(parts.join('/'));
}
}
});
files.forEach(function(file){
fileTree.push(file);
});
depth--;
if (depth===0) {
fileTree.sort(sorter);
for (var n=0;n<fileTree.length;n++) {
fileTree[n] = fileTree[n].replace(sortHack,"")
}
resolve(fileTree);
}
},function(reason){reject(reason)}).catch(function(reason){reject(reason)});
}
walk(dir);
});
}
var start = new Date().getTime();
walker('/home').then(function(tree){
console.log(tree);
var took = new Date().getTime() - start;
console.log(took + 'ms')
},function(reason){
console.log("something went wrong :( " + reason);
}).catch(function(reason){
console.log("something went very wrong :S " + reason);
});
source to share