How to handle reading a directory tree recursively with promises
I am writing trying to write a function that accomplishes the same as the following, written using a callback pattern with a promise pattern:
function readdirRecursive(path,handler,callback) {
var errs = [],
tree = {};
fs.readdir(path,function(err,dir) {
if(err)return callback(err);
var pending = dir.length;
if(!pending)return callback(null,tree);
dir.forEach(function(file) {
var newPath = Path.join(path,file);
fs.stat(newPath,function(err,stats) {
if(stats.isDirectory()) {
readdirRecursive(newPath,handler,function(err,subtree) {
tree[file] = subtree
handler(tree,newPath,file,"directory",function(err) {
if(err)errs.push(err);
if(!--pending)return callback(errs.length>0?errs:null,tree);
});
});
} else {
tree[file] = null;
handler(tree,newPath,file,"file",function(err) {
if(err)errs.push(err);
if(!--pending)return callback(errs.length>0?errs:null,tree);
});
}
});
});
});
};
This is my current attempt:
function readdirRecursive(path) {
var tree = {};
return Q.Promise(function(resolve,reject,notify) {
return readdir(path)
.then(function(dir) {
var futures = [];
var pending = dir.length;
if(!pending)return resolve(tree);
dir.forEach(function(file) {
var deferred = Q.defer();
var subPath = Path.join(path,file);
futures.push(stat(subPath)
.then(function(stats) {
if(stats.isDirectory()) {
tree[file] = tree;
var sub = readdirRecursive(subPath)
sub
.then(function(subtree) {
notify({
path:subPath,
name:file,
type:"directory",
done:deferred,
pending:pending
});
//return subtree;
},reject,notify);
} else {
tree[file] = null;
notify({
tree:tree,
path:subPath,
name:file,
type:"file",
done:deferred,
pending:pending
});
//return null;
}
//console.log("tree",tree);
deferred.promise()
.then(function() {
console.log("pending promise");
if(!--pending)resolve(tree);
}
,function(err) {
reject();
});
}));
});
return Q.all(futures)
.then(function(futures) {
console.log("hi",futures);
});
});
});
};
This code will iterate over the whole tree, but it will not return the tree, and the notification action will happen, but the pending promise will never resolve.
When a pending promise is triggered before notification, nothing happens at all.
I know I could solve this problem by passing the function to be executed to the progress event rather than trying to make some kind of promise, but I want to make as much use of promises as possible here, for example this code does exactly what I want:
function readdirRecursive(path) {
var tree = {};
return Q.Promise(function(resolve,reject,notify) {
return readdir(path)
.then(function(dir) {
var futures = [];
var pending = dir.length;
if(!pending)return resolve(tree);
dir.forEach(function(file) {
var deferred = Q.defer();
var subPath = Path.join(path,file);
console.log("file",file);
/*deferred.promise()
.then(function() {
console.log("pending promise");
if(!--pending)resolve(tree);
}
,function(err) {
reject();
});*/
futures.push(stat(subPath)
.then(function(stats) {
if(stats.isDirectory()) {
var sub = readdirRecursive(subPath)
sub
.then(function(subtree) {
tree[file] = subtree
notify({
path:subPath,
name:file,
type:"directory",
done:function(err) {
console.log("pending promise");
if(err)return reject(err);
if(!--pending)resolve(tree);
},
pending:pending
});
//return subtree;
},reject,notify);
} else {
tree[file] = null;
notify({
tree:tree,
path:subPath,
name:file,
type:"file",
done:function(err) {
console.log("pending promise");
if(err)return reject();
if(!--pending)resolve(tree);
},
pending:pending
});
//return null;
}
//console.log("tree",tree);
}));
});
return Q.all(futures)
.then(function(futures) {
console.log("hi",futures);
});
});
});
};
this is the code that will perform these functions:
readdirRecursive("../").then(function(tree) {
console.log("TREE!!!",tree);
},function(err) {
console.log("ERROR",err);
},function(progress) {
console.log("PRGRESS WAS MADE",progress);
progress.done();
});
source to share
My first thought was to just wrap my original function in a promise. This is usually how I would do it without redesigning the underlying code:
function readdirRecursiveWithPromise (path, handler) {
return new Promise((resolve, reject) => {
readdirRecursive(path, handler, (err, tree) => {
if (err) {
reject(err);
}
else {
resolve(tree);
}
});
})
}
Unfortunately though, when I tried to test this code, I found a couple of basic problems with your code.
First, I have no idea what your "handler" is supposed to do. You have not provided an explanation for this or described what he should do. This is very important to the problem because it controls whether the final callback is eventually called, so I can assume that the "handler" is in control of this operation, and if your "callback" is not called it might be logic related in your "handler".
The next problem is that your "pending" variable is set by the number of files and directories altogether, and yet it is scaled down for directories only. This way, your "pending" variable will never reach 0, and your conditional code that invokes the callback will never get called.
So, I'm going to get rid of the "handler" and the "waiter", and I'll show you how I rewrote this with promises from scratch.
Here's a working code example: https://github.com/ashleydavis/read-directory-with-promises . Read on for an explanation.
Start with a promise-based version of readdir that is not recursive:
function readdir (path) { // Promise-based version of readdir.
return new Promise((resolve, reject) => { // Wrap the underlying operation in a promise.
fs.readdir(path, (err, files) => {
if (err) {
reject(err); // On error, reject the promise.
}
else {
resolve(files); // On success, resolve the promise.
}
});
});
};
We also need a promise-based function that we can use to determine the type (file or directory) of a particular path:
function determineType (parentPath, childPath) { // Promise-based function to determine if the path is a file or directory.
return new Promise((resolve, reject) => {
fs.stat(path.join(parentPath, childPath), (err, stats) => {
if (err) {
reject(err);
}
else {
resolve({
path: childPath,
type: stats.isDirectory() ? 'directory' : 'file' // Check if it a directory or a file.
});
}
});
});
};
Now we can expand determineType
and create a function that takes an array of paths and determines the type of each one. This uses Promise.all
to execute multiple asynchronous operations in parallel:
function determineTypes (parentPath, paths) { // Async function to determine if child paths are directories or files.
return Promise.all(
paths.map(
childPath => determineType(parentPath, childPath) // Is the path a directory or a file?
)
);
};
We can now build our promise-based recursive version readdir
:
function readdirTree (rootPath) { // Read an entire directory tree, the promise-based recursive version.
return readdir(rootPath) // Initial non-recursive directory read.
.then(childPaths => determineTypes(rootPath, childPaths)) // Figure out the type of child paths.
.then(children => {
return Promise.all(children // Use Promise.all to figure out all sub-trees in a parallel.
.filter(child => child.type === 'directory') // Filter so we only directories are remaining.
.map(child => {
return readdirTree(path.join(rootPath, child.path)) // It a directory, recurse to the next level down.
.then(subTree => {
return {
path: child.path,
subTree: subTree,
};
});
})
);
})
.then(children => {
const tree = {}; // Reorganise the list of directories into a tree.
children.forEach(directory => {
tree[directory.path] = directory.subTree;
});
return tree;
});
};
Here's a usage example:
readdirTree("c:\\some-directory")
.then(tree => {
console.log("tree:");
console.log(tree);
})
.catch(err => {
console.error("error:");
console.error(err);
});
I have a complete working example for you on my Github: https://github.com/ashleydavis/read-directory-with-promises
Hope it helps you move forward.
source to share