Javascript - Callback after all nested forEach loops have finished
I'm sure this is a pretty simple task, but I can't wrap it around right now. I have a nested set of forEach loops and I need to have a callback when all the loops are done.
I am open to using async.js
This is what I am working with:
const scanFiles = function(accounts, cb) {
let dirs = ['pending', 'done', 'failed'];
let jobs = [];
accounts.forEach(function(account) {
dirs.forEach(function(dir) {
fs.readdir(account + '/' + dir, function(err, files) {
files.forEach(function(file) {
//do something
//add file to jobs array
jobs.push(file);
});
});
});
});
//return jobs array once all files have been added
cb(jobs);
}
source to share
By using forEach
the 2nd parameter, index, you can check if all loops are executed every time you run the innermost loop.
So with just a few lines added to your code, you end up with this:
const scanFiles = function(accounts, cb) {
let dirs = ['pending', 'done', 'failed'];
let jobs = [];
accounts.forEach(function(account, accIndex) {
dirs.forEach(function(dir, dirIndex) {
fs.readdir(account + '/' + dir, function(err, files) {
files.forEach(function(file, fileIndex) {
//do something
//add file to jobs array
jobs.push(file);
// Check whether each loop is on its last iteration
const filesDone = fileIndex >= files.length - 1;
const dirsDone = dirIndex >= dirs.length - 1;
const accsDone = accIndex >= accounts.length - 1;
// all three need to be true before we can run the callback
if (filesDone && dirsDone && accsDone) {
cb(jobs);
}
});
});
});
});
}
source to share
Simplified solution
No need for loops and clicking on arrays
I noticed that all the answers here use a lot of complex code. You can make it much easier:
let fs = require('mz/fs');
let path = require('path');
let d = ['pending', 'done', 'failed'];
let a = ['A', 'B', 'C']; // <-- example accounts
let paths = [].concat.apply([], d.map(d => (a.map(a => path.join(d,a)))));
Promise.all(paths.map(path => fs.readFile(path, 'utf-8'))).then(files => {
// you have all data here
}).catch(error => {
// handle errors here
});
Description
If you are using the daily version fs
- currently you can use:
let fs = require('mz/fs');
with module mz
:
and it will soon become native to Node, see:
then you should be able to do something like the code below. Using data:
// directories:
let d = ['pending', 'done', 'failed'];
// accounts:
let a = ['A', 'B', 'C'];
You can easily create an array of paths:
let paths = [].concat.apply([], d.map(d => (a.map(a => path.join(d,a)))));
From which you can create an array of promises:
let promises = paths.map(path => fs.readFile(path, 'utf-8'));
You can even use Promise.all()
to read all of your files:
let data = Promise.all(promises);
Now you can use everything like:
data.then(files => {
// you have everything ready here
}).catch(error => {
// some error happened
});
Note. The above code requires two modules to work:
let fs = require('mz/fs');
let path = require('path');
source to share
Simple counter
One easy way is to just keep the counter.
const scanFiles = function(accounts, cb) {
let dirs = ['pending', 'done', 'failed'];
let jobs = [];
// Variables to keep track of
const lastAccountIndex = accounts.length * dirs.length;
let indexCounter = 0;
accounts.forEach(function(account) {
dirs.forEach(function(dir) {
fs.readdir(account + '/' + dir, function(err, files) {
files.forEach(function(file) {
//do something
//add file to jobs array
jobs.push(file);
indexCounter++;
});
//return jobs array once all files have been added
if (lastAccountIndex === indexCounter) {
cb(jobs);
}
});
});
});
}
Promise
Alternatively, fs + prom can be very helpful here.
const scanFiles = function(accounts) {
let dirs = ['pending', 'done', 'failed'];
let jobs = [];
const filePromises = [];
accounts.forEach(function(account) {
dirs.forEach(function(dir) {
filePromises.push(new Promise((resolve, reject) => {
fs.readdir(account + '/' + dir, function(err, files) {
files.forEach(function(file) {
resolve(file);
});
});
}));
});
});
return Promise.all(filePromises);
}
scanFiles(someAccounts)
.then((files) => {
files.forEach((file) => {
// At this point, iwll the files will be scanned
// So, do whatever you want with all the files here.
});
});
fs promise
Or just use https://www.npmjs.com/package/fs-promise
source to share
If you use asyc library https://caolan.github.io/async/docs.html your code will be much faster. (forEach blocks [ JavaScript, Node.js: is Array.forEach asynchronous? ).
const scanFiles = function (accounts, cb) {
let dirs = ['pending', 'done', 'failed'];
let jobs = [];
async.each(accounts, function (account, accountCallback) {
async.each(dirs, function (dir, dirCallback) {
fs.readdir(account + '/' + dir, function (err, files) {
if(err) console.log(err);
async.each(files, function (file, fileCallback) {
//do something
//add file to jobs array
jobs.push(file);
fileCallback();
}, dirCallback);
});
}, accountCallback);
}, function (err) {
//return jobs array once all files have been added
if (err) throw err;
cb(jobs)
});
};
source to share
So the problem is that you were sending an empty result before it fs.readdir
was executed because nodeJS is asynchronous. So the solution is to add a callback inside the fs.readdir function.
const scanFiles = function (accounts, cb) {
let dirs = ['pending', 'done', 'failed'];
let jobs = [];
accounts.forEach(function (account, i) {
dirs.forEach(function (dir, j) {
fs.readdir(account + '/' + dir, function (err, files) {
files.forEach(function (file, k) {
//do something
//add file to jobs array
jobs.push(file);
});
if (i === accounts.length - 1 && j === dirs.length - 1 && k === files.length - 1) {
//return jobs array once all files have been added
cb(jobs);
}
});
});
});
}
source to share