Error: EMFILE: Too many open files
Using nw.js I am just trying to store images in an array of img elements with different random names.
But having multiple errors, is there something wrong with my code?
for (i = 0; i < imgs.length; i++) {
request(imgs[i].getAttribute('src')).on('error', function(err) {
throw err
}).pipe(fs.createWriteStream('data/imgs/' + randomString))
}
imgs [] is an array of 100-500 html img elements, but I get
Error: EMFILE: too many open files, open *<directory>*
And one more error:
"Uncaught Error: socket hang up"
Although it retains some images, some of them are damaged and create too many images than there are.
source to share
Use a graceful-fs
module that is a replacement for the module fs
. It is a wrapper around its own module fs
. Quoting the docs of the modulegraceful-fs
,
Causes calls
open
andreaddir
and retries them as soon as something is closed if there is an errorEMFILE
from too many file descriptors.
Since it provides the same APIs as the native fs
module, you can use it just like a normal fs
module.
// use just like fs
var fs = require('graceful-fs');
Note. ... This library was created by Isaac Z. Schlüter, one of the main developers of Node.js.
source to share
Another simple solution is to just wait for the file to load and only then open a new request using recursion:
var fs = require('fs');
var request = require('request');
var dest = '../data/downloads/';
function _saveAllFiles (fileUrlArray, curIdx, options) {
try {
var count = fileUrlArray.length - 1;
var curFile = fileUrlArray[curIdx];
var stream = request(curFile)
.pipe(fs.createWriteStream(dest + "file_" + curIdx));
stream.on('finish', function () {
console.log("Downloaded", curFile);
stream.close();
if (curIdx + 1 < count) {
//Finished, download next file
_saveAllFiles(fileArray, curIdx + 1, options);
}
});
stream.on('error', function () {
console.log("Error", curFile);
stream.close();
stream.end();
if (curIdx + 1 < count) {
//Error occured, go to next file
_saveAllFiles(fileArray, curIdx + 1, options);
}
});
} catch (err) {
console.error("Failed to download file", err);
}
}
source to share