How to minify / compress thousands of JS files, including some large ones, at the same time or sequentially, without crashing in the console?
Context
From the demo I am currently refactoring, I have a folder src
that contains 196 MB. About 142 MB are composed of two binaries.
About 2,000 of the remaining 2,137 files (about 46 MB) consist of JavaScript files, most of which belong to the official and full distributions of the two large frameworks. The largest JavaScript file is about 23 MB. It unminified code originally written in C ++ and compiled - with emscripten - asm .
I wanted to write a Node.js script that copies all my files from path src
to path dist
and minifies every JS or CSS file it comes across along the way. Unfortunately the number and / or size of the JS files involved seems to be breaking my script.
Skip the steps I took ...
Step 1
I started writing a small assembly script that copied all data from my folder src
to my folder dist
. I was surprised to learn that this process ends in a matter of seconds.
Below is my code for this script. Please note that you need Node 8 to run this code.
const util = require('util');
const fs = require('fs');
const path = require('path');
const mkdir = util.promisify(require('mkdirp'));
const rmdir = util.promisify(require('rimraf'));
const ncp = util.promisify(require('ncp').ncp);
const readdir = util.promisify(fs.readdir);
const readFile = util.promisify(fs.readFile);
const writeFile = util.promisify(fs.writeFile);
const stat = util.promisify(fs.stat);
const moveFrom = path.join(__dirname,"../scr");
const moveTo = path.join(__dirname,"../dist");
var copyFile = function(source, target) {
return new Promise(function(resolve,reject){
const rd = fs.createReadStream(source);
rd.on('error', function(error){
reject(error);
});
const wr = fs.createWriteStream(target);
wr.on('error', function(error){
reject(error);
});
wr.on('close', function(){
resolve();
});
rd.pipe(wr);
});
};
var copy = function(source, target) {
stat(source)
.then(function(stat){
if(stat.isFile()) {
console.log("Copying file %s", source);
switch (path.extname(target)) {
default:
return copyFile(source, target);
}
} else if( stat.isDirectory() ) {
return build(source, target);
}
}).catch(function(error){
console.error(error);
});
};
var build = function(source, target) {
readdir(source)
.then(function(list) {
return rmdir(target).then(function(){
return list;
});
})
.then(function(list) {
return mkdir(target).then(function(){
return list;
});
}).then(function(list) {
list.forEach(function(item, index) {
copy(path.join(source, item), path.join(target, item));
});
}).catch(function(error){
console.error(error);
})
};
build(moveFrom, moveTo);
Step 2
To minify my CSS files whenever I come across them, I have added a CSS evaluation.
To do this, I made the following changes to my code.
First, I added this feature:
var uglifyCSS = function(source, target) {
readFile(source, "utf8")
.then(function(content){
return writeFile(target, require('ycssmin').cssmin(content), "utf8");
}).catch(function(error){
console.error(error);
});
}
Then I changed my copy function, for example:
var copy = function(source, target) {
stat(source)
.then(function(stat){
if(stat.isFile()) {
console.log("Copying file %s", source);
switch (path.extname(target)) {
case ".css":
return uglifyCSS(source, target);
default:
return copyFile(source, target);
}
} else if( stat.isDirectory() ) {
return build(source, target);
}
}).catch(function(error){
console.error(error);
});
};
So far so good. Everything goes smoothly at this stage.
Step 3
Then I did the same to reduce my JS.
So, I added a new function:
var uglifyJS = function(source, target) {
readFile(source, "utf8")
.then(function(content){
return writeFile(target, require('uglify-js').minify(content).code, "utf8");
}).catch(function(error){
console.error(error);
});
}
Then I changed my copy function again:
var copy = function(source, target) {
stat(source)
.then(function(stat){
if(stat.isFile()) {
console.log("Copying file %s", source);
switch (path.extname(target)) {
case ".css":
return uglifyCSS(source, target);
case ".js":
return uglifyJS(source, target);
default:
return copyFile(source, target);
}
} else if( stat.isDirectory() ) {
return build(source, target);
}
}).catch(function(error){
console.error(error);
});
};
Problem
Everything goes wrong here. As the process continues to face more and more JS files, it continues to slow down until the process stops completely.
It looks like there are too many concurrent processes running and continues to consume more and more memory until there is no more memory left and the process just freezes. I tried other minifiers besides UglifyJS and I got the same problem for all of them. So the problem is not specific to UglifyJS.
Any ideas on how to fix this issue?
This is the complete code:
const util = require('util');
const fs = require('fs');
const path = require('path');
const mkdir = util.promisify(require('mkdirp'));
const rmdir = util.promisify(require('rimraf'));
const ncp = util.promisify(require('ncp').ncp);
const readdir = util.promisify(fs.readdir);
const readFile = util.promisify(fs.readFile);
const writeFile = util.promisify(fs.writeFile);
const stat = util.promisify(fs.stat);
const moveFrom = path.join(__dirname,"../scr");
const moveTo = path.join(__dirname,"../dist");
var copyFile = function(source, target) {
return new Promise(function(resolve,reject){
const rd = fs.createReadStream(source);
rd.on('error', function(error){
reject(error);
});
const wr = fs.createWriteStream(target);
wr.on('error', function(error){
reject(error);
});
wr.on('close', function(){
resolve();
});
rd.pipe(wr);
});
};
var uglifyCSS = function(source, target) {
readFile(source, "utf8")
.then(function(content){
return writeFile(target, require('ycssmin').cssmin(content), "utf8");
}).catch(function(error){
console.error(error);
});
}
var uglifyJS = function(source, target) {
readFile(source, "utf8")
.then(function(content){
return writeFile(target, require('uglify-js').minify(content).code, "utf8");
}).catch(function(error){
console.error(error);
});
}
var copy = function(source, target) {
stat(source)
.then(function(stat){
if(stat.isFile()) {
console.log("Copying file %s", source);
switch (path.extname(target)) {
case ".css":
return uglifyCSS(source, target);
case ".js":
return uglifyJS(source, target);
default:
return copyFile(source, target);
}
} else if( stat.isDirectory() ) {
return build(source, target);
}
}).catch(function(error){
console.error(error);
});
};
var build = function(source, target) {
readdir(source)
.then(function(list) {
return rmdir(target).then(function(){
return list;
});
})
.then(function(list) {
return mkdir(target).then(function(){
return list;
});
}).then(function(list) {
list.forEach(function(item, index) {
copy(path.join(source, item), path.join(target, item));
});
}).catch(function(error){
console.error(error);
})
};
build(moveFrom, moveTo);
source to share
Oligofren's suggestion didn't seem to help. However, removing the 23MB JS file fixed the issue. So it looks like the problem was not a large number of files (as I suspected), but a file too large for NodeJs to handle. I suppose playing with NodeJs memory settings (for example node --stack-size
) can fix this.
Anyway, while I still need a solution to get things working without deleting the 23MB file, I think deleting that one file from the files to be processed will have to be done now. This is pretty much just a proof of concept that I was working on.
source to share
Easy fix: your whole problem is that you have no restrictions on your parellization:
list.forEach(function(item, index) {
copy(path.join(source, item), path.join(target, item));
});
You send asynchronous operations synchronously. This means they come back immediately if you don't expect. You either need to make the operations sequential, or set a binding to the operations to be performed. This will make a list of functions:
const copyOperations = list.map((item) => {
return copy(path.join(source, item), path.join(target, item));
});
Then run them in sequence :
const initialValue = Promise.resolve();
copyOperations.reduce((accumulatedPromise, nextFn) => {
return accumulatedPromise.then(nextFn);
}, initialValue);
Now, if you want to wait for all these actions to complete, you need to return the promise, so the copy section of your code will look like this:
.then(function(list) {
const copyOperations = list.map((item) => {
return copy(path.join(source, item), path.join(target, item));
});
const allOperations = copyOperations.reduce((accumulatedPromise, nextFn) => {
return accumulatedPromise.then(nextFn);
}, Promise.resolve());
return allOperations;
})
This, of course, just copies one file at a time, and if you need more operations to be done at the same time, you need a fancier mechanism. Try this promise merging mechanism where you can set a threshold likerequire('os').cpus().length;
An example of limited parallelization using an ES6 generator
just replace the body above the function with then
this
const PromisePool = require('es6-promise-pool')
const maxProcesses = require('os').cpus().length;
const copyOperations = list.map((item) => {
return copy(path.join(source, item), path.join(target, item));
});
const promiseGenerator = function *(){
copyOperations.forEach( operation => yield operation );
}
var pool = new PromisePool(promiseGenerator(), maxProcesses)
return pool.start()
.then(function () {
console.log('Complete')
});
source to share