Moving 100,000 files to Node.js. Synchronization / asynchronous performance and speed

I am trying to find the fastest way, without impacting performance, while moving either synchronously or asynchronously 100,000+ files at the same time using Node.js

I ran sync and async tests with three different loops forEach

async.each

and for

. The loop iterations did not significantly affect the synchronization results. And the heaviest logic affecting performance and speed is moving files either synchronously or asynchronously.

In test cases, it is shown that synchronous is renameSync()

slower by 20%, and then asynchronous rename()

. At the same time asynchronous rename()

, using 3 times more CPU power, then synchronous renameSync()

.

Is it possible to assume that it would be wiser to use synchronous renameSync()

, since the speed acceleration in asynchronous is rename()

not so significant and is equal to 20%. Although the CPU overhead in the asynchronous version seems huge - 300%?

Or have I missed nothing? Perhaps there are better workarounds in terms of speed and performance.

Timings:

eachAsyncRenameAsync()

node rename.js  1.60s user 11.20s system 280% cpu 4.559 total
node rename.js  1.65s user 11.82s system 284% cpu 4.732 total
node rename.js  1.64s user 11.84s system 292% cpu 4.606 total


eachSyncRenameSync()

node rename.js  0.69s user 5.01s system 97% cpu 5.851 total
node rename.js  0.67s user 4.88s system 97% cpu 5.687 total
node rename.js  0.68s user 5.01s system 99% cpu 5.734 total


eachAsyncRenameSync()

node rename.js  0.67s user 4.99s system 97% cpu 5.797 total
node rename.js  0.68s user 4.95s system 97% cpu 5.754 total
node rename.js  0.66s user 4.89s system 97% cpu 5.690 total


eachSyncRenameAsync()

node rename.js  1.63s user 11.12s system 274% cpu 4.638 total
node rename.js  1.66s user 12.29s system 286% cpu 4.874 total
node rename.js  1.66s user 12.23s system 289% cpu 4.795 total


forSyncRenameAsync()

node rename.js  1.72s user 12.04s system 283% cpu 4.862 total
node rename.js  1.69s user 11.88s system 276% cpu 4.904 total
node rename.js  1.64s user 11.89s system 287% cpu 4.712 total


forSyncRenameSync()

node rename.js  0.64s user 4.94s system 97% cpu 5.715 total
node rename.js  0.66s user 5.01s system 97% cpu 5.807 total
node rename.js  0.65s user 4.93s system 99% cpu 5.616 total

      

        
        
        
      

    

Code:

const fs = require('fs')
const path = require('path')
const { each } = require('async')

let dir = '/opt/bin/test/Random 100000'
let dir2 = '/opt/bin/test/Random 100000 2'

function eachAsyncRenameAsync(files) {
        each(files, file => {
              fs.rename(path.join(dir,file), path.join(dir2,file), err => {
                if(err) { console.log(err) }
              })
        })
}

function eachSyncRenameSync(files) {
    files.forEach(file => {
            fs.renameSync(path.join(dir,file), path.join(dir2,file))
    })
}

function eachAsyncRenameSync(files) {
    each(files, file => {
            fs.renameSync(path.join(dir,file), path.join(dir2,file))
    })
}

function eachSyncRenameAsync(files) {
    files.forEach(file => {
        fs.rename(path.join(dir,file), path.join(dir2,file), err => {
                if(err) { console.log(err) }
        })
    })
}

function forSyncRenameAsync(files) {
    for (i=0; i<files.length; i++) {
        fs.rename(path.join(dir, files[i]), path.join(dir2, files[i]), err => {
                if(err) { console.log(err) }
        })
    }

}

function forSyncRenameSync(files) {
    for (i=0; i<files.length; i++) {
        fs.renameSync(path.join(dir, files[i]), path.join(dir2, files[i]))
    }
}

// Reading dir asynchronously and moving files

fs.readdir(dir, (err, files) => {
    if (err) { console.log(err) }
    else {
        console.log('eachAsyncRenameAsync()')
        eachAsyncRenameAsync(files)
    }
})