I'm trying to figure out the fastest way without affecting performance much while moving either synchronous or asynchronous 100.000+ files using Node.js
Few test cases showed, that synchronous renameSync()
is slower at ~20% then asynchronous rename()
. In the same time asynchronous rename()
using 3 times more CPU power, then synchronous renameSync()
.
Am I right to assume that it would be wiser to use synchronous renameSync()
because speed boost in asynchronous rename()
is not so significant (20%) and CPU utilization overhead in async version seems to be huge (300%)?
Or did I miss anything? Maybe there are better workarounds in terms of speed/performance.
The timings:
eachAsyncRenameAsync()
node rename.js 1.60s user 11.20s system 280% cpu 4.559 total
node rename.js 1.65s user 11.82s system 284% cpu 4.732 total
node rename.js 1.64s user 11.84s system 292% cpu 4.606 total
eachSyncRenameSync()
node rename.js 0.69s user 5.01s system 97% cpu 5.851 total
node rename.js 0.67s user 4.88s system 97% cpu 5.687 total
node rename.js 0.68s user 5.01s system 99% cpu 5.734 total
eachAsyncRenameSync()
node rename.js 0.67s user 4.99s system 97% cpu 5.797 total
node rename.js 0.68s user 4.95s system 97% cpu 5.754 total
node rename.js 0.66s user 4.89s system 97% cpu 5.690 total
eachSyncRenameAsync()
node rename.js 1.63s user 11.12s system 274% cpu 4.638 total
node rename.js 1.66s user 12.29s system 286% cpu 4.874 total
node rename.js 1.66s user 12.23s system 289% cpu 4.795 total
forSyncRenameAsync()
node rename.js 1.72s user 12.04s system 283% cpu 4.862 total
node rename.js 1.69s user 11.88s system 276% cpu 4.904 total
node rename.js 1.64s user 11.89s system 287% cpu 4.712 total
forSyncRenameSync()
node rename.js 0.64s user 4.94s system 97% cpu 5.715 total
node rename.js 0.66s user 5.01s system 97% cpu 5.807 total
node rename.js 0.65s user 4.93s system 99% cpu 5.616 total
The code:
const fs = require('fs')
const path = require('path')
const { each } = require('async')
let dir = '/opt/bin/test/Random 100000'
let dir2 = '/opt/bin/test/Random 100000 2'
function eachAsyncRenameAsync(files) {
each(files, file => {
fs.rename(path.join(dir,file), path.join(dir2,file), err => {
if(err) { console.log(err) }
})
})
}
function eachSyncRenameSync(files) {
files.forEach(file => {
fs.renameSync(path.join(dir,file), path.join(dir2,file))
})
}
function eachAsyncRenameSync(files) {
each(files, file => {
fs.renameSync(path.join(dir,file), path.join(dir2,file))
})
}
function eachSyncRenameAsync(files) {
files.forEach(file => {
fs.rename(path.join(dir,file), path.join(dir2,file), err => {
if(err) { console.log(err) }
})
})
}
function forSyncRenameAsync(files) {
for (i=0; i<files.length; i++) {
fs.rename(path.join(dir, files[i]), path.join(dir2, files[i]), err => {
if(err) { console.log(err) }
})
}
}
function forSyncRenameSync(files) {
for (i=0; i<files.length; i++) {
fs.renameSync(path.join(dir, files[i]), path.join(dir2, files[i]))
}
}
// Reading dir asynchronously and moving files
fs.readdir(dir, (err, files) => {
if (err) { console.log(err) }
else {
console.log('eachAsyncRenameAsync()')
eachAsyncRenameAsync(files)
}
})
via Systems Rebooter
No comments:
Post a Comment