在Node.js同步/异步性能和速度中移动100.000+个文件

Question

我试图找出最快的方法而不影响性能,同时使用Node.js同步或异步移动100.000+文件

我有3个不同的循环由同步和异步测试forEach async.each和for.循环迭代不会对时序结果产生太大影响.影响性能和速度的最重要的逻辑是同步或异步移动文件.

测试用例显示,同步renameSync()速度较慢,为20%,然后是异步rename().同时异步rename()使用3倍以上的CPU功率,然后同步renameSync().

---

我是否正确地认为使用同步更明智,renameSync()因为异步中的速度提升rename()不是那么重要且等于20%.虽然异步版本的CPU利用率开销似乎很大 - 300%？

或者我错过了什么？也许在速度/性能方面有更好的解决方法.

时间安排:

eachAsyncRenameAsync()

node rename.js  1.60s user 11.20s system 280% cpu 4.559 total
node rename.js  1.65s user 11.82s system 284% cpu 4.732 total
node rename.js  1.64s user 11.84s system 292% cpu 4.606 total


eachSyncRenameSync()

node rename.js  0.69s user 5.01s system 97% cpu 5.851 total
node rename.js  0.67s user 4.88s system 97% cpu 5.687 total
node rename.js  0.68s user 5.01s system 99% cpu 5.734 total


eachAsyncRenameSync()

node rename.js  0.67s user 4.99s system 97% cpu 5.797 total
node rename.js  0.68s user 4.95s system 97% cpu 5.754 total
node rename.js  0.66s user 4.89s system 97% cpu 5.690 total


eachSyncRenameAsync()

node rename.js  1.63s user 11.12s system 274% cpu 4.638 total
node rename.js  1.66s user 12.29s system 286% cpu 4.874 total
node rename.js  1.66s user 12.23s system 289% cpu 4.795 total


forSyncRenameAsync()

node rename.js  1.72s user 12.04s system 283% cpu 4.862 total
node rename.js  1.69s user 11.88s system 276% cpu 4.904 total
node rename.js  1.64s user 11.89s system 287% cpu 4.712 total


forSyncRenameSync()

node rename.js  0.64s user 4.94s system 97% cpu 5.715 total
node rename.js  0.66s user 5.01s system 97% cpu 5.807 total
node rename.js  0.65s user 4.93s system 99% cpu 5.616 total

代码:

const fs = require('fs')
const path = require('path')
const { each } = require('async')

let dir = '/opt/bin/test/Random 100000'
let dir2 = '/opt/bin/test/Random 100000 2'

function eachAsyncRenameAsync(files) {
        each(files, file => {
              fs.rename(path.join(dir,file), path.join(dir2,file), err => {
                if(err) { console.log(err) }
              })
        })
}

function eachSyncRenameSync(files) {
    files.forEach(file => {
            fs.renameSync(path.join(dir,file), path.join(dir2,file))
    })
}

function eachAsyncRenameSync(files) {
    each(files, file => {
            fs.renameSync(path.join(dir,file), path.join(dir2,file))
    })
}

function eachSyncRenameAsync(files) {
    files.forEach(file => {
        fs.rename(path.join(dir,file), path.join(dir2,file), err => {
                if(err) { console.log(err) }
        })
    })
}

function forSyncRenameAsync(files) {
    for (i=0; i<files.length; i++) {
        fs.rename(path.join(dir, files[i]), path.join(dir2, files[i]), err => {
                if(err) { console.log(err) }
        })
    }

}

function forSyncRenameSync(files) {
    for (i=0; i<files.length; i++) {
        fs.renameSync(path.join(dir, files[i]), path.join(dir2, files[i]))
    }
}

// Reading dir asynchronously and moving files

fs.readdir(dir, (err, files) => {
    if (err) { console.log(err) }
    else {
        console.log('eachAsyncRenameAsync()')
        eachAsyncRenameAsync(files)
    }
})

Answer 1

对于异步执行比较优化，创建一个池来限制并发操作的数量将使您找到最佳结果。

您所做的基准测试表明，最佳结果是在 1 到 n 个异步并发操作之间。