如何合并节点js中的两个csv文件行

Question

我有 2 个 csv 文件，它们具有不同的数据但具有相同的标题，例如。文件 1 数据是

"CODE","NAME","SUB_USER","SCORE"
"01","TEST","1","5"
"01","TEST","2","6"

其他文件FILE2有这样的数据

"CODE","NAME","SUB_USER","SCORE"
    "02","TEST2","3","5"
    "02","TEST2","4","6"

所以我想像这样合并两个文件创建FILE3输出

"CODE","NAME","SUB_USER","SCORE"
"01","TEST","1","5"
"01","TEST","2","6"
"02","TEST2","3","5"
"02","TEST2","4","6"

我试过下面的代码

var express = require('express');
var router = express.Router();
var fs = require('fs');
var parse = require('csv-parse');
var async = require('async');
var csv = require("fast-csv");

var file1 = appRoot + '\\csvFiles\\details1.csv';
var file2 = appRoot + '\\csvFiles\\details2.csv';
var stream = fs.createReadStream(file1);
var stream2 = fs.createReadStream(file2);
var fileData1 = [],
    fileData2 = [];
csv
    .fromStream(stream)
    .on("data", function(data) {
        fileData1.push(data);
    })
    .on("end", function() {
        console.log("done");
    });

csv
    .fromStream(stream2)
    .on("data", function(data) {
        fileData2.push(data);
    })
    .on("end", function() {
        console.log("done");
    });
var fileData3 = fileData1.concat(fileData2);

csv.writeToPath("outputfile.csv", fileData3).on("finish", function() {
    console.log("END");
});

但不工作不知道为什么？请帮我 ///******************************************** **************************// Thax 寻求帮助，但我在这里遇到了新问题

上面的代码经过一些更改后开始工作

var file1 = appRoot + '\\csvFiles\\details1.csv';
var file2 = appRoot + '\\csvFiles\\idetails2.csv';
var stream = fs.createReadStream(file1);
var stream2 = fs.createReadStream(file2);
var fileData1 = [],
    fileData2 = [],
    i = 0;

csv.fromStream(stream).on("data", function(data) {
    fileData1.push(data);
}).on("end", function() {
    csv.fromStream(stream2).on("data", function(data) {
        if (i != 0) {
            fileData2.push(data);
        }
        i++;
    }).on("end", function() {
        console.log("done");
        var fileData3 = fileData1.concat(fileData2);
        csv.writeToPath("outputfile.csv", fileData3).on("finish", function() {
            res.send('Done merge');
        });
    });
});

但问题是，如果我的文件数量增加了，那么我将如何处理那件事

Answer 1

这里最大的问题是一个很常见的问题。您执行异步任务，但在使用结果之前不会等待它们完成。

在调用每个任务的“结束”回调之前连接文件数据。解决方案是等待每个回调被调用，然后处理数据。

我创建了一个小例子使用 Promises

const file1 = 'one.csv';
const file2 = 'two.csv';
const stream = fs.createReadStream(file1);
const stream2 = fs.createReadStream(file2);
const fileData1 = [];
const fileData2 = [];

const file1Promise = new Promise((resolve) => {
  csv
      .parseFile(file1, {headers: true})
      .on('data', function(data) {
        fileData1.push(data);
      })
      .on('end', function() {
        console.log('done');
        resolve();
      });
});

const file2Promise = new Promise((resolve) => {
  csv
      .parseFile(file2, {headers: true})
      .on('data', function(data) {
        fileData2.push(data);
      })
      .on('end', function() {
        console.log('done');
        resolve();
      });
});

Promise.all([
  file1Promise,
  file2Promise,
])
    .then(() => {
      const fileData3 = fileData1.concat(fileData2);
      console.log(fileData3);

      const csvStream = csv.format({headers: true});
      const writableStream = fs.createWriteStream('outputfile.csv');

      writableStream.on('finish', function() {
        console.log('DONE!');
      });

      csvStream.pipe(writableStream);
      fileData3.forEach((data) => {
        csvStream.write(data);
      });
      csvStream.end();
    });

我创建了一个函数，您可以使用它轻松合并多个文件：

function concatCSVAndOutput(csvFilePaths, outputFilePath) {
  const promises = csvFilePaths.map((path) => {
    return new Promise((resolve) => {
      const dataArray = [];
      return csv
          .parseFile(path, {headers: true})
          .on('data', function(data) {
            dataArray.push(data);
          })
          .on('end', function() {
            resolve(dataArray);
          });
    });
  });

  return Promise.all(promises)
      .then((results) => {

        const csvStream = csv.format({headers: true});
        const writableStream = fs.createWriteStream(outputFilePath);

        writableStream.on('finish', function() {
          console.log('DONE!');
        });

        csvStream.pipe(writableStream);
        results.forEach((result) => {
          result.forEach((data) => {
            csvStream.write(data);
          });
        });
        csvStream.end();

      });
}

示例用法

concatCSVAndOutput(['one.csv', 'two.csv'], 'outputfile.csv')
    .then(() => ...doStuff);