/**
* 本地生成csv文件
* @param data
* @param fields
* @param path
*/
async function writeFile(data, fields, path) {
const opts = {fields};
try {
const parser = new Parser(opts);
const csv = parser.parse(data);
// console.log('csv:', csv);
console.log('开始写入文件……')
fs.writeFile(path,csv, 'utf8', (err) => {
if (err) {
console.log('写入失败')
} else {
console.log('写入成功')
}
});
} catch (err) {
console.error(err);
}
}
问题:RangeError: Invalid string length
RangeError: Invalid string length
at D:\zj\nodejs-server -model\node_modules\json2csv\dist\json2csv.cjs.js:1096:45
at Array.reduce (<anonymous>)
at fastJoin (D:\zj\nodejs-server -model\node_modules\json2csv\dist\json2csv.cjs.js:1086:14)
at JSON2CSVParser.processData (D:\zj\nodejs-server -model\node_modules\json2csv\dist\json2csv.cjs.js:1507:14)
at JSON2CSVParser.parse (D:\zj\nodejs-server -model\node_modules\json2csv\dist\json2csv.cjs.js:1465:23)
at writeFile (D:\zj\nodejs-server -model\app\controller\test\util.js:52:28)
at parseTrainingData (D:\zj\nodejs-server -model\app\controller\test\testData.js:153:11)
at process._tickCallback (internal/process/next_tick.js:68:7)
// Convert back to CSV
var csv = Papa.unparse(data);
结果:RangeError: Invalid string length
RangeError: Invalid string length
at D:\zj\nodejs-server -model\node_modules\json2csv\dist\json2csv.cjs.js:1096:45
at Array.reduce (<anonymous>)
at fastJoin (D:\zj\nodejs-server -model\node_modules\json2csv\dist\json2csv.cjs.js:1086:14)
at JSON2CSVParser.processData (D:\zj\nodejs-server -model\node_modules\json2csv\dist\json2csv.cjs.js:1507:14)
at JSON2CSVParser.parse (D:\zj\nodejs-server -model\node_modules\json2csv\dist\json2csv.cjs.js:1465:23)
at writeFile (D:\zj\nodejs-server -model\app\controller\test\util.js:52:28)
at parseTrainingData (D:\zj\nodejs-server -model\app\controller\test\testData.js:153:11)
at process._tickCallback (internal/process/next_tick.js:68:7)
async function writeFileStream(data, fields, path) {
const BATCH_SIZE = 1000;
const BATCH_NUM = data.length / BATCH_SIZE;
console.log("BATCH_NUM:", BATCH_NUM);
let csv = '';
const opts = {fields};
let batch_index = 0;
let from = batch_index * BATCH_SIZE;
let to = from + BATCH_SIZE;
let data_part = data.slice(from, to);
let parser = new Parser(opts);
let csv_part = parser.parse(data_part);
csv += csv_part
batch_index++;
while (batch_index < BATCH_NUM) {
console.log('batch_index:', batch_index)
from = batch_index * BATCH_SIZE;
to = from + BATCH_SIZE;
data_part = data.slice(from, to);
const opts2 = {fields: fields, header: false};
parser = new Parser(opts2);
csv_part = parser.parse(data_part);
csv += '\n';
csv += csv_part;
batch_index++;
}
console.log('开始写入文件……')
fs.writeFile(path,csv, 'utf8', (err) => {
if (err) {
console.log('写入失败')
} else {
console.log('写入成功')
}
});
}
结果:RangeError: Invalid string length
async function writeFileStream(data, fields, path) {
const BATCH_SIZE = 1000;
const BATCH_NUM = data.length / BATCH_SIZE;
console.log("BATCH_NUM:",BATCH_NUM);
let writerStream = fs.createWriteStream(path);
const opts = {fields};
let batch_index = 0;
let from = batch_index * BATCH_SIZE;
let to = from + BATCH_SIZE;
let data_part = data.slice(from, to);
let parser = new Parser(opts);
let csv_part = parser.parse(data_part);
writerStream.write(csv_part,'utf8');
batch_index++;
while (batch_index < BATCH_NUM) {
let csv = '';
console.log('batch_index:', batch_index)
from = batch_index * BATCH_SIZE;
to = from + BATCH_SIZE;
data_part = data.slice(from, to);
//不需要文件头
const opts2 = {fields: fields, header: false};
parser = new Parser(opts2);
csv_part = parser.parse(data_part);
//需要添加回车
csv += '\n';
csv += csv_part;
writerStream.write(csv,'utf8');
batch_index++;
}
// 标记文件末尾
writerStream.end();
// 处理流事件 --> data, end, and error
writerStream.on('finish', function() {
console.log("文件写入完成");
});
}
结果:成功!!!