如何修复将数据导入MongoDB时的JavaScript堆内存不足错误？

Question

数据库操作git：（主）节点import_acparts_to_mongdb.js

<---最后几个GC-->

38787 ms：标记-扫描1384.9(1436.8)->1384.8(1436.8)MB,1181.9/0.0ms[分配失败][请求旧空间中的GC]。39964 ms：标记-扫描1384.8(1436.8)->1384.8(1436.8)MB,1177.7/0.0ms[分配失败][请求旧空间中的GC]。41199 ms：标记-扫描1384.8(1436.8)->1385.8(1420.8)MB,1234.0/0.0ms[最后手段气相色谱]。42429 ms：标记-扫描1385.8(1420.8)->1386.9(1420.8)MB,1229.8/0.0ms[最后手段气相色谱]。

const mongoose  = require('mongoose'),
    parse       = require('csv-parse'),
    path        = require('path'),
    fs          = require('fs'),
    ACpart      = require('./models/acparts');

mongoose.Promise = require('bluebird');

mongoose.connect('mongodb://localhost/automdm_test');

const db = mongoose.connection;

db.on('error', console.error.bind(console, 'connection error:'));

db.once('open', function() {
    // we're connected!

    const p = path.join(__dirname, '/../', 'file-operations', 'csv-files');
    //console.log(p);

    const parser = parse({delimiter: ';'}, function(err, data){
        //console.log(data);
        const facility = data.map((item,i) => data[i][0]);
        const item_number = data.map((item,i) => data[i][1]);
        const part_name = data.map((item,i) => data[i][2]);
        const part_description = data.map((item,i) => data[i][3]);
        const net_weight = data.map((item,i) => data[i][4]);
        const customs_statistical = data.map((item,i) => data[i][5]);

        // Looping and storing the data into mongodb
        for (let i = 1; i < data.length; i++) {

            const newACpart = new ACpart();
            newACpart.facility = facility[i]
            newACpart.item_number = item_number[i];
            newACpart.part_name = part_name[i];
            newACpart.part_description = part_description[i];
            newACpart.net_weight = net_weight[i];
            newACpart.customs_statistical = customs_statistical[i];
            newACpart.save()
            .then(function() {
                mongoose.disconnect();
            })
            .catch(function(err) {
                console.log('There was an error', err);
            });
        }
    });
    fs.createReadStream(p + '/mrsparts.csv').pipe(parser);
});

Answer 1

如果内存大于堆，则无法将所有内容放入内存中。使用流式CSV解析器，例如：

• https://www.npmjs.com/package/csv-stream
• https://www.npmjs.com/package/node-stream-csv
• https://www.npmjs.com/package/stream-csv-enhanced
• https://www.npmjs.com/package/csv-parse
• https://www.npmjs.com/package/fast-csv

然后分批而不是一次全部发送到数据库。