node使用json2csv将大型json转化为csv存入本地失败

原始方法

/**
 * 本地生成csv文件
 * @param data
 * @param fields
 * @param path
 */
async function writeFile(data, fields, path) {
    const opts = {fields};
    try {
        const parser = new Parser(opts);
        const csv = parser.parse(data);
        // console.log('csv:', csv);
        console.log('开始写入文件……')
        fs.writeFile(path,csv, 'utf8', (err) => {
            if (err) {
                console.log('写入失败')
            } else {
                console.log('写入成功')
            }
        });
    } catch (err) {
        console.error(err);
    }
}

问题:RangeError: Invalid string length

RangeError: Invalid string length
    at D:\zj\nodejs-server -model\node_modules\json2csv\dist\json2csv.cjs.js:1096:45
    at Array.reduce (<anonymous>)
    at fastJoin (D:\zj\nodejs-server -model\node_modules\json2csv\dist\json2csv.cjs.js:1086:14)
    at JSON2CSVParser.processData (D:\zj\nodejs-server -model\node_modules\json2csv\dist\json2csv.cjs.js:1507:14)
    at JSON2CSVParser.parse (D:\zj\nodejs-server -model\node_modules\json2csv\dist\json2csv.cjs.js:1465:23)
    at writeFile (D:\zj\nodejs-server -model\app\controller\test\util.js:52:28)
    at parseTrainingData (D:\zj\nodejs-server -model\app\controller\test\testData.js:153:11)
    at process._tickCallback (internal/process/next_tick.js:68:7)

思路1:换插件,使用 papaparse(失败)

papaparse

// Convert back to CSV
var csv = Papa.unparse(data);

结果:RangeError: Invalid string length

RangeError: Invalid string length
    at D:\zj\nodejs-server -model\node_modules\json2csv\dist\json2csv.cjs.js:1096:45
    at Array.reduce (<anonymous>)
    at fastJoin (D:\zj\nodejs-server -model\node_modules\json2csv\dist\json2csv.cjs.js:1086:14)
    at JSON2CSVParser.processData (D:\zj\nodejs-server -model\node_modules\json2csv\dist\json2csv.cjs.js:1507:14)
    at JSON2CSVParser.parse (D:\zj\nodejs-server -model\node_modules\json2csv\dist\json2csv.cjs.js:1465:23)
    at writeFile (D:\zj\nodejs-server -model\app\controller\test\util.js:52:28)
    at parseTrainingData (D:\zj\nodejs-server -model\app\controller\test\testData.js:153:11)
    at process._tickCallback (internal/process/next_tick.js:68:7)

思路2:采用循环的方式批量转换成csv,然后拼接成字符串,最后将字符串写入文件(失败)

async function writeFileStream(data, fields, path) {
    const BATCH_SIZE = 1000;
    const BATCH_NUM = data.length / BATCH_SIZE;
    console.log("BATCH_NUM:", BATCH_NUM);
    let csv = '';
    const opts = {fields};
    let batch_index = 0;
    let from = batch_index * BATCH_SIZE;
    let to = from + BATCH_SIZE;
    let data_part = data.slice(from, to);
    let parser = new Parser(opts);
    let csv_part = parser.parse(data_part);
    csv += csv_part
    batch_index++;

    while (batch_index < BATCH_NUM) {
        console.log('batch_index:', batch_index)
        from = batch_index * BATCH_SIZE;
        to = from + BATCH_SIZE;
        data_part = data.slice(from, to);
        const opts2 = {fields: fields, header: false};
        parser = new Parser(opts2);
        csv_part = parser.parse(data_part);
        csv += '\n';
        csv += csv_part;
        batch_index++;
    }
    console.log('开始写入文件……')
    fs.writeFile(path,csv, 'utf8', (err) => {
        if (err) {
            console.log('写入失败')
        } else {
            console.log('写入成功')
        }
    });
}

结果:RangeError: Invalid string length

思路3:分批次转换成csv,然后采用流式写入的方式(成功)


async function writeFileStream(data, fields, path) {

    const BATCH_SIZE = 1000;
    const BATCH_NUM = data.length / BATCH_SIZE;
    console.log("BATCH_NUM:",BATCH_NUM);
    let writerStream = fs.createWriteStream(path);
    const opts = {fields};
    let batch_index = 0;
    let from = batch_index * BATCH_SIZE;
    let to = from + BATCH_SIZE;
    let data_part = data.slice(from, to);
    let parser = new Parser(opts);
    let csv_part = parser.parse(data_part);
    writerStream.write(csv_part,'utf8');
    batch_index++;

    while (batch_index < BATCH_NUM) {
        let csv = '';
        console.log('batch_index:', batch_index)
        from = batch_index  * BATCH_SIZE;
        to = from + BATCH_SIZE;
        data_part = data.slice(from, to);
        //不需要文件头
        const opts2 = {fields: fields, header: false};
        parser = new Parser(opts2);
        csv_part = parser.parse(data_part);
        //需要添加回车
        csv += '\n';
        csv += csv_part;
        writerStream.write(csv,'utf8');
        batch_index++;
    }
    // 标记文件末尾
    writerStream.end();
    // 处理流事件 --> data, end, and error
    writerStream.on('finish', function() {
        console.log("文件写入完成");
    });

}

结果:成功!!!

分析:归根结底还是字符串长度太大

评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值