Wednesday, 19 April 2017

Out of memory trying to create large csv file

Using Meteor and on the server try to generate a large csv file by looping through a Meteor collection and inserting a row. At some point the server will get an out of memory error - my guess is I run out of memory before the loop finishes, depending on the collection size. How can I fix this (clear the memory somehow)? Here's the code:

    var job = Jobs.findOne();
    var fs = Npm.require('fs');
    var file = '/tmp/csv-' + job._id + '.csv';
    var headers = ["Email", "Processed?", "Integration", "Passed?", "Reason", "Date"];
    var stream = fs.createWriteStream(file);
    var first_line = headers.join() + '\n';
    var wstream = fs.createWriteStream(file);
    var emails = rawEmails.find();
    wstream.write(first_line);
    emails.forEach(function(rawemail) {
        var line_item = [];
        line_item.push(rawemail.email);
        if (rawemail.processed === true || rawemail.processed === false)
            line_item.push(rawemail.processed);
        if (rawemail.integration)
            line_item.push(rawemail.integration);
        if (rawemail.passed === true || rawemail.passed === false)
            line_item.push(rawemail.passed);
        if (rawemail.reason)
            line_item.push(rawemail.reason);
        if (rawemail.updated_at)
            line_item.push(rawemail.updated_at);
        var to_write = line_item.join() + '\n';
        wstream.write(to_write);
    });
    wstream.end();



via landland

No comments:

Post a Comment