Friday, 28 April 2017

readline does not work inside forEach

I do have the problem that I need to process a huge amount of csv files. For this I need to read these file line by line.

I have created the following basic setup:

const testFolder = '/path/to/files';

var adapterFile = 'file1.csv';
var adapterFile2 = 'file2.csv';

const fs = require('fs');
const readline = require('readline');
const path = require('path');
const Promise = require('bluebird');
const glob = require("glob");
const winston = require('winston');

var fileProcessor = function(){};
var processingCount = 0;
var rls = [];
var streams = [];

var logfile = path.join(__dirname, 'test.log');
var logger = new (winston.Logger)({
    transports: [
      new (winston.transports.Console)(),
      new (winston.transports.File)({ filename: logfile })
    ]
});

fileProcessor.prototype.processFile = function(filePath) {

    streams[processingCount] = fs.createReadStream(filePath);

    rls[processingCount] = readline.createInterface({
        input: streams[processingCount]
    });     

    rls[processingCount].on('line', function (line) {
        logger.info('in on line');          
    });

    rls[processingCount].on('close', function (line) {      
        logger.info('in on close');

    }); 
    processingCount++;  
}

var start = function start() {

    fs.readdir(testFolder, function(err, files) {
        files
        .forEach(function(file) { 
            if (file.indexOf('MyTrigger') != -1) {
                var fileProzzi = new fileProcessor();
                fileProzzi.processFile(testFolder + file);
                console.log(rls.length);
            }
        })
    })

}
var fileProzzi = new fileProcessor();
fileProzzi.processFile(testFolder + adapterFile);
var fileProzzi = new fileProcessor();
fileProzzi.processFile(testFolder + adapterFile2);
// start();

If I start the processes manually as you can see above everything works as expected. But if call start() to process all files the files are never read.

Can someone tell me what I'm doing wrong?

Thanks in advance.



via André

No comments:

Post a Comment