Saturday 6 May 2017

downloading a large file with nodejs and my process is hanging

I have a function that downloads a file from an ftp directory with credentials. All the log messages fire as expected but my process never closes. The zip file is about a gigabyte.

Here is the function

var pullZip = function (file_url, username, password, file_name) {

  logger.info('Starting to request zip from: ', file_url);

  var deferred = Q.defer();

  var ws = fs.createWriteStream(__dirname + file_name);

  ws.on('error', function(err) { 
      logger.error(err);
      deferred.reject(err);
  });

  var zipRequest = request({
      url: file_url
      , encoding: null
      , auth: {
          username: username
          , password: password
      }
  });

  zipRequest.on('error', function(err) {
      logger.error(err);
      deferred.reject(err);
  });

  zipRequest.on('response', function(response) {

      logger.info('Response status was ' + response.statusCode);
      logger.info('Started downloading zipfile.');

      response.pipe(ws).on('finish', function() {
        logger.info('Finished writestream.');
        ws.close(function(){
              logger.info('Closing write stream.');
              deferred.resolve();
          });  // close() is async, call cb after close completes.
      });

  });

  return deferred.promise;
};

var pullZipDeferred = pullZip('https://mylocation/dir/file1.zip', 'user', 'pass', 'file1.zip').then(function(response){
    logger.info('done.');
}, function(reason){
    logger.info('done. but with problems.');
});

console spits out

2017-05-07T02:31:11.438Z - info: Starting to request zip from:  https://mylocation/dir/file1.zip
2017-05-07T02:31:12.011Z - info: Response status was 200
2017-05-07T02:33:21.917Z - info: Finished writestream.
2017-05-07T02:33:21.929Z - info: Closing write stream.
2017-05-07T02:33:21.931Z - info: done.

.. the process never exits (I'm not sure how to figure out what's still open) .. and the file1.zip is of 0 bytes.

Any ideas on how to track this one down?



via Kirby

No comments:

Post a Comment