Thursday 1 June 2017

Node.js AWS S3 Hangs Indefinitely on Large Number of Image File getObject Requests

I have an app requesting up to few hundred image files from Amazon S3 at a time. The files are requested in sequence using nested promises (so it is not an issue with some limitation on the number of requests per second or the amount of data that can be transferred at once), and everything goes fine up to a little over 100 files. However, S3 hangs indefinitely thereafter. How do I fix this? See relevant code below.

function one_by_one(objects_array, iterator, callback) {
    let start_promise = objects_array.reduce(function (prom, object) {
        return prom.then(function () {
            return iterator(object);
        });
    }, Promise.resolve()); // initial
    if(callback){
        start_promise.then(callback);
    }else{
        return start_promise;
    }
}

let grabImage = function(image){
      params = {Bucket: 'partpics', Key: image};
      let getObjectPromise = s3.getObject(params).promise();
      return getObjectPromise.then(function(data) {
          console.log('Success');
          res.imageBuffers.push(data.Body);
          bufferTemp = res.imageBuffers;
          console.log('res.imageBuffers index: ',res.imageBuffers.length-1);
      }).catch(function(err) {
        console.log(err);
      });

};
let myCallback = function() {
  next();
}
one_by_one(res.insertThis.Image,grabImage,myCallback);



via 223seneca

No comments:

Post a Comment