Thursday, 27 April 2017

Uploading multiple files to AWS S3 using NodeJS

I'm trying to upload all files within my directory to my S3 instance using NodeJS. I'm able to upload one file at a time if I explicitly give the file path + literal string for the Key: field.

Below is the script I'm using:

var AWS = require('aws-sdk'),
    fs = require('fs');

// For dev purposes only
AWS.config.update({ accessKeyId: '...', secretAccessKey: '...' });

// reg ex to match
var re = /\.txt$/;

// ensure that this file is in the directory of the files you want to run the cronjob on
fs.readdir(".", function(err, files) {
    if (err) {
    console.log( "Could not list the directory.", err)
    process.exit( 1 )
    }


    var matches = files.filter( function(text) { return re.test(text) } )
    // make one pass and print the files in the directoery
    // var matchedfiles = re.exec(files)
    numFiles = matches.length


    if ( numFiles ) {
        // Read in the file, convert it to base64, store to S3

        for( i = 0; i < numFiles; i++ ) {
            fs.readFile(matches[i], function (err, data) {
                if (err) { throw err; }

                // Buffer Pattern; how to handle buffers; straw, intake/outtake analogy
                var base64data = new Buffer(data, 'binary')


                var s3 = new AWS.S3()
                    s3.putObject({
                       Bucket: 'noonebetterhaventakenthisbucketnname',
                        Key: matches[i],
                        Body: base64data,
                        ACL: 'public-read'
                     }, function (resp) {
                        console.log(arguments);
                        console.log('Successfully uploaded package.')
                    })
            })
        }
    }
})

It produces this error for each file attempted to upload to S3:

{ '0': 
   { MissingRequiredParameter: Missing required key 'Key' in params
       at ParamValidator.fail (/Users/stevenle/Documents/aws-nodejs/node_modules/aws-sdk/lib/param_validator.js:50:37)
       at ParamValidator.validateStructure (/Users/stevenle/Documents/aws-nodejs/node_modules/aws-sdk/lib/param_validator.js:61:14)
       at ParamValidator.validateMember (/Users/stevenle/Documents/aws-nodejs/node_modules/aws-sdk/lib/param_validator.js:88:21)
       at ParamValidator.validate (/Users/stevenle/Documents/aws-nodejs/node_modules/aws-sdk/lib/param_validator.js:34:10)
       at Request.VALIDATE_PARAMETERS (/Users/stevenle/Documents/aws-nodejs/node_modules/aws-sdk/lib/event_listeners.js:114:42)
       at Request.callListeners (/Users/stevenle/Documents/aws-nodejs/node_modules/aws-sdk/lib/sequential_executor.js:105:20)
       at callNextListener (/Users/stevenle/Documents/aws-nodejs/node_modules/aws-sdk/lib/sequential_executor.js:95:12)
       at /Users/stevenle/Documents/aws-nodejs/node_modules/aws-sdk/lib/event_listeners.js:74:9
       at finish (/Users/stevenle/Documents/aws-nodejs/node_modules/aws-sdk/lib/config.js:313:7)
       at /Users/stevenle/Documents/aws-nodejs/node_modules/aws-sdk/lib/config.js:331:9
     message: 'Missing required key \'Key\' in params',
     code: 'MissingRequiredParameter',
     time: 2017-04-27T15:56:41.169Z },
  '1': null }
Successfully uploaded package.

How do I get the Key value to take in an iterative variable of file names?



via Steven_

No comments:

Post a Comment