Saturday, 15 April 2017

How to extract tar gz files from s3 bucket and upload the files into another bucket using node js

I need to extract tar.gz file from S3 bucket.Then I need to Iterate the files and upload into s3 bucket without storing the files in temp folder, instead we can use buffer or streams.

In my code,I have tried to call the upload() method while streaming,But the data is coming null.Bcoz data will be returning in "finish" process only.

How to iterate the files and where to call the upload method and how to pass the files

var bucketName = "my.new.SampleBucket";

var fileKey = "Employee.tar.gz";

var params = { Bucket: bucketName, Key: fileKey };

function ExtractZipFileFromS3() {
var tar = require('tar-stream');
var fs = require('fs');
var zlib = require('zlib');
var extract = tar.extract();
var data = '';
var fileName = '';
var folderName = '';

extract.on('entry', function (header, stream, cb) 
    {
    stream.on('data', function (chunk) 
          {
        //if (header.name == 'student/saravt.txt')
        //    data += chunk;

        var filePath = header.name;

        fileName = path.basename(header.name);

        folderName = path.dirname(header.name);

        UploadFilestoS3(folderName, fileName);

        });

    stream.on('end', function () {
        cb();
    });

    stream.resume();
});

extract.on('finish', function () {

    var buf = new Buffer();
    var kk= fileName
    buf = data;

});

  s3.getObject(params).createReadStream()
  .pipe(zlib.createGunzip())
  .pipe(extract);
 }

function UploadFilestoS3(folder,fileName)
{
var filestream = fs.createReadStream(fileName);

s3.upload({
    Key: folder + fileName
    , Body: filestream
    // change contentType as needed
    , ACL: 'public-read'
}
                  , function (err, result) {
    if (err)
        throw err;


});

}



via vishnu

No comments:

Post a Comment