Friday 14 April 2017

EC2 Stream yields empty buffer error” with large size images

I'm using nodejs 'gm' module with imagemagick library to create thumbnails of images and upload on S3. It works well with small images. But with large size images it's not working at all. In fact it return error,

Unhandled rejection Error: Stream yields empty buffer at Socket. (/var/task/node_modules/gm/lib/command.js:57:17) at emitNone (events.js:72:20) at Socket.emit (events.js:166:7) at endReadableNT (_stream_readable.js:905:12) at nextTickCallbackWith2Args (node.js:437:9) at process._tickDomainCallback (node.js:392:17)

I have researched alot and applied all tricks like install some dependencies like libjpeg-dev and libpng-dev. I also installed graphicsmagick but it didn't work with it either. 

When i run 'gm version' command it gives following result:

Feature Support:
  Native Thread Safe       yes
  Large Files (> 32 bit)   yes
  Large Memory (> 32 bit)  yes
  BZIP                     yes
  DPS                      no
  FlashPix                 no
  FreeType                 yes
  Ghostscript (Library)    no
  JBIG                     yes
  JPEG-2000                yes
  JPEG                     yes
  Little CMS               yes
  Loadable Modules         no
  OpenMP                   yes (201307)
  PNG                      yes
  TIFF                     yes
  TRIO                     no
  UMEM                     no
  WebP                     yes
  WMF                      yes
  X11                      yes
  XML                      yes
  ZLIB                     yes

Host type: x86_64-pc-linux-gnu

Configured using the command:
  ./configure  '--build' 'x86_64-linux-gnu' '--enable-shared' '--enable-static' '--enable-libtool-verbose' '--prefix=/usr' '--mandir=${prefix}/share/man' '--infodir=${prefix}/share/info' '--docdir=${prefix}/share/doc/graphicsmagick' '--with-gs-font-dir=/usr/share/fonts/type1/gsfonts' '--with-x' '--x-includes=/usr/include/X11' '--x-libraries=/usr/lib/X11' '--without-dps' '--without-modules' '--without-frozenpaths' '--with-webp' '--with-perl' '--with-perl-options=INSTALLDIRS=vendor' '--enable-quantum-library-names' '--with-quantum-depth=16' 'build_alias=x86_64-linux-gnu' 'CFLAGS=-Wall -g -fno-strict-aliasing -O2' 'LDFLAGS=' 'CXXFLAGS=-Wall -g -fno-strict-aliasing -O2'

Final Build Parameters:
  CC       = gcc
  CFLAGS   = -fopenmp -Wall -g -fno-strict-aliasing -O2 -Wall -pthread
  CPPFLAGS = -I/usr/include/X11 -I/usr/include/freetype2 -I/usr/include/libxml2
  CXX      = g++
  CXXFLAGS = -Wall -g -fno-strict-aliasing -O2 -pthread
  LDFLAGS  = -L/usr/lib/X11
  LIBS     = -ljbig -lwebp -llcms2 -ltiff -lfreetype -ljasper -ljpeg -lpng12 -lwmflite -lXext -lSM -lICE -lX11 -llzma -lbz2 -lxml2 -lz -lm -lgomp -lpthread

I also read about setting the maximum size [here][1] but i don't know how to do it. Following is my code:

var _200px = { width: 200, dstnKey: srckey, destinationPath: "small" }; var _45px = { width: 45, dstnKey: srckey, destinationPath: "thumbnail" }; var _sizesArray = [_200px, _45px];

            async.forEachOf(_sizesArray, function(value, key, callback) {
                    async.waterfall([
                        function download(next) {
                            // Download the image from S3 into a buffer.

                            var data = ''
                            var params = {Bucket: 'meetupimages', Key: srckey};
                            s3.getObject(
                                params
                                , next);
                        },
                        function transform(response, next) {

                            // Transform the image buffer in memory.
                            gm(response.Body, srckey)
                                .resize(_sizesArray[key].width, _sizesArray[key].width)
                                .toBuffer(imageType, function (err, buffer) {
                                    if (err) {
                                        next(err);

                                    } else {
                                        next(null, response.ContentType, buffer);
                                    }
                                });

                        },
                        function upload(contentType, data, next) {

                            // Stream the transformed image to a different folder.
                            s3.upload({
                                    Bucket: 'meetupimages',
                                    //    Key: req.user._id+'/files/'+file.filename +'_'+ _sizesArray[key].destinationPath+extension,
                                    Key: srckey + '/' + _sizesArray[key].destinationPath,
                                    Body: data,
                                    ACL: "public-read",
                                    ContentType: contentType
                                },
                                next);

                        } 

Please help me solve this issue. Thanks



via Aneela Saleem Ramzan

No comments:

Post a Comment