Tuesday, 11 April 2017

Duplicate S3 OnCreated Events - Lambda not expiring

Description of Problem

I have an S3 bucket that I am using to store stages of a multipart process that runs as follows:

  1. Get list of new objects in Database
  2. Request each object from Database
  3. Parse object from XML to JSON
  4. Send JSON to API

I'm currently using an EventHandler function to direct the OnCreated:* events:

exports.handler = (event, context, callback) => {
    var eventString = JSON.stringify(event, null, 2)

    var key = event.Records[0].s3.object.key;
    var suffix = key.substr(key.lastIndexOf('.'));

    var lambda = new aws.Lambda({
        region: 'eu-west-1'
    });

    function invokeLambda(func){
        console.log('Received event:', eventString);
        lambda.invoke({
            FunctionName: func,
            Payload: eventString,
            InvocationType: 'Event'
        }, function(error, data) {
          if (error) {
            context.done('error: '+ error);
          }
          if(data){
              context.succeed(data);
          }
        });
    }

    switch(suffix) {
    case '.ppapgo':
        console.log('invoking 2');
        invokeLambda('2');
        break;
    case '.ppap':
        console.log('invoking 3');
        invokeLambda('3');
        break;
    case '.json':
        console.log('invoking 4');
        invokeLambda('4');
        break;
    }
};

Each of the files in created using S3.upload(file) however I am getting duplicate events being created for single files. The only differences between the two events are:

  • eventTime
  • responseElements.x-amz-request-id
  • responseElements.x-amz-id-2
  • s3.object.sequencer

I know for definite the process 2 is completing successfully and a single .ppap file is created and both instances of process 3 complete successfully without going near their time or memory limits.

Other Notes:

  • The files that I seem to have problems with are at least 9KB.
  • During process 2 multiple files pdf files are created.

Example Duplicate Events:

Event 1

{
    "Records": [
        {
            "eventVersion": "2.0",
            "eventSource": "aws:s3",
            "awsRegion": "eu-west-1",
            "eventTime": **"2017-04-11T10:22:04.663Z"**,
            "eventName": "ObjectCreated:Put",
            "userIdentity": {
                "principalId": "AWS:SomeJunk:2"
            },
            "requestParameters": {
                "sourceIPAddress": "IPADDRESS"
            },
            "responseElements": {
                "x-amz-request-id": **"REQUESTID1"**,
                "x-amz-id-2": **"AMZID1"**
            },
            "s3": {
                "s3SchemaVersion": "1.0",
                "configurationId": "ConfigId",
                "bucket": {
                    "name": "MyBucket",
                    "ownerIdentity": {
                        "principalId": "PRINCIPALID"
                    },
                    "arn": "arn:aws:s3:::mybucket"
                },
                "object": {
                    "key": "filename.ppap",
                    "size": 10126,
                    "eTag": "eTag",
                    "sequencer": **"SEQUENCER1"**
                }
            }
        }
    ]
}

Event 2:

{
    "Records": [
        {
            "eventVersion": "2.0",
            "eventSource": "aws:s3",
            "awsRegion": "eu-west-1",
            "eventTime": **"2017-04-11T10:22:05.133Z"**,
            "eventName": "ObjectCreated:Put",
            "userIdentity": {
                "principalId": "AWS:SomeJunk:2"
            },
            "requestParameters": {
                "sourceIPAddress": "IPADDRESS"
            },
            "responseElements": {
                "x-amz-request-id": **"REQUESTID2"**,
                "x-amz-id-2": **"AMZID2"**
            },
            "s3": {
                "s3SchemaVersion": "1.0",
                "configurationId": "ConfigId",
                "bucket": {
                    "name": "MyBucket",
                    "ownerIdentity": {
                        "principalId": "PRINCIPALID"
                    },
                    "arn": "arn:aws:s3:::mybucket"
                },
                "object": {
                    "key": "filename.ppap",
                    "size": 10126,
                    "eTag": "etag",
                    "sequencer": **"SEQUENCER2"**
                }
            }
        }
    ]
}



via Simon

No comments:

Post a Comment