Description of Problem
I have an S3 bucket that I am using to store stages of a multipart process that runs as follows:
- Get list of new objects in Database
- Request each object from Database
- Parse object from XML to JSON
- Send JSON to API
I'm currently using an EventHandler function to direct the OnCreated:* events:
exports.handler = (event, context, callback) => {
var eventString = JSON.stringify(event, null, 2)
var key = event.Records[0].s3.object.key;
var suffix = key.substr(key.lastIndexOf('.'));
var lambda = new aws.Lambda({
region: 'eu-west-1'
});
function invokeLambda(func){
console.log('Received event:', eventString);
lambda.invoke({
FunctionName: func,
Payload: eventString,
InvocationType: 'Event'
}, function(error, data) {
if (error) {
context.done('error: '+ error);
}
if(data){
context.succeed(data);
}
});
}
switch(suffix) {
case '.ppapgo':
console.log('invoking 2');
invokeLambda('2');
break;
case '.ppap':
console.log('invoking 3');
invokeLambda('3');
break;
case '.json':
console.log('invoking 4');
invokeLambda('4');
break;
}
};
Each of the files in created using S3.upload(file)
however I am getting duplicate events being created for single files. The only differences between the two events are:
- eventTime
- responseElements.x-amz-request-id
- responseElements.x-amz-id-2
- s3.object.sequencer
I know for definite the process 2 is completing successfully and a single .ppap
file is created and both instances of process 3 complete successfully without going near their time or memory limits.
Other Notes:
- The files that I seem to have problems with are at least 9KB.
- During process 2 multiple files pdf files are created.
Example Duplicate Events:
Event 1
{
"Records": [
{
"eventVersion": "2.0",
"eventSource": "aws:s3",
"awsRegion": "eu-west-1",
"eventTime": **"2017-04-11T10:22:04.663Z"**,
"eventName": "ObjectCreated:Put",
"userIdentity": {
"principalId": "AWS:SomeJunk:2"
},
"requestParameters": {
"sourceIPAddress": "IPADDRESS"
},
"responseElements": {
"x-amz-request-id": **"REQUESTID1"**,
"x-amz-id-2": **"AMZID1"**
},
"s3": {
"s3SchemaVersion": "1.0",
"configurationId": "ConfigId",
"bucket": {
"name": "MyBucket",
"ownerIdentity": {
"principalId": "PRINCIPALID"
},
"arn": "arn:aws:s3:::mybucket"
},
"object": {
"key": "filename.ppap",
"size": 10126,
"eTag": "eTag",
"sequencer": **"SEQUENCER1"**
}
}
}
]
}
Event 2:
{
"Records": [
{
"eventVersion": "2.0",
"eventSource": "aws:s3",
"awsRegion": "eu-west-1",
"eventTime": **"2017-04-11T10:22:05.133Z"**,
"eventName": "ObjectCreated:Put",
"userIdentity": {
"principalId": "AWS:SomeJunk:2"
},
"requestParameters": {
"sourceIPAddress": "IPADDRESS"
},
"responseElements": {
"x-amz-request-id": **"REQUESTID2"**,
"x-amz-id-2": **"AMZID2"**
},
"s3": {
"s3SchemaVersion": "1.0",
"configurationId": "ConfigId",
"bucket": {
"name": "MyBucket",
"ownerIdentity": {
"principalId": "PRINCIPALID"
},
"arn": "arn:aws:s3:::mybucket"
},
"object": {
"key": "filename.ppap",
"size": 10126,
"eTag": "etag",
"sequencer": **"SEQUENCER2"**
}
}
}
]
}
via Simon
No comments:
Post a Comment