diff --git a/tests/placebo/TestZappa.test_add_event_source/s3.GetBucketNotificationConfiguration_1.json b/tests/placebo/TestZappa.test_add_event_source/s3.GetBucketNotificationConfiguration_1.json new file mode 100644 index 000000000..f51bf8641 --- /dev/null +++ b/tests/placebo/TestZappa.test_add_event_source/s3.GetBucketNotificationConfiguration_1.json @@ -0,0 +1,18 @@ +{ + "status_code": 200, + "data": { + "ResponseMetadata": { + "RequestId": "6GER0T0H4R7XAV8T", + "HostId": "6EEJhlP3r8RO7cOo21OEiNjvg998YnDsnctx/hZ3ZWYMuylW2+TPTaVEW00nzTECWfb91K9jA1c=", + "HTTPStatusCode": 200, + "HTTPHeaders": { + "x-amz-id-2": "6EEJhlP3r8RO7cOo21OEiNjvg998YnDsnctx/hZ3ZWYMuylW2+TPTaVEW00nzTECWfb91K9jA1c=", + "x-amz-request-id": "6GER0T0H4R7XAV8T", + "date": "Tue, 23 Jun 2020 14:33:30 GMT", + "transfer-encoding": "chunked", + "server": "AmazonS3" + }, + "RetryAttempts": 0 + } + } +} \ No newline at end of file diff --git a/tests/placebo/TestZappa.test_add_event_source/s3.GetBucketNotificationConfiguration_2.json b/tests/placebo/TestZappa.test_add_event_source/s3.GetBucketNotificationConfiguration_2.json new file mode 100644 index 000000000..a414d54f8 --- /dev/null +++ b/tests/placebo/TestZappa.test_add_event_source/s3.GetBucketNotificationConfiguration_2.json @@ -0,0 +1,18 @@ +{ + "status_code": 200, + "data": { + "ResponseMetadata": { + "RequestId": "A1147AFA4BEF07BA", + "HostId": "Fxv8kpOjAeISFoOayckNGqtMKqYci9oe/Z2GZ3Sqoc+cGxRpmdpNhdPr8SZWmKH39HvNqDbX1qg=", + "HTTPStatusCode": 200, + "HTTPHeaders": { + "x-amz-id-2": "Fxv8kpOjAeISFoOayckNGqtMKqYci9oe/Z2GZ3Sqoc+cGxRpmdpNhdPr8SZWmKH39HvNqDbX1qg=", + "x-amz-request-id": "A1147AFA4BEF07BA", + "date": "Tue, 23 Jun 2020 14:33:30 GMT", + "transfer-encoding": "chunked", + "server": "AmazonS3" + }, + "RetryAttempts": 0 + } + } +} \ No newline at end of file diff --git a/tests/placebo/TestZappa.test_add_event_source/s3.GetBucketNotificationConfiguration_3.json b/tests/placebo/TestZappa.test_add_event_source/s3.GetBucketNotificationConfiguration_3.json new file mode 100644 index 000000000..cec02b4cd --- /dev/null +++ b/tests/placebo/TestZappa.test_add_event_source/s3.GetBucketNotificationConfiguration_3.json @@ -0,0 +1,27 @@ +{ + "status_code": 200, + "data": { + "ResponseMetadata": { + "RequestId": "68727B2ED4A5E5DB", + "HostId": "h2Tim4RpazYTXTo+NzdC8vH5gqlYtEGYHhHxjqTKRoUyFCgetU6gGc17mg/nNOF7lVxMJgBSB5s=", + "HTTPStatusCode": 200, + "HTTPHeaders": { + "x-amz-id-2": "h2Tim4RpazYTXTo+NzdC8vH5gqlYtEGYHhHxjqTKRoUyFCgetU6gGc17mg/nNOF7lVxMJgBSB5s=", + "x-amz-request-id": "68727B2ED4A5E5DB", + "date": "Tue, 23 Jun 2020 14:33:30 GMT", + "transfer-encoding": "chunked", + "server": "AmazonS3" + }, + "RetryAttempts": 0 + }, + "LambdaFunctionConfigurations": [ + { + "Id": "lambda:test_settings.callback", + "LambdaFunctionArn": "arn:aws:lambda:lambda:lambda:lambda:lambda", + "Events": [ + "s3:ObjectCreated:*" + ] + } + ] + } +} \ No newline at end of file diff --git a/tests/placebo/TestZappa.test_add_event_source/s3.GetBucketNotificationConfiguration_4.json b/tests/placebo/TestZappa.test_add_event_source/s3.GetBucketNotificationConfiguration_4.json new file mode 100644 index 000000000..86075387d --- /dev/null +++ b/tests/placebo/TestZappa.test_add_event_source/s3.GetBucketNotificationConfiguration_4.json @@ -0,0 +1,27 @@ +{ + "status_code": 200, + "data": { + "ResponseMetadata": { + "RequestId": "AB1396B752AF96FA", + "HostId": "paLR5Q18m5aW8gKTFSWY17Cw8oZWnuANlsCGFEnzTb1Vb/Ja/+vyK5GT1l+KOK0n8+ieM6MvuaM=", + "HTTPStatusCode": 200, + "HTTPHeaders": { + "x-amz-id-2": "paLR5Q18m5aW8gKTFSWY17Cw8oZWnuANlsCGFEnzTb1Vb/Ja/+vyK5GT1l+KOK0n8+ieM6MvuaM=", + "x-amz-request-id": "AB1396B752AF96FA", + "date": "Tue, 23 Jun 2020 14:33:58 GMT", + "transfer-encoding": "chunked", + "server": "AmazonS3" + }, + "RetryAttempts": 0 + }, + "LambdaFunctionConfigurations": [ + { + "Id": "lambda:test_settings.callback", + "LambdaFunctionArn": "arn:aws:lambda:lambda:lambda:lambda:lambda", + "Events": [ + "s3:ObjectCreated:*" + ] + } + ] + } +} \ No newline at end of file diff --git a/tests/placebo/TestZappa.test_add_event_source/s3.PutBucketNotificationConfiguration_1.json b/tests/placebo/TestZappa.test_add_event_source/s3.PutBucketNotificationConfiguration_1.json new file mode 100644 index 000000000..ce6fa9908 --- /dev/null +++ b/tests/placebo/TestZappa.test_add_event_source/s3.PutBucketNotificationConfiguration_1.json @@ -0,0 +1,18 @@ +{ + "status_code": 200, + "data": { + "ResponseMetadata": { + "RequestId": "7R9WFV4Q3WFWFH2W", + "HostId": "yKZnYszkf/RBcrnCHwT566JbRjTYanGk/pmujo9dXLLIMdxofz+z1srIfyFxo2mMQzPzkuH8Ybg=", + "HTTPStatusCode": 200, + "HTTPHeaders": { + "x-amz-id-2": "yKZnYszkf/RBcrnCHwT566JbRjTYanGk/pmujo9dXLLIMdxofz+z1srIfyFxo2mMQzPzkuH8Ybg=", + "x-amz-request-id": "7R9WFV4Q3WFWFH2W", + "date": "Tue, 23 Jun 2020 14:33:30 GMT", + "content-length": "0", + "server": "AmazonS3" + }, + "RetryAttempts": 0 + } + } +} \ No newline at end of file diff --git a/tests/placebo/TestZappa.test_add_event_source/s3.PutBucketNotificationConfiguration_2.json b/tests/placebo/TestZappa.test_add_event_source/s3.PutBucketNotificationConfiguration_2.json new file mode 100644 index 000000000..4447a4060 --- /dev/null +++ b/tests/placebo/TestZappa.test_add_event_source/s3.PutBucketNotificationConfiguration_2.json @@ -0,0 +1,18 @@ +{ + "status_code": 200, + "data": { + "ResponseMetadata": { + "RequestId": "61E24EFFB461E8E9", + "HostId": "sB3PWivILvgM5clr+VFuq0ymED90/wd76dk7DoRbC0u/7XsxlC9sWHT7Jvi8fF/bksgjEWiUT10=", + "HTTPStatusCode": 200, + "HTTPHeaders": { + "x-amz-id-2": "sB3PWivILvgM5clr+VFuq0ymED90/wd76dk7DoRbC0u/7XsxlC9sWHT7Jvi8fF/bksgjEWiUT10=", + "x-amz-request-id": "61E24EFFB461E8E9", + "date": "Tue, 23 Jun 2020 14:33:58 GMT", + "content-length": "0", + "server": "AmazonS3" + }, + "RetryAttempts": 0 + } + } +} \ No newline at end of file diff --git a/tests/tests_placebo.py b/tests/tests_placebo.py index 47e477173..2d769746d 100644 --- a/tests/tests_placebo.py +++ b/tests/tests_placebo.py @@ -481,11 +481,9 @@ def test_add_event_source(self, session): except ValueError: pass - event_source = {'arn': 's3:s3:s3:s3', 'events': [ - "s3:ObjectCreated:*" - ]} - add_event_source(event_source, 'lambda:lambda:lambda:lambda', 'test_settings.callback', session, dry=True) - remove_event_source(event_source, 'lambda:lambda:lambda:lambda', 'test_settings.callback', session, dry=True) + event_source = {'arn': 's3:s3:s3:s3', 'events': ["s3:ObjectCreated:*"], 'key_filters': [{'type': 'prefix', 'value': 'value'}]} + add_event_source(event_source, 'lambda:lambda:lambda:lambda', 'test_settings.callback', session, dry=False) + remove_event_source(event_source, 'lambda:lambda:lambda:lambda', 'test_settings.callback', session, dry=False) # get_event_source_status(event_source, 'lambda:lambda:lambda:lambda', 'test_settings.callback', session, dry=True) @placebo_session diff --git a/zappa/utilities.py b/zappa/utilities.py index 49c3b3351..541f842a9 100644 --- a/zappa/utilities.py +++ b/zappa/utilities.py @@ -351,10 +351,83 @@ def add(self, function): if self.filters: self.add_filters(function) + # This class allows for correct addition, updates, and deletes of S3 trigger events when there are multiple events + # on a bucket that may be tied to different stages with different filters by limiting changes only to events tied + # to the targeted method and stage unlike the handler in Kappa. + # Related: https://github.com/Miserlou/Zappa/issues/2111 + # This entire class is reimplemented rather than extending the Kappa version because only the constructor is reused + # and I didn't want any future usages or extentions to accidentally fall through to Kappa's object via one of the + # 'aliases' it sets up. + class ImprovedS3EventSource(kappa.event_source.base.EventSource): + def __init__(self, context, config): + super(ImprovedS3EventSource, self).__init__(context, config) + self._s3 = kappa.awsclient.create_client('s3', context.session) + + def _get_bucket_name(self): + return self.arn.split(':')[-1] + + # this method is overridden / implemented below but I wanted the placeholder here for clarity and to make IDEs + # less cranky + def _make_notification_id(self, function_name): + raise NotImplementedError + + def add(self, function): + event_config = { + 'Id': self._make_notification_id(function.name), + 'Events': [e for e in self._config['events']], + 'LambdaFunctionArn': '%s:%s' % (function.arn, function._context.environment), + } + + # Add S3 key filters + if 'key_filters' in self._config: + filters_spec = {'Key': {'FilterRules': []}} + for key_filter in self._config['key_filters']: + if 'type' in key_filter and 'value' in key_filter and key_filter['type'] in ('prefix', 'suffix'): + rule = {'Name': key_filter['type'], 'Value': key_filter['value']} + filters_spec['Key']['FilterRules'].append(rule) + event_config['Filter'] = filters_spec + + try: + bucket_config = self._s3.call('get_bucket_notification_configuration', Bucket=self._get_bucket_name()) + del bucket_config['ResponseMetadata'] + if 'LambdaFunctionConfigurations' in bucket_config: + bucket_config['LambdaFunctionConfigurations'].append(event_config) + else: + bucket_config['LambdaFunctionConfigurations'] = [event_config] + response = self._s3.call( + 'put_bucket_notification_configuration', + Bucket=self._get_bucket_name(), + NotificationConfiguration=bucket_config) + LOG.debug(response) + except Exception as exc: + LOG.debug(exc.response) + LOG.exception('Unable to add S3 event source') + + def remove(self, function): + LOG.debug('removing s3 notification') + bucket_config = self._s3.call('get_bucket_notification_configuration', Bucket=self._get_bucket_name()) + LOG.debug(bucket_config) + + new_config = [] + for configuration in bucket_config.get('LambdaFunctionConfigurations', []): + if configuration['Id'] != self._make_notification_id(function.name): + new_config.append(configuration) + response = self._s3.call('put_bucket_notification_configuration', Bucket=self._get_bucket_name(), + NotificationConfiguration={'LambdaFunctionConfigurations': new_config}) + LOG.debug(response) + + def status(self, function): + LOG.debug('status for s3 notification for %s', function.name) + bucket_config = self._s3.call('get_bucket_notification_configuration', Bucket=self._get_bucket_name()) + LOG.debug(bucket_config) + for configuration in bucket_config.get('LambdaFunctionConfigurations', []): + if configuration['Id'] == self._make_notification_id(function.name): + return {'LambdaFunctionConfiguration': configuration, 'State': 'Enabled'} + event_source_map = { 'dynamodb': kappa.event_source.dynamodb_stream.DynamoDBStreamEventSource, 'kinesis': kappa.event_source.kinesis.KinesisEventSource, - 's3': kappa.event_source.s3.S3EventSource, + 's3': ImprovedS3EventSource, 'sns': ExtendedSnsEventSource, 'sqs': SqsEventSource, 'events': kappa.event_source.cloudwatch.CloudWatchEventSource