diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d4cc4dd --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +.env +**/*.pyc +lambder.json diff --git a/README.md b/README.md new file mode 100644 index 0000000..0dcf219 --- /dev/null +++ b/README.md @@ -0,0 +1,44 @@ +# lambder-replicate-snapshots + +replicate-snapshots is an AWS Lambda function for use with Lambder. + +REQUIRES: +* python-lambder + +## Getting Started + +1) Test the sample lambda function + + python lambda/replicate-snapshots/replicate-snapshots.py + +2) Deploy the sample Lambda function to AWS + + lambder functions deploy + +3) Invoke the sample Lambda function in AWS + + lambder functions invoke --input input/ping.json + +4) Add useful code to lambda/replicate-snapshots/replicate-snapshots.py + +5) Add any permissions you need to access other AWS resources to iam/policy.json + +6) Update your lambda and permissions policy in one go + + lambder functions deploy + +## Sharing your lambder function + +If you decide to share your lambder function, you want to be sure you don't share +the name of your s3 bucket. We suggest you add `lambder.json` to your +`.gitignore` so it won't be commited to your repo. Instead, copy it to +`example_lambder.json` and remove any secrets before pushing to a public +repository. + +## Using virtualenvwrapper + +Your Lambdas should be as small as possible to reduce spinup time. If you need +to include extra python modules, use virtualenvwrapper. +The deploy script will look for a site-packages directory in +$WORKON_HOME/lambder-replicate-snapshots and bundle those packages into the zip +that it uploads to AWS Lambda. diff --git a/example_lambder.json b/example_lambder.json new file mode 100644 index 0000000..ff5cf5d --- /dev/null +++ b/example_lambder.json @@ -0,0 +1,7 @@ +{ + "name": "replicate-snapshots", + "s3_bucket": "devopsbucket", + "timeout": 30, + "memory": 128, + "description": "" +} diff --git a/iam/policy.json b/iam/policy.json new file mode 100644 index 0000000..7b9b0be --- /dev/null +++ b/iam/policy.json @@ -0,0 +1,23 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents" + ], + "Resource": "arn:aws:logs:*:*:*" + }, + { + "Effect": "Allow", + "Action": [ + "ec2:DescribeSnapshots", + "ec2:CopySnapshot", + "ec2:CreateTags" + ], + "Resource": "*" + } + ] +} diff --git a/input/ping.json b/input/ping.json new file mode 100644 index 0000000..a2dc0d3 --- /dev/null +++ b/input/ping.json @@ -0,0 +1,3 @@ +{ + "ping": true +} diff --git a/lambda/replicate-snapshots/config.json b/lambda/replicate-snapshots/config.json new file mode 100644 index 0000000..7de62e8 --- /dev/null +++ b/lambda/replicate-snapshots/config.json @@ -0,0 +1,4 @@ +{ + "AWS_SOURCE_REGION": "us-east-1", + "AWS_DEST_REGION": "us-west-2" +} diff --git a/lambda/replicate-snapshots/replicate-snapshots.py b/lambda/replicate-snapshots/replicate-snapshots.py new file mode 100644 index 0000000..67a5273 --- /dev/null +++ b/lambda/replicate-snapshots/replicate-snapshots.py @@ -0,0 +1,44 @@ +import logging +from replicator import Replicator + +logger = logging.getLogger() +logger.setLevel(logging.INFO) +# logger.setLevel(logging.DEBUG) + +replicator = Replicator() + +# This is the method that will be registered +# with Lambda and run on a schedule +# This is the method that will be registered +# with Lambda and run on a schedule +def handler(event={}, context={}): + if 'ping' in event: + logger.info('pong') + return {'message': 'pong'} + + replicator.run() + +# If being called locally, just call handler +if __name__ == '__main__': + import os + import json + import sys + + logging.basicConfig() + event = {} + + # TODO if argv[1], read contents, parse into json + if len(sys.argv) > 1: + input_file = sys.argv[1] + with open(input_file, 'r') as f: + data = f.read() + event = json.loads(data) + + result = handler(event) + output = json.dumps( + result, + sort_keys=True, + indent=4, + separators=(',', ':') + ) + logger.info(output) diff --git a/lambda/replicate-snapshots/replicator.py b/lambda/replicate-snapshots/replicator.py new file mode 100644 index 0000000..74d5d79 --- /dev/null +++ b/lambda/replicate-snapshots/replicator.py @@ -0,0 +1,85 @@ +import boto3 +import logging +import pprint +import os +import os.path +import json +from datetime import datetime + +class Replicator: + + REPLICATE_TAG = "LambderReplicate" + BACKUP_TAG = "LambderBackup" + + def __init__(self): + logging.basicConfig() + self.logger = logging.getLogger() + + # set location of config file + script_dir = os.path.dirname(__file__) + config_file = script_dir + '/config.json' + + # if there is a config file in place, load it in. if not, bail. + if not os.path.isfile(config_file): + self.logger.error(config_file + " does not exist") + exit(1) + else: + config_data=open(config_file).read() + config_json = json.loads(config_data) + self.AWS_SOURCE_REGION=config_json['AWS_SOURCE_REGION'] + self.AWS_DEST_REGION=config_json['AWS_DEST_REGION'] + + self.ec2_source = boto3.resource('ec2', region_name=self.AWS_SOURCE_REGION) + self.ec2_dest = boto3.resource('ec2', region_name=self.AWS_DEST_REGION) + + def get_source_snapshots(self): + filters = [{'Name':'tag-key', 'Values': [self.REPLICATE_TAG]}] + snapshots = self.ec2_source.snapshots.filter(Filters=filters) + return snapshots + + def get_dest_snapshots(self,snapid,backupname): + filters = [{'Name':'description', 'Values': [self.AWS_SOURCE_REGION+'_'+snapid+'_'+backupname]}] + snapshots = self.ec2_dest.snapshots.filter(Filters=filters) + return snapshots + + # Takes an snapshot or volume, returns the backup source + def get_backup_source(self, resource): + tags = filter(lambda x: x['Key'] == self.BACKUP_TAG, resource.tags) + + if len(tags) < 1: + return None + + return tags[0]['Value'] + + def copy_snapshot(self,snapshot): + sourcesnapid=snapshot.snapshot_id + sourcebackupname=self.get_backup_source(snapshot) + self.logger.info("Looking for existing replicas of snapshot {0}".format(sourcesnapid)) + dest_snapshots=self.get_dest_snapshots(sourcesnapid,sourcebackupname) + dest_snapshot_count = len(list(dest_snapshots)) + if dest_snapshot_count != 0: + self.logger.info("Replica found, no need to copy snapshot") + else: + self.logger.info("No replica found, copying snapshot {0}".format(sourcesnapid)) + sourcesnap = self.ec2_dest.Snapshot(sourcesnapid) + dest_snap_description=self.AWS_SOURCE_REGION+'_'+sourcesnapid+'_'+sourcebackupname + copy_output=sourcesnap.copy(DryRun=False,SourceRegion=self.AWS_SOURCE_REGION,SourceSnapshotId=sourcesnapid,Description=dest_snap_description) + destsnapid=copy_output['SnapshotId'] + destsnap = self.ec2_dest.Snapshot(destsnapid) + destsnap.create_tags(Tags=[ + {'Key': self.REPLICATE_TAG, 'Value': dest_snap_description}, + {'Key': self.BACKUP_TAG, 'Value': sourcebackupname}]) + + def copy_snapshots(self,snapshots): + for snapshot in snapshots: + self.copy_snapshot(snapshot) + + def run(self): + + # replicate any snapshots that need to be replicated + source_snapshots = self.get_source_snapshots() + source_snapshot_count = len(list(source_snapshots)) + + self.logger.info("Found {0} source snapshots".format(source_snapshot_count)) + + self.copy_snapshots(source_snapshots)