working version of replicate-snapshots lambder

.gitignore

@@ -0,0 +1,3 @@
+.env
+**/*.pyc
+lambder.json

README.md

@@ -0,0 +1,44 @@
+# lambder-replicate-snapshots
+
+replicate-snapshots is an AWS Lambda function for use with Lambder.
+
+REQUIRES:
+* python-lambder
+
+## Getting Started
+
+1) Test the sample lambda function
+
+    python lambda/replicate-snapshots/replicate-snapshots.py
+
+2) Deploy the sample Lambda function to AWS
+
+    lambder functions deploy
+
+3) Invoke the sample Lambda function in AWS
+
+    lambder functions invoke --input input/ping.json
+
+4) Add useful code to lambda/replicate-snapshots/replicate-snapshots.py
+
+5) Add any permissions you need to access other AWS resources to iam/policy.json
+
+6) Update your lambda and permissions policy in one go
+
+    lambder functions deploy
+
+## Sharing your lambder function
+
+If you decide to share your lambder function, you want to be sure you don't share
+the name of your s3 bucket. We suggest you add `lambder.json` to your
+`.gitignore` so it won't be commited to your repo. Instead, copy it to
+`example_lambder.json` and remove any secrets before pushing to a public
+repository.
+
+## Using virtualenvwrapper
+
+Your Lambdas should be as small as possible to reduce spinup time. If you need
+to include extra python modules, use virtualenvwrapper.
+The deploy script will look for a site-packages directory in
+$WORKON_HOME/lambder-replicate-snapshots and bundle those packages into the zip
+that it uploads to AWS Lambda.

example_lambder.json

@@ -0,0 +1,7 @@
+{
+  "name": "replicate-snapshots",
+  "s3_bucket": "devopsbucket",
+  "timeout": 30,
+  "memory": 128,
+  "description": ""
+}

iam/policy.json

@@ -0,0 +1,23 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": [
+        "logs:CreateLogGroup",
+        "logs:CreateLogStream",
+        "logs:PutLogEvents"
+      ],
+      "Resource": "arn:aws:logs:*:*:*"
+    },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "ec2:DescribeSnapshots",
+        "ec2:CopySnapshot",
+        "ec2:CreateTags"
+      ],
+      "Resource": "*"
+    }
+  ]
+}

input/ping.json

@@ -0,0 +1,3 @@
+{
+  "ping": true
+}

lambda/replicate-snapshots/config.json

@@ -0,0 +1,4 @@
+{
+  "AWS_SOURCE_REGION": "us-east-1",
+  "AWS_DEST_REGION": "us-west-2"
+}

lambda/replicate-snapshots/replicate-snapshots.py

@@ -0,0 +1,44 @@
+import logging
+from replicator import Replicator
+
+logger = logging.getLogger()
+logger.setLevel(logging.INFO)
+# logger.setLevel(logging.DEBUG)
+
+replicator = Replicator()
+
+# This is the method that will be registered
+# with Lambda and run on a schedule
+# This is the method that will be registered
+# with Lambda and run on a schedule
+def handler(event={}, context={}):
+  if 'ping' in event:
+    logger.info('pong')
+    return {'message': 'pong'}
+
+  replicator.run()
+
+# If being called locally, just call handler
+if __name__ == '__main__':
+  import os
+  import json
+  import sys
+
+  logging.basicConfig()
+  event = {}
+
+  # TODO if argv[1], read contents, parse into json
+  if len(sys.argv) > 1:
+    input_file = sys.argv[1]
+    with open(input_file, 'r') as f:
+      data = f.read()
+    event = json.loads(data)
+
+  result = handler(event)
+  output = json.dumps(
+    result,
+    sort_keys=True,
+    indent=4,
+    separators=(',', ':')
+  )
+  logger.info(output)

lambda/replicate-snapshots/replicator.py

@@ -0,0 +1,85 @@
+import boto3
+import logging
+import pprint
+import os
+import os.path
+import json
+from datetime import datetime
+
+class Replicator:
+
+  REPLICATE_TAG = "LambderReplicate"
+  BACKUP_TAG = "LambderBackup"
+
+  def __init__(self):
+    logging.basicConfig()
+    self.logger = logging.getLogger()
+
+    # set location of config file
+    script_dir = os.path.dirname(__file__)
+    config_file = script_dir + '/config.json'
+
+    # if there is a config file in place, load it in. if not, bail.
+    if not os.path.isfile(config_file):
+      self.logger.error(config_file + " does not exist")
+      exit(1)
+    else:
+      config_data=open(config_file).read()
+      config_json = json.loads(config_data)
+      self.AWS_SOURCE_REGION=config_json['AWS_SOURCE_REGION']
+      self.AWS_DEST_REGION=config_json['AWS_DEST_REGION']
+
+    self.ec2_source = boto3.resource('ec2', region_name=self.AWS_SOURCE_REGION)
+    self.ec2_dest = boto3.resource('ec2', region_name=self.AWS_DEST_REGION)
+
+  def get_source_snapshots(self):
+    filters = [{'Name':'tag-key', 'Values': [self.REPLICATE_TAG]}]
+    snapshots = self.ec2_source.snapshots.filter(Filters=filters)
+    return snapshots
+
+  def get_dest_snapshots(self,snapid,backupname):
+    filters = [{'Name':'description', 'Values': [self.AWS_SOURCE_REGION+'_'+snapid+'_'+backupname]}]
+    snapshots = self.ec2_dest.snapshots.filter(Filters=filters)
+    return snapshots
+
+  # Takes an snapshot or volume, returns the backup source
+  def get_backup_source(self, resource):
+    tags = filter(lambda x: x['Key'] == self.BACKUP_TAG, resource.tags)
+
+    if len(tags) < 1:
+      return None
+
+    return tags[0]['Value']
+
+  def copy_snapshot(self,snapshot):
+    sourcesnapid=snapshot.snapshot_id
+    sourcebackupname=self.get_backup_source(snapshot)
+    self.logger.info("Looking for existing replicas of snapshot {0}".format(sourcesnapid))
+    dest_snapshots=self.get_dest_snapshots(sourcesnapid,sourcebackupname)
+    dest_snapshot_count = len(list(dest_snapshots))
+    if dest_snapshot_count != 0:
+      self.logger.info("Replica found, no need to copy snapshot")
+    else:
+      self.logger.info("No replica found, copying snapshot {0}".format(sourcesnapid))
+      sourcesnap = self.ec2_dest.Snapshot(sourcesnapid)
+      dest_snap_description=self.AWS_SOURCE_REGION+'_'+sourcesnapid+'_'+sourcebackupname
+      copy_output=sourcesnap.copy(DryRun=False,SourceRegion=self.AWS_SOURCE_REGION,SourceSnapshotId=sourcesnapid,Description=dest_snap_description)
+      destsnapid=copy_output['SnapshotId']
+      destsnap = self.ec2_dest.Snapshot(destsnapid)
+      destsnap.create_tags(Tags=[
+        {'Key': self.REPLICATE_TAG, 'Value': dest_snap_description},
+        {'Key': self.BACKUP_TAG, 'Value': sourcebackupname}])
+
+  def copy_snapshots(self,snapshots):
+    for snapshot in snapshots:
+      self.copy_snapshot(snapshot)
+
+  def run(self):
+
+    # replicate any snapshots that need to be replicated
+    source_snapshots = self.get_source_snapshots()
+    source_snapshot_count = len(list(source_snapshots))
+
+    self.logger.info("Found {0} source snapshots".format(source_snapshot_count))
+
+    self.copy_snapshots(source_snapshots)