snapshotManager.js

/* jshint node: true */
/* jshint -W097 */
/*
 * Copyright 2014, Amazon.com, Inc. or its affiliates. All Rights Reserved.
 *
 * Licensed under the Amazon Software License (the "License").
 * You may not use this file except in compliance with the License.
 * A copy of the License is located at
 *
 * http://aws.amazon.com/asl/
 *
 * or in the "license" file accompanying this file. This file is distributed
 * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
 * express or implied. See the License for the specific language governing
 * permissions and limitations under the License.
 */
'use strict';

var constants = require("./constants");

var setRegion = process.env['AWS_REGION'];
if (!setRegion) {
	console.log("Setting default region us-east-1");
	setRegion = "us-east-1";
}
var AWS = require("aws-sdk");
AWS.config.apiVersions = {
	redshift : '2012-12-01'
};
AWS.config.update({
	region : setRegion
});

var redshift = new AWS.Redshift();
var async = require('async');
var moment = require('moment');

function getFriendlyDate(t) {
	return t.format(constants.dateFormat);
}

function getSnapshotId(config) {
	return config.namespace + "-" + config.targetResource + "-" + getFriendlyDate(moment());
}

function getTags(config, forDate) {
	var tags = [ {
		Key : constants.createdByName,
		Value : constants.createdByValue
	}, {
		Key : constants.namespaceTagName,
		Value : config.namespace
	} ];

	if (forDate) {
		tags.push({
			Key : constants.createdAtName,
			Value : getFriendlyDate(forDate)
		});
	}
	return tags;
}

// function to query for snapshots created since the configuration start time
function getSnapshots(config, callback) {

	var snapStartTime;
	var intervalExpression;

	if (config.snapshotInterval) {
		snapStartTime = moment().subtract(config.snapshotInterval.duration, config.snapshotInterval.units);
		intervalExpression = config.snapshotInterval.duration + " " + config.snapshotInterval.units;
	} else {
		snapStartTime = moment().subtract(config.snapshotIntervalHours, 'hours');
		intervalExpression = config.snapshotIntervalHours + " hours";
	}

	exports.getSnapshotsInInterval(config.targetResource, config.namespace, snapStartTime, undefined, undefined,
			function(err, snapshots) {
				if (err) {
					callback(err);
				} else {
					console.log("Resolved " + snapshots.length + " Snapshots in the last " + intervalExpression);

					callback(null, config, snapshots);
				}
			});
}
exports.getSnapshots = getSnapshots;

function getSnapshotsInInterval(targetResource, namespace, startTime, endTime, extraSearchCriteria, callback) {
	var startSearchLabel = startTime ? startTime.format() : "any";
	var endSearchLabel = endTime ? endTime.format() : "any";

	console.log("Requesting '" + namespace + "' Snapshots for " + targetResource + " in interval (" + startSearchLabel
			+ " - " + endSearchLabel + ")");

	var describeParams = {
		ClusterIdentifier : targetResource,
		// only search for snapshots in the current namespace
		TagKeys : [ constants.namespaceTagName ],
		TagValues : [ namespace ]
	};

	// add time search criteria
	if (startTime) {
		describeParams.StartTime = startTime.toDate();
	}

	if (endTime) {
		describeParams.EndTime = endTime.toDate();
	}

	// add any provided extra search items
	if (extraSearchCriteria) {
		Object.keys(extraSearchCriteria).map(function(item) {
			describeParams[item] = extraSearchCriteria[item];
		});
	}

	// console.log(JSON.stringify(describeParams));

	redshift.describeClusterSnapshots(describeParams, function(err, data) {
		// got a set of snapshots, or not, so call the callback with the snap list
		if (err) {
			callback(err);
		} else {
			callback(null, data.Snapshots);
		}
	});
}
exports.getSnapshotsInInterval = getSnapshotsInInterval;

// function to create a manual snapshot
function createSnapshot(config, callback) {
	var now = moment();

	var newSnapshotId = getSnapshotId(config);

	console.log("Creating new Snapshot " + newSnapshotId + " for " + config.targetResource);

	var params = {
		ClusterIdentifier : config.targetResource,
		SnapshotIdentifier : newSnapshotId,
		Tags : getTags(config, now)
	};

	redshift.createClusterSnapshot(params, function(err, data) {
		if (err) {
			callback(err);
		} else {
			console.log("Created new Manual Snapshot " + data.Snapshot.SnapshotIdentifier);
			callback(null, config);
		}
	});
}
exports.createSnapshot = createSnapshot;

// function to convert an automatic snapshot into a manual snapshot
function convertAutosnapToManual(config, snapshot, callback) {
	console.log("Converting Automatic Snapshot " + snapshot.SnapshotIdentifier + " to manual for long term retention");

	redshift.copyClusterSnapshot({
		SourceSnapshotIdentifier : snapshot.SnapshotIdentifier,
		TargetSnapshotIdentifier : getSnapshotId(config),
	}, function(err, data) {
		if (err) {
			callback(err);
		} else {
			// tag the converted snapshot
			var params = {
				ResourceName : data.SnapshotIdentifier,
				Tags : getTags(config)
			};
			redshift.createTags(params, function(err, data) {
				if (err) {
					callback(err);
				} else {
					console.log("Converted latest Snapshot " + snapshot.SnapshotIdentifier + " to new Manual Snapshot "
							+ data.SnapshotIdentifier);
					callback(null, config);
				}
			});
		}
	});
}
exports.convertAutosnapToManual = convertAutosnapToManual;

// function to cleanup snapshots older than the specified retention period
function cleanupSnapshots(config, callback) {
	if (!config.snapshotRetentionDays && !config.snapshotRetention) {
		// customer may not have configured a snapshots retention - ok
		console.log("No Snapshot retention limits configured - all Snapshots will be retained");
		callback(null);
	} else {

		var snapEndTime;
		var intervalExpression;

		if (config.snapshotRetention) {
			snapEndTime = moment().subtract(config.snapshotRetention.duration, config.snapshotRetention.units);
			intervalExpression = config.snapshotRetention.duration + " " + config.snapshotRetention.units + " ("
					+ snapEndTime.format() + ")";
		} else {
			snapEndTime = moment().subtract(config.snapshotRetentionDays, 'days');
			intervalExpression = config.snapshotRetentionDays + " days (" + snapEndTime.format() + ")";
		}

		console.log("Cleaning up '" + config.namespace + "' Snapshots older than ");

		exports.getSnapshotsInInterval(config.targetResource, config.namespace, undefined, snapEndTime, {
			SnapshotType : "manual"
		}, function(err, snapshots) {
			if (err) {
				callback(err);
			} else {
				if (snapshots && snapshots.length > 0) {
					console.log("Cleaning up " + snapshots.length + " previous Snapshots");

					async.map(snapshots, function(item, mapCallback) {
						console.log("Deleting Snapshot " + item.SnapshotIdentifier);

						// delete this manual snapshot
						redshift.deleteClusterSnapshot({
							SnapshotIdentifier : item.SnapshotIdentifier,
							SnapshotClusterIdentifier : config.targetResource
						}, function(err, data) {
							if (err) {
								mapCallback(err);
							} else {
								callback(null);
							}
						});
					}, function(err, results) {
						if (err) {
							callback(err);
						} else {
							callback(null);
						}
					});
				} else {
					// there are no snapshots older than the retention period - so all ok
					console.log("No old snapshots found to clear");
					callback(null);
				}
			}
		});
	}
}
exports.cleanupSnapshots = cleanupSnapshots;

/*
 * function which determines if we need to create or convert snapshots based on
 * the snapshots that currently exist
 */
function createOrConvertSnapshots(config, snapshotList, callback) {
	if (snapshotList.length === 0) {
		// create a new snapshot
		createSnapshot(config, callback);
	} else {
		// convert the latest auto snapshot created within the specified
		// period to a manual snapshot
		var manualExists = false;
		snapshotList.map(function(item) {
			if (item.SnapshotType === "manual") {
				manualExists = true;
			}
		});
		// only do an auto conversion if there are no manual snapshots in
		// the period
		if (!manualExists) {
			convertAutosnapToManual(config, snapshotList[snapshotList.length - 1], callback);
		} else {
			callback(null, config);
		}
	}
}
exports.createOrConvertSnapshots = createOrConvertSnapshots;

function validateConfig(config, callback) {
	var error;

	if (!config.namespace || config.namespace === "") {
		error = "You must provide a configuration namespace";
	}

	if (config.snapshotInterval && constants.validTimeUnits.indexOf(config.snapshotInterval.units) == -1) {
		error = "Snapshot Interval Units must be one of: " + JSON.stringify(constants.validTimeUnits);
	}

	if (config.snapshotRetention && constants.validTimeUnits.indexOf(config.snapshotRetention.units) == -1) {
		error = "Snapshot Retention Units must be one of: " + JSON.stringify(constants.validTimeUnits);
	}

	callback(error);
}
exports.validateConfig = validateConfig;

// processor entry point
function run(config, callback) {
	var checksPassed = true;
	var error;
	if (!config.targetResource) {
		checksPassed = false;
		error = "Unable to resolve target resource from provided config.targetResource";
	}

	if (!config.namespace) {
		checksPassed = false;
		error = "Configuration namespace must be provided";
	}

	console.log("Configuration Provided: " + JSON.stringify(config));

	if (!checksPassed) {
		console.log(JSON.stringify(config));
		callback(error);
	} else {
		validateConfig(config, function(err) {
			if (err) {
				callback(err);
			} else {
				async.waterfall([
				// check for whether we have a snapshot taken within the required period
				getSnapshots.bind(undefined, config),
				// process the list of automatic snapshots, and determine if we need to
				// create additional snaps
				createOrConvertSnapshots.bind(undefined),
				// now cleanup the existing snapshots
				cleanupSnapshots.bind(undefined) ], function(err) {
					callback(err);
				});
			}
		});
	}
}
exports.run = run;