From 3a2ae8fe327ec0276adc48768c0eac4ac96ec966 Mon Sep 17 00:00:00 2001 From: Patrick Stadler Date: Sun, 16 Feb 2014 15:22:35 +0100 Subject: [PATCH] implement transfer() for copying files to remote hosts. several minor improvements --- README.md | 54 +++++++++++++++++++++++++++----- bin/fly.js | 2 +- lib/briefing.js | 24 ++++++--------- lib/flight.js | 42 +++++++++++++------------ lib/flightplan.js | 6 ++-- lib/logger.js | 14 ++++++--- lib/remote.js | 10 +++--- lib/transport/shell.js | 51 ++++++++++++++++++++++++++++-- lib/transport/ssh.js | 3 +- lib/transport/transport.js | 63 +++++++++++++++++++++++++++++++++----- package.json | 2 +- 11 files changed, 207 insertions(+), 64 deletions(-) diff --git a/README.md b/README.md index 9dab6f8..1290122 100644 --- a/README.md +++ b/README.md @@ -56,10 +56,10 @@ plan.local(function(local) { local.log('Run build'); local.exec('gulp build'); - local.log('Copy files to remote host'); - var filesToCopy = '(git ls-files -z;find assets/public -type f -print0)'; - local.exec(filesToCopy + '|rsync --files-from - -avz0 --rsh="ssh"' - + ' ./ pstadler@pstadler.sh:/tmp/' + tmpDir); + local.log('Copy files to remote hosts'); + var filesToCopy = local.exec('git ls-files', {silent: true}); + // rsync `filesToCopy` to all the destination's hosts + local.transfer(filesToCopy, '/tmp/' + tmpDir); }); // run commands on remote hosts (destinations) @@ -328,13 +328,54 @@ transport.sudo('echo Hello world', {user: 'www'}); transport.sudo('echo Hello world', {user: 'www', silent: true, failsafe: true}); ``` +### transport.transfer(files, remoteDir[, options]) → [results] + +Copy a list of files to the current destination's remote host(s) using +`rsync` with the SSH protocol. File transfers are executed in parallel. + After finishing all transfers, an array containing results from +`transport.exec()` is returned. This method is only available on local +flights. + +```javascript +var files = ['path/to/file1', 'path/to/file2']; +local.transfer(files, '/tmp/foo'); +``` + +#### Files argument +To make things more comfortable, the `files` argument doesn't have to be +passed as an array. Results from previous commands and zero-terminated +strings are handled as well: + +```javascript +// use result from a previous command +var files = local.git('ls-files', {silent: true}); // get list of files under version control +local.transfer(files, '/tmp/foo'); + +// use zero-terminated result from a previous command +var files = local.exec('(git ls-files -z;find node_modules -type f -print0)', {silent: true}); +local.transfer(files, '/tmp/foo'); + +// use results from multiple commands +var result1 = local.git('ls-files', {silent: true}).stdout.split('\n'); +var result2 = local.find('node_modules -type f', {silent: true}).stdout.split('\n'); +var files = result1.concat(result2); +files.push('path/to/another/file'); +local.transfer(files, '/tmp/foo'); +``` + +`transfer()` will use the current host's username defined with +`briefing()` unless `fly` is called with the `-u|--username` option. +In this case the latter will be used. If debugging is enabled +(either with `briefing()` or with `fly --debug`), `rsync` is executed +in verbose mode (`-v`). + ### transport.log(message) Print a message to stdout. Flightplan takes care that the message is formatted correctly within the current context. ```javascript -transport.log('Copying files to remote host'); +transport.log('Copying files to remote hosts'); ``` ### transport.silent() @@ -395,7 +436,7 @@ Print a debug message to stdout. Flightplan takes care that the message is formatted correctly within the current context. ```javascript -remote.debug('Copying files to remote host'); +remote.debug('Copying files to remote hosts'); ``` ### transport.abort([message]) @@ -415,7 +456,6 @@ remote.abort('Severe turbulences over the atlantic ocean!'); ## What's planned? - Add possibility to define a `sudoUser` per host with `briefing()`. -- Add a simple interface for file transport to remote hosts (e.g. `rsync`). - Tests will be implemented with upcoming releases. A part of this will be driven by bug reports. [npm-url]: https://npmjs.org/package/flightplan diff --git a/bin/fly.js b/bin/fly.js index 5845a95..5053473 100755 --- a/bin/fly.js +++ b/bin/fly.js @@ -23,7 +23,7 @@ if(!fs.existsSync(flightFile)) { var flightplan = require(flightFile) , options = { username: program.username || null, - debug: program.debug || false + debug: program.debug || null }; var destination = program.args[0]; diff --git a/lib/briefing.js b/lib/briefing.js index fec63df..e42836b 100644 --- a/lib/briefing.js +++ b/lib/briefing.js @@ -2,14 +2,10 @@ var util = require('util'); function Briefing(flightplan, config) { this.flightplan = flightplan; - this.logger = flightplan.logger; - - this.config = util._extend({ - debug: false, - destinations: {} - }, config); - - this.logger.enableDebug(!!this.config.debug); + config = config || {}; + this.debug = config.debug || false; + this.destinations = config.destinations || {}; + this.flightplan.logger.enableDebug(this.debug); } Briefing.prototype = { @@ -24,19 +20,19 @@ Briefing.prototype = { } }.bind(this)); } - if(options.debug) { - this.config.debug = options.debug; - this.flightplan.logger.enableDebug(this.config.debug); + if(options.debug !== undefined && options.debug !== null) { + this.debug = options.debug; + this.flightplan.logger.enableDebug(this.debug); } }, getDestinations: function() { - return Object.keys(this.config.destinations); + return Object.keys(this.destinations); }, getHostsForDestination: function(destination) { try { - var hosts = this.config.destinations[destination]; + var hosts = this.destinations[destination]; return (hosts instanceof Array) ? hosts : [hosts]; } catch(e) { return null; @@ -45,7 +41,7 @@ Briefing.prototype = { hasDestination: function(destination) { try { - return !!this.config.destinations[destination]; + return !!this.destinations[destination]; } catch(e) { return false; } diff --git a/lib/flight.js b/lib/flight.js index 0eb9cdf..21bbe1e 100644 --- a/lib/flight.js +++ b/lib/flight.js @@ -6,6 +6,7 @@ function Flight(flightplan, transportClass, fn) { this.fn = fn; this.transportClass = transportClass; this.logger = flightplan.logger; + this.hosts = null; this.status = { aborted: false, crashRecordings: null, @@ -15,14 +16,34 @@ function Flight(flightplan, transportClass, fn) { Flight.prototype = { - liftoff: function(config) { + start: function(destination) { + this.hosts = this.flightplan.briefing().getHostsForDestination(destination); + this.__start(); + return this.getStatus(); + }, + + abort: function(msg) { + this.flightplan.abort(); + this.status.aborted = true; + this.status.crashRecordings = msg || null; + }, + + isAborted: function() { + return this.status.aborted; + }, + + getStatus: function() { + return this.status; + }, + + __start: function() { var future = new Future(); var task = function() { Fiber(function() { var t = process.hrtime(); - var transport = new this.transportClass(this, config); + var transport = new this.transportClass(this); this.fn(transport); transport.close(); @@ -40,24 +61,7 @@ Flight.prototype = { }.bind(this); Future.wait(task()); - - return this.status; - }, - - abort: function(msg) { - this.flightplan.abort(); - this.status.aborted = true; - this.status.crashRecordings = msg || null; - }, - - isAborted: function() { - return this.status.aborted; - }, - - getStatus: function() { - return this.status; } - }; module.exports = Flight; \ No newline at end of file diff --git a/lib/flightplan.js b/lib/flightplan.js index 3ad1f71..edf084d 100644 --- a/lib/flightplan.js +++ b/lib/flightplan.js @@ -80,10 +80,10 @@ function Flightplan() { }.bind(this)); process.on('uncaughtException', function(err) { - this.logger.error(err); - this.logger.error('Flightplan aborted'.error); + this.logger.error(err.stack); this.disasterCallback(); this.debriefingCallback(); + this.logger.error('Flightplan aborted'.error); process.exit(1); }.bind(this)); @@ -279,7 +279,7 @@ Flightplan.prototype = { this.logger.info('Flight'.info, this.logger.format('%s/%s', i+1, len).magenta, 'launched...'.info); this.logger.space(); - flight.liftoff(this.briefing().getHostsForDestination(destination)); + flight.start(destination); var status = flight.getStatus(); if(flight.isAborted()) { diff --git a/lib/logger.js b/lib/logger.js index 51e12ec..4ef175c 100644 --- a/lib/logger.js +++ b/lib/logger.js @@ -13,8 +13,10 @@ var messageTypes = { colors.setTheme(messageTypes); +var __debug = false; // persistent + function Logger(debug) { - this._debug = debug || false; + __debug = (debug !== null && debug !== undefined) ? debug : __debug; this.symbol = '✈'; this.prefix = ''; @@ -75,11 +77,15 @@ Object.keys(messageTypes).forEach(function(type) { Logger.prototype = util._extend(Logger.prototype, { enableDebug: function(flag) { - this._debug = !!flag; + __debug = !!flag; + }, + + debugEnabled: function() { + return __debug; }, clone: function() { - return new Logger(this._debug); + return new Logger(); }, cloneWithPrefix: function(prefix) { @@ -98,7 +104,7 @@ Logger.prototype = util._extend(Logger.prototype, { }, debug: function() { - if(this._debug) { + if(__debug) { var msg = this._parseArgs(arguments); this._log(this.symbol.debug, this.prefix, msg); } diff --git a/lib/remote.js b/lib/remote.js index 5055330..d1dcc9a 100644 --- a/lib/remote.js +++ b/lib/remote.js @@ -15,14 +15,14 @@ function RemoteFlight(flightplan, fn) { util.inherits(RemoteFlight, Flight); -RemoteFlight.prototype.liftoff = function(hosts) { - var task = function(config) { +RemoteFlight.prototype.__start = function() { + var task = function(host) { var future = new Future(); var flight = new RemoteFlight(this.flightplan, this.fn); Fiber(function() { var t = process.hrtime(); - var transport = new flight.transportClass(flight, config); + var transport = new flight.transportClass(flight, host); flight.fn(transport); transport.close(); @@ -42,8 +42,8 @@ RemoteFlight.prototype.liftoff = function(hosts) { }.bind(this); var tasks = []; - for(var i=0, len=hosts.length; i < len; i++) { - tasks.push(task(hosts[i])); + for(var i=0, len=this.hosts.length; i < len; i++) { + tasks.push(task(this.hosts[i])); } Future.wait(tasks); diff --git a/lib/transport/shell.js b/lib/transport/shell.js index 87e5760..3cbc46c 100644 --- a/lib/transport/shell.js +++ b/lib/transport/shell.js @@ -1,11 +1,13 @@ var util = require('util') , exec = require("child_process").exec , Fiber = require('fibers') + , Future = require('fibers/future') , Transport = require('./transport'); function ShellTransport(flight) { ShellTransport.super_.call(this, flight); - this.logger = this.logger.cloneWithPrefix('local'); + this.host = 'local'; + this.logger = this.logger.cloneWithPrefix(this.host); } util.inherits(ShellTransport, Transport); @@ -45,7 +47,7 @@ ShellTransport.prototype.__exec = function(cmd, args, options) { this.logger.warn(this.logger.format('failed safely').warn, 'with exit code:', ret.code); } else { this.logger.error(this.logger.format('failed').error, 'with exit code:', ret.code); - this.flight.abort(this.logger.format('`%s` failed on %s', cmd.white, 'local')); + this.flight.abort(this.logger.format('`%s` failed on localhost', cmd.white)); } fiber.run(ret); }.bind(this)); @@ -56,4 +58,49 @@ ShellTransport.prototype.__exec = function(cmd, args, options) { return Fiber.yield(); }; +ShellTransport.prototype.__transfer = function(files, remoteDir, options) { + if(!remoteDir) { + throw new Error('transfer: missing remote path'); + } + + if(files instanceof Array) { + files = files.join('\n'); + } else if(files instanceof Object) { + if(!files.hasOwnProperty('stdout')) { + throw new Error('transfer: invalid object passed'); + } + files = files.stdout; + } + + files = (files || '').trim().replace(/[\r|\n|\0]/mg, '\\n'); + if(!files) { + throw new Error('transfer: empty file list passed'); + } + + var rsyncFlags = 'az'; + if(this.logger.debugEnabled()) { + rsyncFlags += 'v'; + } + var _results = []; + var task = function(config) { + var future = new Future(); + + Fiber(function() { + var cmd = util.format('(echo "%s") | rsync --files-from - -%s --rsh="ssh" ./ %s@%s:%s' + , files, rsyncFlags, config.username, config.host, remoteDir); + _results.push(this.exec(cmd, options)); + return future.return(); + }.bind(this)).run(); + + return future; + }.bind(this); + + var tasks = []; + for(var i=0, len=this.flight.hosts.length; i < len; i++) { + tasks.push(task(this.flight.hosts[i])); + } + Future.wait(tasks); + return _results; +}; + module.exports = ShellTransport; \ No newline at end of file diff --git a/lib/transport/ssh.js b/lib/transport/ssh.js index 0b41bdd..8f78a37 100644 --- a/lib/transport/ssh.js +++ b/lib/transport/ssh.js @@ -6,8 +6,9 @@ var util = require('util') function SSHTransport(flight, config) { SSHTransport.super_.call(this, flight); this.config = config; + this.host = this.config.host; + this.logger = this.logger.cloneWithPrefix(this.host); this.connection = new Connection(); - this.logger = this.logger.cloneWithPrefix(this.config.host); var _fiber = Fiber.current; diff --git a/lib/transport/transport.js b/lib/transport/transport.js index c93797d..a935051 100644 --- a/lib/transport/transport.js +++ b/lib/transport/transport.js @@ -115,12 +115,59 @@ Transport.prototype = { return this._execOrSkip('sudo', args, opts); }, + /** + * Copy a list of files to the current destination's remote host(s) using + * `rsync` with the SSH protocol. File transfers are executed in parallel. + * After finishing all transfers, an array containing results from + * `transport.exec()` is returned. This method is only available on local + * flights. + * + * ```javascript + * var files = ['path/to/file1', 'path/to/file2']; + * local.transfer(files, '/tmp/foo'); + * ``` + * + * #### Files argument + * To make things more comfortable, the `files` argument doesn't have to be + * passed as an array. Results from previous commands and zero-terminated + * strings are handled as well: + * + * ```javascript + * // use result from a previous command + * var files = local.git('ls-files', {silent: true}); // get list of files under version control + * local.transfer(files, '/tmp/foo'); + * + * // use zero-terminated result from a previous command + * var files = local.exec('(git ls-files -z;find node_modules -type f -print0)', {silent: true}); + * local.transfer(files, '/tmp/foo'); + * + * // use results from multiple commands + * var result1 = local.git('ls-files', {silent: true}).stdout.split('\n'); + * var result2 = local.find('node_modules -type f', {silent: true}).stdout.split('\n'); + * var files = result1.concat(result2); + * files.push('path/to/another/file'); + * local.transfer(files, '/tmp/foo'); + * ``` + * + * `transfer()` will use the current host's username defined with + * `briefing()` unless `fly` is called with the `-u|--username` option. + * In this case the latter will be used. If debugging is enabled + * (either with `briefing()` or with `fly --debug`), `rsync` is executed + * in verbose mode (`-v`). + * + * @method transfer(files, remoteDir[, options]) + * @return [results] + */ + transfer: function(files, remoteDir, options) { + this.__transfer(files, remoteDir, options); + }, + /** * Print a message to stdout. Flightplan takes care that the message * is formatted correctly within the current context. * * ```javascript - * transport.log('Copying files to remote host'); + * transport.log('Copying files to remote hosts'); * ``` * * @method log(message) @@ -206,7 +253,7 @@ Transport.prototype = { * is formatted correctly within the current context. * * ```javascript - * remote.debug('Copying files to remote host'); + * remote.debug('Copying files to remote hosts'); * ``` * * @method debug(message) @@ -230,21 +277,23 @@ Transport.prototype = { this.flight.abort.apply(this.flight, arguments); }, + close: function() { + this.__close(); + }, + __close: function() { }, __exec: function() { }, - close: function() { - if(this.__close) { - this.__close(); - } + __transfer: function() { + throw new Error('transfer: transport does not support this method'); }, _execOrSkip: function(cmd, args, opts) { if(this.flight.isAborted()) { - return { code: 0, stdout: null, stderr: null }; + return { code: null, stdout: null, stderr: null }; } var result = this.__exec(cmd, args, opts); this.logger.space(); diff --git a/package.json b/package.json index 91824b0..88bc5c3 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "flightplan", "description": "Run a sequence of commands against local and remote hosts", - "version": "0.1.2", + "version": "0.1.3", "author": "Patrick Stadler ", "keywords": [ "deploy",