Skip to content

Commit

Permalink
Merge pull request #1 from mediamonks/feature/clean-up
Browse files Browse the repository at this point in the history 
Feature/clean up
  • Loading branch information
@mientjan
mientjan authored Dec 4, 2018
2 parents 78c78f0 + 8c1cae3 commit 23ac88a
Show file tree
Hide file tree
Showing 3 changed files with 58 additions and 51 deletions.
65 changes: 28 additions & 37 deletions src/lib/Uploader.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ const chalk = require('chalk');
const fs = require('fs');
const mime = require('mime');


export type Options = {
bucket: string;
localPath: string;
Expand All @@ -20,7 +19,7 @@ export type Options = {
glob?: string;
concurrency?: number;
dryRun?: boolean;
cacheControl?: string | { [key: string]: string; };
cacheControl?: string | { [key: string]: string };
s3Client?: S3;
};

Expand All @@ -35,21 +34,18 @@ export default class Uploader {
private options: Options;
private bar: any;

constructor(options: Options)
{
constructor(options: Options) {
this.options = {
...defaultOptions,
...options,
};

if (this.options.config)
{
if (this.options.config) {
AWS.config.loadFromPath(this.options.config);
}

// TODO: more checks on other options?
if (!this.options.bucket)
{
if (!this.options.bucket) {
throw new Error('No bucket defined!');
}

Expand All @@ -60,8 +56,9 @@ export default class Uploader {
return this.run();
}

private async run():Promise<void> {
private async run(): Promise<void> {
const files = await this.getFiles();
const { concurrency, localPath, remotePath } = this.options;

// a nice progress bar to show during upload
this.bar = new ProgressBar('[:bar] :percent | :etas | :current / :total | :rate/fps ', {
Expand All @@ -74,13 +71,10 @@ export default class Uploader {
// do the work!
await streamBatch({
files,
concurrency: this.options.concurrency,
processItem: (file:string):Promise<void> => {
const key = path.join(this.options.remotePath, file);
return this.uploadFile(
path.resolve(this.options.localPath, file),
key,
);
concurrency,
processItem: (file: string): Promise<void> => {
const key = path.join(remotePath, file);
return this.uploadFile(path.resolve(localPath, file), key);
},
onProgress: () => this.bar.tick(),
});
Expand All @@ -90,42 +84,41 @@ export default class Uploader {
}

private getFiles(): Promise<Array<string>> {
const gatheringSpinner = ora(
`Gathering files from ${chalk.blue(this.options.localPath)} (please wait) ...`,
);
const { localPath, glob: globPath } = this.options;
const gatheringSpinner = ora(`Gathering files from ${chalk.blue(localPath)} (please wait) ...`);

gatheringSpinner.start();

return new Promise((resolve, reject) => {
glob(`**/${this.options.glob}`, { cwd: path.resolve(this.options.localPath) }, (err, files) => {
glob(`**/${globPath}`, { cwd: path.resolve(localPath) }, (err, files) => {
if (err) {
gatheringSpinner.fail(err);
reject(err);
}

gatheringSpinner.succeed(
`Found ${chalk.green(files.length)} files at ${chalk.blue(
this.options.localPath,
)}, starting upload:`,
`Found ${chalk.green(files.length)} files at ${chalk.blue(localPath)}, starting upload:`,
);

resolve(files);
});
});
}

uploadFile(localFilePath:string, remotePath:string): Promise<void> {
public uploadFile(localFilePath: string, remotePath: string): Promise<void> {
const body = fs.createReadStream(localFilePath);
const { dryRun, bucket: Bucket } = this.options;

const params = {
Bucket: this.options.bucket,
Bucket,
Key: remotePath.replace(/\\/, '/'),
Body: body,
ContentType: mime.getType(localFilePath),
CacheControl: this.getCacheControlValue(localFilePath),
};

return new Promise((resolve) => {
if (!this.options.dryRun) {
return new Promise(resolve => {
if (!dryRun) {
this.s3.upload(params, err => {
// tslint:disable-next-line no-console
if (err) console.error('err:', err);
Expand All @@ -134,25 +127,23 @@ export default class Uploader {
} else {
resolve();
}
})
});
}

getCacheControlValue(file:string ): string {
if (this.options.cacheControl) {
public getCacheControlValue(file: string): string {
const { cacheControl } = this.options;
if (cacheControl) {
// return single option for all files
if (typeof this.options.cacheControl === 'string') {
return this.options.cacheControl;
if (typeof cacheControl === 'string') {
return cacheControl;
}

// find match in glob patterns
const match = Object.keys(this.options.cacheControl).find(key => minimatch(file, key));
return match && this.options.cacheControl[match] || '';
const match = Object.keys(cacheControl).find(key => minimatch(file, key));
return (match && cacheControl[match]) || '';
}

// return default value
return '';
}
}



4 changes: 2 additions & 2 deletions src/lib/batch.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ export default function streamBatch({
files,
processItem,
onProgress,
}: Options):Promise<void> {
}: Options): Promise<void> {
return new Promise(resolve => {
let count = 0;
const total = files.length;
Expand All @@ -25,7 +25,7 @@ export default function streamBatch({
setTimeout(() => {
onProgress();
resolve();
}, 50)
}, 50);
} else {
onProgress();

Expand Down
40 changes: 28 additions & 12 deletions src/lib/cli.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,18 +5,29 @@ import Uploader from './Uploader';

yargs
.usage('Usage: $0 <command> [options]')
.command(['$0', 'upload'], 'Upload files to s3', () => {}, (argv) => {
new Uploader(argv).upload();
})
.example('$0 -b bucket-name -p ./files -r /data', 'Upload files from a local folder to a s3 bucket path')
.command(
['$0', 'upload'],
'Upload files to s3',
() => {},
argv => {
new Uploader(argv).upload();
},
)
.example(
'$0 -b bucket-name -p ./files -r /data',
'Upload files from a local folder to a s3 bucket path',
)
.example('$0 ... -a "max-age: 300"', 'Set cache-control for all files')
.example('$0 ... -a \'{ "**/*.json": "max-age: 300", "**/*.*": "3600" }\'', 'Upload files from a local folder to a s3 bucket path')
.example(
'$0 ... -a \'{ "**/*.json": "max-age: 300", "**/*.*": "3600" }\'',
'Upload files from a local folder to a s3 bucket path',
)
.example('$0 -d ...', 'Dry run upload')
.option('d', {
alias: 'dry-run',
default: false,
describe: 'Do a dry run, don\'t do any upload.',
type: 'boolean'
describe: "Do a dry run, don't do any upload.",
type: 'boolean',
})
.option('b', {
alias: 'bucket',
Expand Down Expand Up @@ -56,7 +67,8 @@ yargs
.option('a', {
alias: 'cache-control',
default: '',
describe: 'Cache control for uploaded files, can be string for single value or list of glob settings',
describe:
'Cache control for uploaded files, can be string for single value or list of glob settings',
type: 'string',
nargs: 1,
coerce: (value) => {
Expand All @@ -78,11 +90,15 @@ yargs
type: 'string',
nargs: 1,
})
.demandOption(['bucket', 'local-path', 'remote-path'], 'Please provide at least the required arguments to upload.')
.demandOption(
['bucket', 'local-path', 'remote-path'],
'Please provide at least the required arguments to upload.',
)
.group(['bucket', 'local-path', 'remote-path'], 'Required:')
.help('h')
.alias('h', 'help')
.epilogue('for more information about AWS authentication, please visit https://docs.aws.amazon.com/sdk-for-javascript/v2/developer-guide/setting-credentials-node.html')
.epilogue(
'for more information about AWS authentication, please visit https://docs.aws.amazon.com/sdk-for-javascript/v2/developer-guide/setting-credentials-node.html',
)
.version(false)
.wrap(Math.min(120, yargs.terminalWidth()))
.argv;
.wrap(Math.min(120, yargs.terminalWidth())).argv;

0 comments on commit 23ac88a

Please sign in to comment.