diff --git a/.gitignore b/.gitignore index 8765e0b..030065e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,12 +1,19 @@ -node_modules - # Logs logs *.log npm-debug.log* +# Dependency directory +node_modules + +# Optional npm cache directory +.npm + +# Optional REPL history +.node_repl_history + # Typescript stuff -typings build -docs coverage + +package-lock.json diff --git a/.npmignore b/.npmignore index ec45734..312497f 100644 --- a/.npmignore +++ b/.npmignore @@ -1,22 +1,8 @@ -build -typings - # Logs logs *.log npm-debug.log* -# Runtime data -pids -*.pid -*.seed - -# Directory for instrumented libs generated by jscoverage/JSCover -lib-cov - -# Coverage directory used by tools like istanbul -coverage - # Dependency directory node_modules @@ -26,20 +12,18 @@ node_modules # Optional REPL history .node_repl_history - # Typescript stuff -typings -build -docs -coverage +build/ +coverage/ +config/ +tsconfig.json +tslint.json -# Files +# Typescript files (but not the definitions) *.ts !*.d.ts *.js.map -*.spec.js -*.spec.ts -*.spec.d.ts -*.spec.js.map -.travis.yml +# Testfiles +jest.json +test/ diff --git a/.npmrc b/.npmrc new file mode 100644 index 0000000..9cf9495 --- /dev/null +++ b/.npmrc @@ -0,0 +1 @@ +package-lock=false \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index 47aad24..7bd9f66 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,21 +1,25 @@ language: node_js + +cache: + directories: + - node_modules + +notifications: + email: false + node_js: -- '6' -- '5' -- '4.2' + - '8' + - '6' -script: npm run citest -after_script: npm install coveralls@^2.11.9 && cat ./coverage/lcov.info | coveralls +before_script: + - npm prune -before_deploy: -- npm run bootstrap -- tsc --outDir . +after_success: + - npm install coveralls@^2.11.9 && cat ./coverage/lcov.info | coveralls + - npm i + - npm run build + - npm run semantic-release -deploy: - provider: npm - email: christoph.buehler@bluewin.ch - api_key: - secure: NoAP3zXpaw7efTltwCT5v//pWSTsCk1pvMVR02gFimdOI4kc9yr45GUAemmMBKDMPyxqwS2rJHl1A6Wps+P6gz2oMAFyioiK5TZI0qQNTVu9ggD7d1hlCr5Is1WZckblU863YckES6u+HoO0eXFC3iGZnh+1BldbTPR1PA9LWFJPGsZzJsDQRpxOWY6TLKRPlE53QmdclcIc//o4vVCldrCAdKojGlOVKhc03IaWt4i+onCszP81rUYkTTPdE2ODwQZrlbn1D01ojPu8UvPM4REKB90P+C9J/9PL1LR95UjHNPr2+nsa5dmbI2IQhAEudvAKOyBJV27DfZ2AcNNQyXc1e3OnH7zhP3Y19rceTzk0DOXnXKrQbzZEZ9ICnPfxv5PaQ+vOFJB4XkyrrF7LBWqNMRiYywt+vCBRelU7RUepXtO5wcj4TJmXUyB2fESL4W90HtPh1AZNvRPL2dH2DZF3tE2XSnG0MSHbwcrO+9ZUFZ2xBxCNM4WW5VCzH5H9z+ieLYqQmDwjBCgQp52Nw5UnJSmHk72Tlbbzphg4e7e40xH00AIVfLQiBHzEMOyRfZbtvS7cI/1SSMpP/sU+9rFUl5GiavupPVbFnzDUNBltXOk6HSaFtlarafO2llUcAGgxRithYLpjGbD1/4ZU+5NyOR4AFBejj0oq1cDAn7Y= - on: - tags: true - repo: smartive/proc-that-elastic-loader +branches: + except: + - /^v\d+\.\d+\.\d+$/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 0db4d63..6d9ea93 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,8 @@ All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](http://semver.org/). ## [Unreleased] +### Changed +- Update node to v6/v8 and upgrade modules ## [v0.4.0] diff --git a/config/tsconfig.base.json b/config/tsconfig.base.json new file mode 100644 index 0000000..9b2c17d --- /dev/null +++ b/config/tsconfig.base.json @@ -0,0 +1,29 @@ +{ + "compilerOptions": { + "target": "es6", + "module": "commonjs", + "moduleResolution": "node", + "removeComments": false, + "outDir": "../build", + "rootDir": "../src", + "declaration": true, + "sourceMap": false, + "importHelpers": true, + "strictNullChecks": true, + "experimentalDecorators": true, + "emitDecoratorMetadata": true, + "noUnusedLocals": true, + "noUnusedParameters": true, + "lib": [ + "es2015", + "dom" + ] + }, + "include": [ + "../src/**/*" + ], + "exclude": [ + "../node_modules", + "../build" + ] +} diff --git a/config/tsconfig.build.json b/config/tsconfig.build.json new file mode 100644 index 0000000..4bd0edc --- /dev/null +++ b/config/tsconfig.build.json @@ -0,0 +1,6 @@ +{ + "extends": "./tsconfig.base.json", + "compilerOptions": { + "outDir": "../" + } +} diff --git a/index.ts b/index.ts deleted file mode 100644 index fb11a2c..0000000 --- a/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './ElasticLoader'; \ No newline at end of file diff --git a/jest.json b/jest.json new file mode 100644 index 0000000..8e751c3 --- /dev/null +++ b/jest.json @@ -0,0 +1,19 @@ +{ + "collectCoverage": true, + "mapCoverage": true, + "transform": { + "^.+\\.tsx?$": "/node_modules/ts-jest/preprocessor.js" + }, + "testMatch": [ + "**/test/**/*.spec.ts" + ], + "testPathIgnorePatterns": [ + "/node_modules/" + ], + "moduleFileExtensions": [ + "ts", + "tsx", + "js", + "json" + ] +} diff --git a/package.json b/package.json index 5f15ad9..43758b0 100644 --- a/package.json +++ b/package.json @@ -1,16 +1,16 @@ { "name": "proc-that-elastic-loader", - "version": "0.4.0", + "version": "0.0.0-development", "description": "Loader for proc-that. Loads processed items into an elasticsearch index.", "main": "index.js", "scripts": { - "clean": "rimraf build", - "bootstrap": "npm install && typings install", - "pretest": "npm run bootstrap && npm run clean && tsc", - "test": "mocha --ui bdd --recursive ./build", - "precitest": "npm run bootstrap && npm run clean && tsc", - "citest": "istanbul cover -x \"**/*.spec.*\" _mocha --report lcovonly -- --ui bdd --recursive ./build", - "develop": "npm run clean && tsc -w --sourceMap --pretty" + "clean": "del-cli ./build ./coverage", + "build": "npm run clean && tsc -p ./config/tsconfig.build.json", + "develop": "npm run clean && tsc -p .", + "lint": "tslint -c ./tslint.json -p ./config/tsconfig.build.json", + "test": "npm run lint && npm run clean && jest -c ./jest.json", + "test:watch": "npm run clean && jest -c ./jest.json --watch", + "semantic-release": "semantic-release pre && npm publish && semantic-release post" }, "keywords": [ "etl", @@ -21,31 +21,31 @@ "elasticsearch" ], "engines": { - "node": ">=4.2" + "node": ">=6" }, "repository": { "type": "git", - "url": "git@github.com:smartive/proc-that-elastic-loader.git" + "url": "https://github.com/smartive/proc-that-elastic-loader.git" }, "bugs": "https://github.com/smartive/proc-that-elastic-loader/issues", "author": "Christoph Bühler ", "license": "MIT", "devDependencies": { - "chai": "^3.5.0", - "chai-as-promised": "^5.3.0", - "del-cli": "^0.2.0", - "istanbul": "^0.4.4", - "mocha": "^2.5.3", - "mocha-lcov-reporter": "^1.2.0", - "rimraf": "^2.5.3", - "sinon": "^1.17.4", - "sinon-chai": "^2.8.0", - "typescript": "^1.8.10", - "typings": "^1.3.1" + "@types/jest": "^20.0.5", + "del-cli": "^1.1.0", + "jest": "^20.0.4", + "ts-jest": "^20.0.7", + "tslint": "^5.5.0", + "tslint-config-airbnb": "^5.2.1", + "tsutils": "^2.8.0", + "typescript": "^2.4.2", + "semantic-release": "^6.3.6" }, "dependencies": { - "elasticsearch": "^11.0.1", + "elasticsearch": "^13.2.0", "proc-that": "^0.4.0", - "rxjs": "^5.0.0-beta.10" + "@types/node": "^8.0.17", + "rxjs": "^5.4.2", + "tslib": "^1.7.1" } } diff --git a/ElasticLoader.ts b/src/ElasticLoader.ts similarity index 58% rename from ElasticLoader.ts rename to src/ElasticLoader.ts index 7ff8be7..d4e83d2 100644 --- a/ElasticLoader.ts +++ b/src/ElasticLoader.ts @@ -1,21 +1,28 @@ -import {Loader} from 'proc-that'; -import {Observable} from 'rxjs'; -import {Buffer} from './helpers/Buffer'; +import { Loader } from 'proc-that'; +import { Observable } from 'rxjs'; -let elasticsearch = require('elasticsearch'); +import { Buffer } from './helpers/Buffer'; + +const elasticsearch = require('elasticsearch'); class NoIdProvidedError extends Error { - constructor(private object:any) { + constructor(public object: any) { super('No id provided by object'); } } export class ElasticLoader implements Loader { - private esClient:any; - private buffer:Buffer = new Buffer(); - - constructor(config:any, private index:string, private type:string, private predicate:(obj:any) => boolean = o => true, private idSelector:(obj:any) => any = o => o.id) { - let esConfig = JSON.parse(JSON.stringify(config)); + private esClient: any; + private buffer: Buffer = new Buffer(); + + constructor( + config: any, + private index: string, + private type: string, + private predicate: (obj: any) => boolean = () => true, + private idSelector: (obj: any) => any = o => o.id, + ) { + const esConfig = JSON.parse(JSON.stringify(config)); if (!esConfig.requestTimeout) { // set requestTimeout to 5min. // reason: when you shoot many index requests to the esClient, elasticsearch buffers your requests. @@ -28,28 +35,28 @@ export class ElasticLoader implements Loader { } } - write(object: any): Observable { + public write(object: any): Observable { if (!this.predicate(object)) { return Observable.empty(); } - let id = this.idSelector(object); + const id = this.idSelector(object); if (id === null || id === undefined) { return Observable.throw(new NoIdProvidedError(object)); } - let promise = this.buffer + const promise = this.buffer .read() - .then(obj => this.esClient.index({ + .then(() => this.esClient.index({ + id, index: this.index, type: this.type, - id: id, - body: object + body: object, })); this.buffer.write(object); return Observable.fromPromise(promise); } -} \ No newline at end of file +} diff --git a/helpers/Buffer.ts b/src/helpers/Buffer.ts similarity index 85% rename from helpers/Buffer.ts rename to src/helpers/Buffer.ts index db7855e..ad7ba83 100644 --- a/helpers/Buffer.ts +++ b/src/helpers/Buffer.ts @@ -1,4 +1,4 @@ -import {EventEmitter} from 'events'; +import { EventEmitter } from 'events'; export class BufferSealedError extends Error { constructor() { @@ -43,9 +43,9 @@ export class Buffer extends EventEmitter { } } - public read(): Promise { + public read(): Promise { if (!this.isEmpty) { - let content = this.content.shift(); + const content = this.content.shift(); this.emit('release', content); if (this.isEmpty) { @@ -56,12 +56,12 @@ export class Buffer extends EventEmitter { return Promise.resolve(content); } - return new Promise(resolve => { + return new Promise((resolve) => { this.once('write', () => resolve(this.read())); }); } - public write(object: T): Promise | Promise { + public write(object: T): Promise { if (this.sealed) { return Promise.reject(new BufferSealedError()); } @@ -72,8 +72,8 @@ export class Buffer extends EventEmitter { return Promise.resolve(object); } - return new Promise(resolve => { + return new Promise((resolve) => { this.once('release', () => resolve(this.write(object))); }); } -} \ No newline at end of file +} diff --git a/src/index.ts b/src/index.ts new file mode 100644 index 0000000..b975181 --- /dev/null +++ b/src/index.ts @@ -0,0 +1 @@ +export * from './ElasticLoader'; diff --git a/helpers/Buffer.spec.ts b/test/Buffer.spec.ts similarity index 56% rename from helpers/Buffer.spec.ts rename to test/Buffer.spec.ts index d5e0e59..82efa29 100644 --- a/helpers/Buffer.spec.ts +++ b/test/Buffer.spec.ts @@ -1,48 +1,40 @@ -import chai = require('chai'); -import asPromised = require('chai-as-promised'); -import sinon = require('sinon'); -import sinonChai = require('sinon-chai'); -import {Buffer} from './Buffer'; - -let should = chai.should(); -chai.use(asPromised); -chai.use(sinonChai); +import { Buffer } from '../src/helpers/Buffer'; describe('Buffer', () => { - let buf:Buffer; + let buf: Buffer; beforeEach(() => { buf = new Buffer(); }); it('should set size to default 10', () => { - buf.size.should.equal(10); + expect(buf.size).toBe(10); }); it('should extend EventEmitter', () => { - should.exist(buf.addListener); + expect(buf.addListener).toBeDefined(); }); it('should be empty on init', () => { - buf.isEmpty.should.be.true; + expect(buf.isEmpty).toBeTruthy(); }); it('should not be full on init', () => { - buf.isFull.should.be.false; + expect(buf.isFull).toBeFalsy(); }); it('should change size with property', () => { - buf.size.should.equal(10); + expect(buf.size).toMatchSnapshot(); buf.size = 20; - buf.size.should.equal(20); + expect(buf.size).toMatchSnapshot(); }); describe('read()', () => { it('should return a Promise', () => { buf.write(''); - buf.read().should.be.a('Promise'); + expect(buf.read()).toBeInstanceOf(Promise); }); it('should not resolve without write while empty', done => { @@ -55,19 +47,17 @@ describe('Buffer', () => { }, 500); }); - it('should resolve on write', () => { - return Promise.all([ - buf.read().should.eventually.equal('hello'), - buf.write('hello').should.be.fulfilled - ]); + it('should resolve on write', async () => { + expect(buf.read()).resolves.toBe('hello'); + await buf.write('hello'); }); it('should emit release event', () => { - let spy = sinon.spy(); + const spy = jest.fn(); buf.on('release', spy); buf.read(); buf.write(''); - spy.should.have.callCount(1); + expect(spy.mock.calls.length).toBe(1); }); }); @@ -81,7 +71,7 @@ describe('Buffer', () => { it('should return a Promise', () => { buf.read(); - buf.write('').should.be.a('Promise'); + expect(buf.write('')).toBeInstanceOf(Promise); }); it('should not resolve without read while full', done => { @@ -94,22 +84,21 @@ describe('Buffer', () => { }, 500); }); - it('should resolve on read', () => { - return Promise.all([ - buf.write('world').should.be.fulfilled, - buf.read().should.eventually.equal('hello'), - buf.read().should.eventually.equal('world') - ]); + it('should resolve on read', async () => { + buf.write('world'); + + expect(await buf.read()).toBe('hello'); + expect(await buf.read()).toBe('world'); }); it('should emit write event', () => { - let spy = sinon.spy(); + let spy = jest.fn(); buf.on('write', spy); buf.read(); buf.write(''); - spy.should.have.callCount(1); + expect(spy.mock.calls.length).toBe(1); }); }); -}); \ No newline at end of file +}); diff --git a/ElasticLoader.spec.ts b/test/ElasticLoader.spec.ts similarity index 70% rename from ElasticLoader.spec.ts rename to test/ElasticLoader.spec.ts index 220d719..7588760 100644 --- a/ElasticLoader.spec.ts +++ b/test/ElasticLoader.spec.ts @@ -1,26 +1,18 @@ -import chai = require('chai'); -import asPromised = require('chai-as-promised'); -import sinon = require('sinon'); -import sinonChai = require('sinon-chai'); -import {Observable} from 'rxjs'; -import {ElasticLoader} from './ElasticLoader'; +import { Observable } from 'rxjs/Observable'; -let should = chai.should(); -chai.use(asPromised); -chai.use(sinonChai); +import { ElasticLoader } from '../src/ElasticLoader'; describe('ElasticLoader', () => { let loader: ElasticLoader; let client: any; - let stub: any; beforeEach(() => { client = { index: o => Observable.of(o) }; - stub = sinon.stub(client, 'index', o => Observable.of(o)); + client.index = jest.fn(client.index); }); it('should resolve on correct usage', done => { @@ -36,10 +28,8 @@ describe('ElasticLoader', () => { loader.write({id: 1, text: 'test'}).subscribe(null, done, () => { try { - client.index.should.have.been.calledOnce; - client.index.should.have.been.calledWithMatch({ - index: 'testIndex' - }); + expect(client.index.mock.calls.length).toBe(1); + expect(client.index.mock.calls[0][0]).toMatchSnapshot(); done(); } catch (e) { done(e); @@ -53,10 +43,8 @@ describe('ElasticLoader', () => { loader.write({id: 1, text: 'test'}).subscribe(null, done, () => { try { - client.index.should.have.been.calledOnce; - client.index.should.have.been.calledWithMatch({ - type: 'testType' - }); + expect(client.index.mock.calls.length).toBe(1); + expect(client.index.mock.calls[0][0]).toMatchSnapshot(); done(); } catch (e) { done(e); @@ -70,10 +58,8 @@ describe('ElasticLoader', () => { loader.write({id: 1, text: 'test'}).subscribe(null, done, () => { try { - client.index.should.have.been.calledOnce; - client.index.should.have.been.calledWithMatch({ - id: 1 - }); + expect(client.index.mock.calls.length).toBe(1); + expect(client.index.mock.calls[0][0]).toMatchSnapshot(); done(); } catch (e) { done(e); @@ -87,10 +73,8 @@ describe('ElasticLoader', () => { loader.write({id: 1, text: 'test'}).subscribe(null, done, () => { try { - client.index.should.have.been.calledOnce; - client.index.should.have.been.calledWithMatch({ - body: {id: 1, text: 'test'} - }); + expect(client.index.mock.calls.length).toBe(1); + expect(client.index.mock.calls[0][0]).toMatchSnapshot(); done(); } catch (e) { done(e); @@ -104,7 +88,7 @@ describe('ElasticLoader', () => { loader.write({id: 1, text: 'test'}).subscribe(null, done, () => { try { - client.index.should.have.been.calledOnce; + expect(client.index.mock.calls.length).toBe(1); done(); } catch (e) { done(e); @@ -118,7 +102,7 @@ describe('ElasticLoader', () => { loader.write({id: 1, text: 'test'}).subscribe(null, done, () => { try { - client.index.should.not.have.been.called; + expect(client.index.mock.calls.length).toBe(0); done(); } catch (e) { done(e); @@ -132,11 +116,8 @@ describe('ElasticLoader', () => { loader.write({myId: 1, text: 'test'}).subscribe(null, done, () => { try { - client.index.should.have.been.calledOnce; - client.index.should.have.been.calledWithMatch({ - id: 1, - body: {myId: 1, text: 'test'} - }); + expect(client.index.mock.calls.length).toBe(1); + expect(client.index.mock.calls[0][0]).toMatchSnapshot(); done(); } catch (e) { done(e); diff --git a/test/__snapshots__/Buffer.spec.ts.snap b/test/__snapshots__/Buffer.spec.ts.snap new file mode 100644 index 0000000..00cd6fd --- /dev/null +++ b/test/__snapshots__/Buffer.spec.ts.snap @@ -0,0 +1,5 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`Buffer should change size with property 1`] = `10`; + +exports[`Buffer should change size with property 2`] = `20`; diff --git a/test/__snapshots__/ElasticLoader.spec.ts.snap b/test/__snapshots__/ElasticLoader.spec.ts.snap new file mode 100644 index 0000000..1d67e58 --- /dev/null +++ b/test/__snapshots__/ElasticLoader.spec.ts.snap @@ -0,0 +1,61 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`ElasticLoader should be called with correct id when using custom selector 1`] = ` +Object { + "body": Object { + "myId": 1, + "text": "test", + }, + "id": 1, + "index": "testIndex", + "type": "testType", +} +`; + +exports[`ElasticLoader should use correct body 1`] = ` +Object { + "body": Object { + "id": 1, + "text": "test", + }, + "id": 1, + "index": "testIndex", + "type": "testType", +} +`; + +exports[`ElasticLoader should use correct id 1`] = ` +Object { + "body": Object { + "id": 1, + "text": "test", + }, + "id": 1, + "index": "testIndex", + "type": "testType", +} +`; + +exports[`ElasticLoader should use correct index 1`] = ` +Object { + "body": Object { + "id": 1, + "text": "test", + }, + "id": 1, + "index": "testIndex", + "type": "testType", +} +`; + +exports[`ElasticLoader should use correct type 1`] = ` +Object { + "body": Object { + "id": 1, + "text": "test", + }, + "id": 1, + "index": "testIndex", + "type": "testType", +} +`; diff --git a/tsconfig.json b/tsconfig.json index d9bee7e..c40f1e5 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,15 +1,7 @@ { - "compilerOptions": { - "target": "es5", - "module": "commonjs", - "moduleResolution": "node", - "removeComments": false, - "outDir": "./build", - "declaration": true, - "sourceMap": false - }, - "exclude": [ - "node_modules", - "build" - ] -} \ No newline at end of file + "extends": "./config/tsconfig.base.json", + "compilerOptions": { + "watch": true, + "sourceMap": true + } +} diff --git a/tslint.json b/tslint.json new file mode 100644 index 0000000..f26a210 --- /dev/null +++ b/tslint.json @@ -0,0 +1,56 @@ +{ + "extends": "tslint-config-airbnb", + "rules": { + "function-name": false, + "max-line-length": [ + true, + 125 + ], + "member-ordering": [ + true, + { + "order": [ + "public-static-field", + "protected-static-field", + "private-static-field", + "public-instance-field", + "protected-instance-field", + "private-instance-field", + "constructor", + "public-static-method", + "protected-static-method", + "private-static-method", + "public-instance-method", + "protected-instance-method", + "private-instance-method" + ] + } + ], + "no-increment-decrement": false, + "no-trailing-whitespace": [ + true, + "ignore-jsdoc", + "ignore-template-strings" + ], + "ter-indent": [ + true, + 4, + { + "SwitchCase": 1 + } + ], + "typedef": [ + true, + "call-signature", + "parameter", + "property-declaration", + "member-variable-declaration" + ], + "variable-name": [ + true, + "check-format", + "ban-keywords", + "allow-leading-underscore" + ] + } +} diff --git a/typings.json b/typings.json deleted file mode 100644 index 5edf886..0000000 --- a/typings.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "name": "proc-that-elastic-loader", - "dependencies": {}, - "globalDependencies": { - "es6-shim": "registry:dt/es6-shim#0.31.2+20160602141504", - "node": "registry:dt/node#6.0.0+20160709114037" - }, - "globalDevDependencies": { - "mocha": "registry:dt/mocha#2.2.5+20160619032855" - }, - "devDependencies": { - "chai": "registry:npm/chai#3.5.0+20160415060238", - "chai-as-promised": "registry:npm/chai-as-promised#5.1.0+20160310030142", - "sinon": "registry:npm/sinon#1.16.0+20160427193336", - "sinon-chai": "registry:npm/sinon-chai#2.8.0+20160310030142" - } -} \ No newline at end of file