From 002bc4c54f7345f1b2b14c27edddc495056f5786 Mon Sep 17 00:00:00 2001 From: Ali Demirci Date: Fri, 12 Nov 2021 16:22:06 +0300 Subject: [PATCH] chore(lib): initial checks --- lib/clickhouse.module.ts | 6 +- lib/client/ClickHouseClient.ts | 63 +++--- lib/enums/ClickHouseCompressionMethod.ts | 2 +- lib/index.ts | 5 +- lib/interfaces/ClickHouseModuleOptions.ts | 12 +- lib/interfaces/index.ts | 1 + package-lock.json | 236 ++++++++++++++++++++++ package.json | 6 +- yarn.lock | 5 + 9 files changed, 287 insertions(+), 49 deletions(-) create mode 100644 lib/interfaces/index.ts create mode 100644 package-lock.json diff --git a/lib/clickhouse.module.ts b/lib/clickhouse.module.ts index 8af52c1..d748f93 100644 --- a/lib/clickhouse.module.ts +++ b/lib/clickhouse.module.ts @@ -4,7 +4,7 @@ import { ClickHouseClient } from './client/ClickHouseClient'; import { ClickHouseModuleOptions } from './interfaces/ClickHouseModuleOptions'; @Module({}) -export class ClickhouseModule { +export class ClickHouseModule { static register(options: ClickHouseModuleOptions[]): DynamicModule { const clients = (options || []).map(item => { if (!item) { @@ -14,13 +14,13 @@ export class ClickhouseModule { } return { - provide: item.serverName, + provide: item.name, useValue: new ClickHouseClient(item) } }); return { - module: ClickhouseModule, + module: ClickHouseModule, providers: clients, exports: clients, }; diff --git a/lib/client/ClickHouseClient.ts b/lib/client/ClickHouseClient.ts index eeec9e7..1cf4228 100644 --- a/lib/client/ClickHouseClient.ts +++ b/lib/client/ClickHouseClient.ts @@ -21,11 +21,6 @@ export class ClickHouseClient { */ private logger = new Logger('ClickHouseModule'); - /** - * Observable Reference Store - */ - private readonly observables: Observable[] = []; - /** * ClickHouse Service */ @@ -56,7 +51,7 @@ export class ClickHouseClient { database: this.options.database }; - if (this.options.compression != ClickHouseCompressionMethod.DEFAULT) { + if (this.options.compression != ClickHouseCompressionMethod.NONE) { params['enable_http_compression'] = 1; } @@ -101,33 +96,10 @@ export class ClickHouseClient { return headers; } - /** - * Handle request errors - */ - private _handleRequestError(reason: any) { - if (reason && reason.response) { - let err: string = ''; - - reason - .response - .data - .on('data', chunk => { - err += chunk.toString('utf8') - }) - .on('end', () => { - this.logger.error(err.trim()); - - err = ''; - }); - } else { - this.logger.error(reason.code); - } - } - /** * Create a Readable Query Stream */ - public query(query: string) { + public query(query: string) { return new Observable(subscriber => { axios .request( @@ -156,7 +128,25 @@ export class ClickHouseClient { } }) .catch((reason) => { - this._handleRequestError(reason); + if (reason && reason.response) { + let err: string = ''; + + reason + .response + .data + .on('data', chunk => { + err += chunk.toString('utf8') + }) + .on('end', () => { + this.logger.error(err.trim()); + subscriber.error(err.trim()); + + err = ''; + }); + } else { + subscriber.error(reason.code); + this.logger.error(reason.code); + } }) }) } @@ -164,7 +154,7 @@ export class ClickHouseClient { /** * Insert data to table */ - public insert(table: string, data: any[]) { + public insert(table: string, data: T[]) { return new Observable(subscriber => { let query = `INSERT INTO ${table}`; let _data: any; @@ -197,14 +187,11 @@ export class ClickHouseClient { .on('end', () => { subscriber.complete(); }); - }) - .catch(c => { - console.log(c) - subscriber.error(c); - this.logger.error(c); + .catch(reason => { + subscriber.error(reason); + this.logger.error(reason); }) }); - } } \ No newline at end of file diff --git a/lib/enums/ClickHouseCompressionMethod.ts b/lib/enums/ClickHouseCompressionMethod.ts index 4abdbda..7c91fc3 100644 --- a/lib/enums/ClickHouseCompressionMethod.ts +++ b/lib/enums/ClickHouseCompressionMethod.ts @@ -1,5 +1,5 @@ export enum ClickHouseCompressionMethod { - DEFAULT, + NONE, GZIP, BROTLI, DEFLATE diff --git a/lib/index.ts b/lib/index.ts index be2f575..14e619b 100644 --- a/lib/index.ts +++ b/lib/index.ts @@ -1 +1,4 @@ -export * from './clickhouse.module'; \ No newline at end of file +export * from './clickhouse.module'; +export * from './client/ClickHouseClient' +export * from './enums'; +export * from './interfaces'; \ No newline at end of file diff --git a/lib/interfaces/ClickHouseModuleOptions.ts b/lib/interfaces/ClickHouseModuleOptions.ts index cc3108b..3570f95 100644 --- a/lib/interfaces/ClickHouseModuleOptions.ts +++ b/lib/interfaces/ClickHouseModuleOptions.ts @@ -18,8 +18,10 @@ export class ClickHouseSettings { * * If a result body is larger than this threshold, the buffer is written to the HTTP channel, and the remaining data is sent directly to the HTTP channel. * To ensure that the entire response is buffered, set wait_end_of_query=1. In this case, the data that is not stored in memory will be buffered in a temporary server file. + * + * Default: 1 */ - public wait_end_of_query?: 0 | 1 = 0; + public wait_end_of_query?: 0 | 1 = 1; /** * You can enable response buffering on the server-side. The buffer_size and wait_end_of_query URL parameters are provided for this purpose. @@ -27,17 +29,19 @@ export class ClickHouseSettings { * * If a result body is larger than this threshold, the buffer is written to the HTTP channel, and the remaining data is sent directly to the HTTP channel. * To ensure that the entire response is buffered, set wait_end_of_query=1. In this case, the data that is not stored in memory will be buffered in a temporary server file. + * + * Default: 1048576 */ public buffer_size?: number = 1048576; } export class ClickHouseModuleOptions { /** - * ClickHouse Connection Name + * ClickHouse Server Identifier * * Default: CLICKHOUSE_DEFAULT */ - public serverName?: string = 'CLICKHOUSE_DEFAULT'; + public name?: string = 'CLICKHOUSE_DEFAULT'; /** * ClickHouse Host @@ -86,7 +90,7 @@ export class ClickHouseModuleOptions { * * Default: NONE */ - public compression?: ClickHouseCompressionMethod = ClickHouseCompressionMethod.DEFAULT; + public compression?: ClickHouseCompressionMethod = ClickHouseCompressionMethod.NONE; /** * Input & Output Data Format diff --git a/lib/interfaces/index.ts b/lib/interfaces/index.ts new file mode 100644 index 0000000..34ac40a --- /dev/null +++ b/lib/interfaces/index.ts @@ -0,0 +1 @@ +export * from './ClickHouseModuleOptions'; \ No newline at end of file diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..343f4e1 --- /dev/null +++ b/package-lock.json @@ -0,0 +1,236 @@ +{ + "name": "@depyronick/nestjs-clickhouse", + "version": "1.0.0", + "lockfileVersion": 1, + "requires": true, + "dependencies": { + "@nestjs/common": { + "version": "8.2.0", + "resolved": "https://registry.npmjs.org/@nestjs/common/-/common-8.2.0.tgz", + "integrity": "sha512-Vska6eEQtnwM9CBnL7QINQUE8Xq+d4s/CXkpxrUab1HgTA9m5LjmkCPdLw7tBFxMQ6Px+kmcXkQI3rKMfq9Wzw==", + "dev": true, + "requires": { + "axios": "0.24.0", + "iterare": "1.2.1", + "tslib": "2.3.1", + "uuid": "8.3.2" + } + }, + "@nestjs/core": { + "version": "8.2.0", + "resolved": "https://registry.npmjs.org/@nestjs/core/-/core-8.2.0.tgz", + "integrity": "sha512-TjN705n0DQrcL8hN1021dITol2BgzIT+FFv3RPKJWMwck8y0ebWK0PTI4vXeLial75fPd+4K4LhL6nAO4WyjFw==", + "dev": true, + "requires": { + "@nuxtjs/opencollective": "0.3.2", + "fast-safe-stringify": "2.1.1", + "iterare": "1.2.1", + "object-hash": "2.2.0", + "path-to-regexp": "3.2.0", + "tslib": "2.3.1", + "uuid": "8.3.2" + } + }, + "@nuxtjs/opencollective": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/@nuxtjs/opencollective/-/opencollective-0.3.2.tgz", + "integrity": "sha512-um0xL3fO7Mf4fDxcqx9KryrB7zgRM5JSlvGN5AGkP6JLM5XEKyjeAiPbNxdXVXQ16isuAhYpvP88NgL2BGd6aA==", + "dev": true, + "requires": { + "chalk": "^4.1.0", + "consola": "^2.15.0", + "node-fetch": "^2.6.1" + } + }, + "@types/node": { + "version": "16.11.7", + "resolved": "https://registry.npmjs.org/@types/node/-/node-16.11.7.tgz", + "integrity": "sha512-QB5D2sqfSjCmTuWcBWyJ+/44bcjO7VbjSbOE0ucoVbAsSNQc4Lt6QkgkVXkTDwkL4z/beecZNDvVX15D4P8Jbw==", + "dev": true + }, + "@types/stream-chain": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@types/stream-chain/-/stream-chain-2.0.1.tgz", + "integrity": "sha512-D+Id9XpcBpampptkegH7WMsEk6fUdf9LlCIX7UhLydILsqDin4L0QT7ryJR0oycwC7OqohIzdfcMHVZ34ezNGg==", + "dev": true, + "requires": { + "@types/node": "*" + } + }, + "@types/stream-json": { + "version": "1.7.1", + "resolved": "https://registry.npmjs.org/@types/stream-json/-/stream-json-1.7.1.tgz", + "integrity": "sha512-BNIK/ix6iJvWvoXbDVVJhw5LNG1wie/rXcUo7jw4hBqY3FhIrg0e+RMXFN5UreKclBIStl9FDEHNSDLuuQ9/MQ==", + "dev": true, + "requires": { + "@types/node": "*", + "@types/stream-chain": "*" + } + }, + "ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "requires": { + "color-convert": "^2.0.1" + } + }, + "axios": { + "version": "0.24.0", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.24.0.tgz", + "integrity": "sha512-Q6cWsys88HoPgAaFAVUb0WpPk0O8iTeisR9IMqy9G8AbO4NlpVknrnQS03zzF9PGAWgO3cgletO3VjV/P7VztA==", + "requires": { + "follow-redirects": "^1.14.4" + } + }, + "chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, + "requires": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + } + }, + "color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "requires": { + "color-name": "~1.1.4" + } + }, + "color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true + }, + "consola": { + "version": "2.15.3", + "resolved": "https://registry.npmjs.org/consola/-/consola-2.15.3.tgz", + "integrity": "sha512-9vAdYbHj6x2fLKC4+oPH0kFzY/orMZyG2Aj+kNylHxKGJ/Ed4dpNyAQYwJOdqO4zdM7XpVHmyejQDcQHrnuXbw==", + "dev": true + }, + "fast-safe-stringify": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/fast-safe-stringify/-/fast-safe-stringify-2.1.1.tgz", + "integrity": "sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA==", + "dev": true + }, + "follow-redirects": { + "version": "1.14.5", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.5.tgz", + "integrity": "sha512-wtphSXy7d4/OR+MvIFbCVBDzZ5520qV8XfPklSN5QtxuMUJZ+b0Wnst1e1lCDocfzuCkHqj8k0FpZqO+UIaKNA==" + }, + "has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true + }, + "iterare": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/iterare/-/iterare-1.2.1.tgz", + "integrity": "sha512-RKYVTCjAnRthyJes037NX/IiqeidgN1xc3j1RjFfECFp28A1GVwK9nA+i0rJPaHqSZwygLzRnFlzUuHFoWWy+Q==", + "dev": true + }, + "node-fetch": { + "version": "2.6.6", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.6.tgz", + "integrity": "sha512-Z8/6vRlTUChSdIgMa51jxQ4lrw/Jy5SOW10ObaA47/RElsAN2c5Pn8bTgFGWn/ibwzXTE8qwr1Yzx28vsecXEA==", + "dev": true, + "requires": { + "whatwg-url": "^5.0.0" + } + }, + "object-hash": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/object-hash/-/object-hash-2.2.0.tgz", + "integrity": "sha512-gScRMn0bS5fH+IuwyIFgnh9zBdo4DV+6GhygmWM9HyNJSgS0hScp1f5vjtm7oIIOiT9trXrShAkLFSc2IqKNgw==", + "dev": true + }, + "path-to-regexp": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-3.2.0.tgz", + "integrity": "sha512-jczvQbCUS7XmS7o+y1aEO9OBVFeZBQ1MDSEqmO7xSoPgOPoowY/SxLpZ6Vh97/8qHZOteiCKb7gkG9gA2ZUxJA==", + "dev": true + }, + "rxjs": { + "version": "7.4.0", + "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-7.4.0.tgz", + "integrity": "sha512-7SQDi7xeTMCJpqViXh8gL/lebcwlp3d831F05+9B44A4B0WfsEwUQHR64gsH1kvJ+Ep/J9K2+n1hVl1CsGN23w==", + "dev": true, + "requires": { + "tslib": "~2.1.0" + }, + "dependencies": { + "tslib": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.1.0.tgz", + "integrity": "sha512-hcVC3wYEziELGGmEEXue7D75zbwIIVUMWAVbHItGPx0ziyXxrOMQx4rQEVEV45Ut/1IotuEvwqPopzIOkDMf0A==", + "dev": true + } + } + }, + "stream-chain": { + "version": "2.2.4", + "resolved": "https://registry.npmjs.org/stream-chain/-/stream-chain-2.2.4.tgz", + "integrity": "sha512-9lsl3YM53V5N/I1C2uJtc3Kavyi3kNYN83VkKb/bMWRk7D9imiFyUPYa0PoZbLohSVOX1mYE9YsmwObZUsth6Q==" + }, + "stream-json": { + "version": "1.7.3", + "resolved": "https://registry.npmjs.org/stream-json/-/stream-json-1.7.3.tgz", + "integrity": "sha512-Y6dXn9KKWSwxOqnvHGcdZy1PK+J+7alBwHCeU3W9oRqm4ilLRA0XSPmd1tWwhg7tv9EIxJTMWh7KF15tYelKJg==", + "requires": { + "stream-chain": "^2.2.4" + } + }, + "supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "requires": { + "has-flag": "^4.0.0" + } + }, + "tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha1-gYT9NH2snNwYWZLzpmIuFLnZq2o=", + "dev": true + }, + "tslib": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.3.1.tgz", + "integrity": "sha512-77EbyPPpMz+FRFRuAFlWMtmgUWGe9UOG2Z25NqCwiIjRhOf5iKGuzSe5P2w1laq+FkRy4p+PCuVkJSGkzTEKVw==", + "dev": true + }, + "uuid": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", + "dev": true + }, + "webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha1-JFNCdeKnvGvnvIZhHMFq4KVlSHE=", + "dev": true + }, + "whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha1-lmRU6HZUYuN2RNNib2dCzotwll0=", + "dev": true, + "requires": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + } + } +} diff --git a/package.json b/package.json index 26f2081..3cabc6a 100644 --- a/package.json +++ b/package.json @@ -16,11 +16,13 @@ "@nestjs/core": "8.2.0", "@types/node": "^16.11.7", "@types/stream-json": "^1.7.1", - "rxjs": "^7.4.0" + "rxjs": "^7.4.0", + "reflect-metadata": "0.1.13" }, "peerDependencies": { "@nestjs/common": "^8.0.0", - "rxjs": "^7.4.0" + "rxjs": "^7.4.0", + "reflect-metadata": "^0.1.13" }, "dependencies": { "axios": "^0.24.0", diff --git a/yarn.lock b/yarn.lock index 90f8a59..a11d730 100644 --- a/yarn.lock +++ b/yarn.lock @@ -130,6 +130,11 @@ path-to-regexp@3.2.0: resolved "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-3.2.0.tgz#fa7877ecbc495c601907562222453c43cc204a5f" integrity sha512-jczvQbCUS7XmS7o+y1aEO9OBVFeZBQ1MDSEqmO7xSoPgOPoowY/SxLpZ6Vh97/8qHZOteiCKb7gkG9gA2ZUxJA== +reflect-metadata@0.1.13: + version "0.1.13" + resolved "https://registry.yarnpkg.com/reflect-metadata/-/reflect-metadata-0.1.13.tgz#67ae3ca57c972a2aa1642b10fe363fe32d49dc08" + integrity sha512-Ts1Y/anZELhSsjMcU605fU9RE4Oi3p5ORujwbIKXfWa+0Zxs510Qrmrce5/Jowq3cHSZSJqBjypxmHarc+vEWg== + rxjs@^7.4.0: version "7.4.0" resolved "https://registry.yarnpkg.com/rxjs/-/rxjs-7.4.0.tgz#a12a44d7eebf016f5ff2441b87f28c9a51cebc68"