From 6b0b3d4ea871cc1eb851322e1c991b48f496bcf2 Mon Sep 17 00:00:00 2001 From: Steven Myers Date: Sat, 17 Mar 2018 15:38:08 -0600 Subject: [PATCH] implement data management --- .gitignore | 1 + integrationTests/seed/index.ts | 2 - src/modules/Program/program.service.ts | 2 - src/modules/api/api.controller.ts | 3 +- src/modules/app.module.ts | 4 +- src/modules/backup/README.md | 11 -- src/modules/constants.readonly.ts | 26 ++- src/modules/data/Schema.ts | 15 ++ src/modules/data/backup.service.ts | 81 ++++++++ src/modules/data/data.controller.ts | 28 +++ src/modules/data/data.module.ts | 21 ++ src/modules/data/init.service.ts | 128 ++++++++++++ src/modules/data/upload.service.ts | 185 ++++++++++++++++++ .../db.elasticsearch/client.service.ts | 1 - src/modules/protected/protected.controller.ts | 2 - src/modules/query/ApplicationQuery.service.ts | 1 - src/modules/screener/screener.service.ts | 2 - 17 files changed, 482 insertions(+), 31 deletions(-) delete mode 100644 src/modules/backup/README.md create mode 100644 src/modules/data/Schema.ts create mode 100644 src/modules/data/backup.service.ts create mode 100644 src/modules/data/data.controller.ts create mode 100644 src/modules/data/data.module.ts create mode 100644 src/modules/data/init.service.ts create mode 100644 src/modules/data/upload.service.ts diff --git a/.gitignore b/.gitignore index b5e5f97..590ea1c 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ /.idea /.awcache /.vscode +integrationTests/seed/.idea/ # misc npm-debug.log diff --git a/integrationTests/seed/index.ts b/integrationTests/seed/index.ts index d289af7..c574df8 100644 --- a/integrationTests/seed/index.ts +++ b/integrationTests/seed/index.ts @@ -36,7 +36,6 @@ export class Seeder { ]).catch(e => { console.error(e); - console.log("HERHEHRE"); process.exit(1); }); @@ -76,7 +75,6 @@ export class Seeder { index, }) .catch(err => { - console.log("bebrebrbe"); console.log("\x1b[31m", err); process.exit(69); return Error(err) diff --git a/src/modules/Program/program.service.ts b/src/modules/Program/program.service.ts index 3ecb106..68ff16d 100644 --- a/src/modules/Program/program.service.ts +++ b/src/modules/Program/program.service.ts @@ -30,11 +30,9 @@ export class ProgramService { findAll(): Observable { return Observable.fromPromise( this.clientService.findAll(this.baseParams) ) - // .do(programs => programs.map(({guid}) => guid).forEach(console.log)); } index(program: ProgramDto) { - console.log(program); program.created = Date.now(); return this.clientService.index(program, this.INDEX, this.TYPE, program.guid) } diff --git a/src/modules/api/api.controller.ts b/src/modules/api/api.controller.ts index a0cbc1e..851970b 100644 --- a/src/modules/api/api.controller.ts +++ b/src/modules/api/api.controller.ts @@ -33,7 +33,6 @@ export class ApiController { @Post('notification') getProgramsFromForm(@Body() body): Observable { - console.log(body) - return this.percolateService.precolate(body).do(console.log) + return this.percolateService.precolate(body); } } \ No newline at end of file diff --git a/src/modules/app.module.ts b/src/modules/app.module.ts index f4370e6..157277e 100644 --- a/src/modules/app.module.ts +++ b/src/modules/app.module.ts @@ -5,6 +5,7 @@ import { QueryModule } from "./query" import { KeyModule } from "./key" import { ProtectedModule } from "./protected" import { ApiModule } from "./api" +import { DataModule } from './data/data.module'; @Module({ modules: [ @@ -12,7 +13,8 @@ import { ApiModule } from "./api" QueryModule, KeyModule, ProtectedModule, - ApiModule + ApiModule, + DataModule ], controllers: [AppController], components: [], diff --git a/src/modules/backup/README.md b/src/modules/backup/README.md deleted file mode 100644 index e8a341e..0000000 --- a/src/modules/backup/README.md +++ /dev/null @@ -1,11 +0,0 @@ -### Backup Module - -The plan is to implement an endpoint that can extract the -data from elasticsearch. - -Data will be small in total... likely under 1MB. - -Also, consider a cron job in this module to do rolling backups -on the file system. - -Consider exporting these backups to Google drive. \ No newline at end of file diff --git a/src/modules/constants.readonly.ts b/src/modules/constants.readonly.ts index 21863ad..2763660 100644 --- a/src/modules/constants.readonly.ts +++ b/src/modules/constants.readonly.ts @@ -1,15 +1,27 @@ -const determineHost = () => { +export class ConstantsReadonly { + readonly host:string = determineHost(); + readonly logLevel:string = determineLogLevel(); + readonly domain:string = determineTenant(); +} + +function determineHost() { if (process.env.NODE_ENV === "INTEGRATION_TEST") return "http://localhost:9400"; if (process.env.COMPOSER_ENV === "true") return `elasticsearch:${process.env.ELASTICSEARCH_PORT}`; return "http://localhost:9200" -}; +} +function determineLogLevel() { + if (process.env.NODE_ENV === "development") return 'trace'; -export class ConstantsReadonly { - readonly host:string = determineHost(); - readonly logLevel:string = process.env.COMPOSER_ENV === "true"? '' : 'trace'; - private readonly _domain:string = "EDMONTON"; - readonly domain:string = process.env.NODE_ENV === "development" ? "devel" : this._domain + if (process.env.COMPOSER_ENV === "true") return ''; + + return 'trace'; +} + +function determineTenant() { + if (process.env.NODE_ENV === "development") return 'devel'; + + return "EDMONTON" } \ No newline at end of file diff --git a/src/modules/data/Schema.ts b/src/modules/data/Schema.ts new file mode 100644 index 0000000..9c600f1 --- /dev/null +++ b/src/modules/data/Schema.ts @@ -0,0 +1,15 @@ +export const Schema = { + "master_screener": { + index: "questions", + type: "screener", + + }, + "queries": { + index: "master_screener", + type: "queries", + }, + "programs": { + index: "programs", + type: "user_facing", + } +}; \ No newline at end of file diff --git a/src/modules/data/backup.service.ts b/src/modules/data/backup.service.ts new file mode 100644 index 0000000..eb7a0d0 --- /dev/null +++ b/src/modules/data/backup.service.ts @@ -0,0 +1,81 @@ +import {Component} from '@nestjs/common'; +import {ClientService} from '../db.elasticsearch/client.service'; +import { Client } from "elasticsearch" +import {Schema} from './Schema'; + +@Component() +export class BackupService { + private client: Client; + private readonly PAGE_SIZE = 10000; + + constructor(private clientService: ClientService){ + this.client = this.clientService.client + } + + async execute(): Promise { + const requests = [ + this.client.search({ + index: Schema.programs.index, + type: Schema.programs.type, + size: this.PAGE_SIZE, + body: { query: { match_all: {} } } + }), + + this.client.indices.getMapping({ + index: Schema.programs.index, + type: Schema.programs.type + }), + + this.client.search({ + index: Schema.queries.index, + type: Schema.queries.type, + size: this.PAGE_SIZE, + body: { query: { match_all: {} } } + }), + + this.client.indices.getMapping({ + index: Schema.queries.index, + type: Schema.queries.type + }), + + this.client.search( { + index: Schema.master_screener.index, + type: Schema.master_screener.type, + size: this.PAGE_SIZE, + body: { query: { match_all: {} } } + }), + + this.client.indices.getMapping({ + index: Schema.master_screener.index, + type: Schema.master_screener.type + }), + ]; + + const [ + programs, + programMappings, + queries, + queryMappings, + master_screener, + screenerMappings + ] = await Promise.all(requests); + + return { + programs: this.filterSource(programs), + queries: this.filterSource(queries), + screener: this.getRecentScreener(this.filterSource(master_screener)), + programMappings, + queryMappings, + screenerMappings + }; + } + + private filterSource(result): any[] { + return result.hits.hits.map(h => h._source); + } + + private getRecentScreener(results: any[]): any { + const max = Math.max.apply(Math, results.map(screener => screener.created)); + return results.find(r => r.created === max) || {} + } +} \ No newline at end of file diff --git a/src/modules/data/data.controller.ts b/src/modules/data/data.controller.ts new file mode 100644 index 0000000..61f5896 --- /dev/null +++ b/src/modules/data/data.controller.ts @@ -0,0 +1,28 @@ +import {Body, Controller, Get, Post} from '@nestjs/common'; +import {InitService} from './init.service'; +import {BackupService} from './backup.service'; +import {UploadService} from './upload.service'; + +@Controller('data') +export class DataController { + constructor( + private initService: InitService, + private backupService: BackupService, + private uploadService: UploadService + ){} + + @Get('/backup') + downloadData(): Promise { + return this.backupService.execute() + } + + @Post('/init') + init(@Body() body): Promise { + return this.initService.initialize(body.force) + } + + @Post('/upload') + upload(@Body() body): Promise { + return this.uploadService.execute(body) + } +} diff --git a/src/modules/data/data.module.ts b/src/modules/data/data.module.ts new file mode 100644 index 0000000..24cd104 --- /dev/null +++ b/src/modules/data/data.module.ts @@ -0,0 +1,21 @@ +import { Module } from '@nestjs/common'; +import { DbElasticsearchModule } from '../db.elasticsearch/db.elasticsearch.module'; +import {DataController} from './data.controller'; +import {InitService} from './init.service'; +import {BackupService} from './backup.service'; +import {UploadService} from './upload.service'; + +@Module({ + modules: [ + DbElasticsearchModule, + ], + controllers: [ + DataController + ], + components: [ + InitService, + BackupService, + UploadService + ], +}) +export class DataModule {} diff --git a/src/modules/data/init.service.ts b/src/modules/data/init.service.ts new file mode 100644 index 0000000..9288691 --- /dev/null +++ b/src/modules/data/init.service.ts @@ -0,0 +1,128 @@ +import { Component } from '@nestjs/common'; +import { Client } from "elasticsearch" +import { ClientService } from '../db.elasticsearch/client.service'; + +@Component() +export class InitService { + private client: Client; + + constructor(private clientService: ClientService){ + this.client = this.clientService.client + } + + + async initialize(force = false): Promise { + const masterScreenerExists = await this.client.indices.exists({ index: 'master_screener'}); + const questionsExists = await this.client.indices.exists({ index: 'questions'}); + const programsExists = await this.client.indices.exists({ index: 'programs'}); + + const hasBeenInitialized = masterScreenerExists || questionsExists || programsExists; + + if (hasBeenInitialized && !force) { + throw new Error("Database has already been initialized."); + } + + if (masterScreenerExists) { + await this.client.indices.delete({ index: 'master_screener'}); + } + + if (questionsExists) { + await this.client.indices.delete({ index: 'questions'}); + } + + if (programsExists) { + await this.client.indices.delete({ index: 'programs'}); + } + + await this.client.indices.create({ index: 'master_screener'}); + const masterScreenerPutMapping = await this.client.indices.putMapping({ + index: 'master_screener', + type: 'queries', + body: { properties: { ...PERCOLATOR_MAPPING } } + }); + + await this.client.indices.create({ index: 'questions'}); + const questionScreenerMapping = await this.client.indices.putMapping({ + index: 'questions', + type: 'screener', + body: { properties: { ...SCREENER_MAPPING } } + }); + + await this.client.indices.create({ index: 'programs'}); + const programsMapping = await this.client.indices.putMapping({ + index: 'programs', + type: 'user_facing', + body: { properties: { ...PROGRAM_MAPPING } } + }); + + return [ + [ masterScreenerExists, masterScreenerPutMapping], + [ questionsExists, questionScreenerMapping ], + [ programsExists, programsMapping ] + ] + } +} + +const PERCOLATOR_MAPPING = { + "query": { "type": "percolator" }, + "meta":{ + "properties":{ + "id":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}}, + "program_guid":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}} + } + } +}; + +const PROGRAM_MAPPING = { + "created":{"type":"date"}, + "description":{"type":"text", "fields":{"keyword":{"type":"keyword","ignore_above":256}}}, + "detailLinks":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}}, + "details":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}}, + "externalLink":{"type":"text"}, + "guid":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}}, + "tags":{"type":"keyword"}, + "title":{"type":"text"} +}; + +const SCREENER_MAPPING = { + "conditionalQuestions":{ + "properties":{ + "controlType":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}}, + "expandable":{"type":"boolean"}, + "id":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}}, + "index":{"type":"long"}, + "key":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}}, + "label":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}}, + "multiSelectOptions":{ + "properties":{ + "key":{ + "properties":{ + "name":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}}, + "type":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}} + } + }, + "text":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}}}}, + "options":{"type":"long"}}}, + "created":{"type":"long"}, + "questions":{ + "properties":{ + "conditionalQuestions":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}}, + "controlType":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}}, + "expandable":{"type":"boolean"}, + "id":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}}, + "index":{"type":"long"}, + "key":{"type":"text", "fields":{"keyword":{"type":"keyword","ignore_above":256}}}, + "label":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}}, + "multiSelectOptions":{ + "properties":{ + "key":{ + "properties":{ + "name":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}}, + "type":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}}} + }, + "text":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}} + } + } + } + } +}; \ No newline at end of file diff --git a/src/modules/data/upload.service.ts b/src/modules/data/upload.service.ts new file mode 100644 index 0000000..68a6511 --- /dev/null +++ b/src/modules/data/upload.service.ts @@ -0,0 +1,185 @@ +import {Component} from '@nestjs/common'; +import { Client } from "elasticsearch" +import {ClientService} from '../db.elasticsearch/client.service'; +import {Schema} from './Schema'; + +@Component() +export class UploadService { + private client: Client; + + constructor(private clientService: ClientService){ + this.client = this.clientService.client + } + + async execute(data) { + let screenerRes, + programRes, + queryRes, + programMappings, + queryMappings, + screenerMappings; + + const masterScreenerExists = await this.client.indices.exists({ index: Schema.queries.index}); + const questionsExists = await this.client.indices.exists({ index: Schema.master_screener.index}); + const programsExists = await this.client.indices.exists({ index: Schema.programs.index}); + + if (masterScreenerExists) { + await this.client.indices.delete({ index: Schema.queries.index}); + } + await this.client.indices.create({ index: Schema.queries.index}); + if (data.queryMappings) { + const normalizedMapping = this.normaliseMapping( + data, + Schema.queries.type, + Schema.queries.index, + 'queryMappings' + ); + queryMappings = await this.client.indices.putMapping({ + index: Schema.queries.index, + type: Schema.queries.type, + body: { properties: { ...normalizedMapping } } + }) + } else { + throw new Error("No queryMappings") + } + if (data.queries) { + queryRes = await this.uploadQueries(data.queries) + } else { + throw new Error("No queries") + } + + if (programsExists) { + await this.client.indices.delete({ index: Schema.programs.index}); + } + await this.client.indices.create({ index: Schema.programs.index}); + if (data.programMappings) { + const normalizedMapping = this.normaliseMapping( + data, + Schema.programs.type, + Schema.programs.index, + 'programMappings' + ); + programMappings = await this.client.indices.putMapping({ + index: Schema.programs.index, + type: Schema.programs.type, + body: { properties: { ...normalizedMapping } } + }) + } else { + throw new Error("No programMappings") + } + if (data.programs) { + programRes = await this.uploadPrograms(data.programs) + } else { + throw new Error("No programs") + } + + if (questionsExists) { + await this.client.indices.delete({ index: Schema.master_screener.index}); + } + await this.client.indices.create({ index: Schema.master_screener.index}); + if (data.screenerMappings) { + const normalizedMapping = this.normaliseMapping( + data, + Schema.master_screener.type, + Schema.master_screener.index, + 'screenerMappings' + ); + screenerMappings = await this.client.indices.putMapping({ + index: Schema.master_screener.index, + type: Schema.master_screener.type, + body: { properties: { ...normalizedMapping } } + }) + } else { + throw new Error("No screenerMappings") + } + if (data.screener) { + screenerRes = await this.uploadScreener(data.screener) + } else { + throw new Error("No screener") + } + + return { + screenerRes, + programRes, + queryRes, + queryMappings, + programMappings, + screenerMappings + } + } + + private uploadScreener(screener): Promise { + screener['created'] = Date.now(); + + return this.client.index({ + index: Schema.master_screener.index, + type: Schema.master_screener.type, + body: screener + }).catch(err => { + console.log("\x1b[31m", 'ERROR: uploading screener'); + console.log(err); + process.exit(100); + return new Error(err) + }) + } + + private uploadPrograms(programs): Promise { + return Promise.all(this.uploadProgramsWithOverwrite(programs)) + } + + private uploadProgramsWithOverwrite(programs): Promise[] { + return programs.map(program => this.uploadProgram(program)) + } + + private uploadProgram(program): Promise { + return this.client.index({ + index: Schema.programs.index, + type: Schema.programs.type, + id: program.guid, + body: program + }).catch(err => { + console.log("\x1b[31m", 'ERROR: uploading program'); + console.log(err); + process.exit(101); + return new Error(err) + }) + } + + private uploadQueries(queries): Promise { + const _queries = this.uploadQueriesWithOverwrite(queries); + return Promise.all(_queries) + } + + private uploadQueriesWithOverwrite(queries): Promise[] { + return queries.map( (query, i) => this.client.index( { + index: Schema.queries.index, + type: Schema.queries.type, + id: query['meta'].id, + body: { + query: query['query'], + meta: query['meta'] + } + }).catch(err => { + console.log("\x1b[31m", 'ERROR: uploading queries'); + console.log(err); + process.exit(102); + return new Error(err) + }) + ) + } + + normaliseMapping(mappings, type, index, container) { + let val; + + if (mappings[container].mappings) { + val = mappings[container].mappings[type].properties; + } else if (mappings[container][index]) { + val = mappings[container][index].mappings[type].properties + } else { + throw new Error("FOILED AGAIN!"); + } + + return val; + + } +} \ No newline at end of file diff --git a/src/modules/db.elasticsearch/client.service.ts b/src/modules/db.elasticsearch/client.service.ts index 44ea3bf..e81d4d0 100644 --- a/src/modules/db.elasticsearch/client.service.ts +++ b/src/modules/db.elasticsearch/client.service.ts @@ -57,7 +57,6 @@ export class ClientService { return err }) .then((res: any) => { - console.log(res) return { created: res.created || null, result: res.result} } ) } diff --git a/src/modules/protected/protected.controller.ts b/src/modules/protected/protected.controller.ts index 9ed2544..8367de9 100644 --- a/src/modules/protected/protected.controller.ts +++ b/src/modules/protected/protected.controller.ts @@ -21,8 +21,6 @@ const path = require('path'); @Controller('protected') export class ProtectedController { - private readonly constants = new ConstantsReadonly(); - constructor( private programService: ProgramService, private queryService: ApplicationQueryService, diff --git a/src/modules/query/ApplicationQuery.service.ts b/src/modules/query/ApplicationQuery.service.ts index b9fca71..cd74ee0 100644 --- a/src/modules/query/ApplicationQuery.service.ts +++ b/src/modules/query/ApplicationQuery.service.ts @@ -32,7 +32,6 @@ export class ApplicationQueryService { async index(applicationDto: ApplicationQueryDto): Promise { const model = new EsQueryModel(applicationDto); const esDto = model.buildEsQuery(); - console.log(esDto); return this.queryService.index(esDto) } diff --git a/src/modules/screener/screener.service.ts b/src/modules/screener/screener.service.ts index b1318b8..7372e88 100644 --- a/src/modules/screener/screener.service.ts +++ b/src/modules/screener/screener.service.ts @@ -22,8 +22,6 @@ export class ScreenerService { } update(data: ScreenerDto, id?: string): Observable<{[key: string]: boolean}> { - console.log(id); - console.log(data); data.created = Date.now(); return Observable.fromPromise( this.clientService.index(data, this.INDEX, this.TYPE, id || this.constants.domain) ); }