implement data management

.gitignore

@@ -5,6 +5,7 @@
 /.idea
 /.awcache
 /.vscode
+integrationTests/seed/.idea/
 
 # misc
 npm-debug.log

integrationTests/seed/index.ts

@@ -36,7 +36,6 @@ export class Seeder {
 
         ]).catch(e => {
             console.error(e);
-            console.log("HERHEHRE");
             process.exit(1);
         });
 
@@ -76,7 +75,6 @@ export class Seeder {
             index,
         })
             .catch(err => {
-                console.log("bebrebrbe");
                 console.log("\x1b[31m", err);
                 process.exit(69);
                 return Error(err)

src/modules/Program/program.service.ts

@@ -30,11 +30,9 @@ export class ProgramService {
 
     findAll(): Observable<ProgramDto[]> {
         return Observable.fromPromise( this.clientService.findAll(this.baseParams) )
-           // .do(programs => programs.map(({guid}) => guid).forEach(console.log));
     }
 
     index(program: ProgramDto) {
-        console.log(program);
         program.created = Date.now();
         return this.clientService.index(program, this.INDEX, this.TYPE, program.guid)
     }

src/modules/api/api.controller.ts

@@ -33,7 +33,6 @@ export class ApiController {
 
     @Post('notification')
     getProgramsFromForm(@Body() body): Observable<ProgramDto[]> {
-        console.log(body)
-        return this.percolateService.precolate(body).do(console.log)
+        return this.percolateService.precolate(body);
     }
 }

src/modules/app.module.ts

@@ -5,14 +5,16 @@ import { QueryModule } from "./query"
 import { KeyModule } from "./key"
 import { ProtectedModule } from "./protected"
 import { ApiModule } from "./api"
+import { DataModule } from './data/data.module';
 
 @Module({
   modules: [
       ProgramModule,
       QueryModule,
       KeyModule,
       ProtectedModule,
-      ApiModule
+      ApiModule,
+      DataModule
   ],
   controllers: [AppController],
   components: [],

src/modules/constants.readonly.ts

@@ -1,15 +1,27 @@
-const determineHost = () => {
+export class ConstantsReadonly {
+    readonly host:string = determineHost();
+    readonly logLevel:string = determineLogLevel();
+    readonly domain:string = determineTenant();
+}
+
+function determineHost() {
     if (process.env.NODE_ENV === "INTEGRATION_TEST") return "http://localhost:9400";
 
     if (process.env.COMPOSER_ENV === "true") return `elasticsearch:${process.env.ELASTICSEARCH_PORT}`;
 
     return "http://localhost:9200"
-};
+}
 
+function determineLogLevel() {
+    if (process.env.NODE_ENV === "development") return 'trace';
 
-export class ConstantsReadonly {
-    readonly host:string = determineHost();
-    readonly logLevel:string = process.env.COMPOSER_ENV === "true"? '' : 'trace';
-    private readonly _domain:string = "EDMONTON";
-    readonly domain:string = process.env.NODE_ENV === "development" ? "devel" : this._domain
+    if (process.env.COMPOSER_ENV === "true") return '';
+
+    return 'trace';
+}
+
+function determineTenant() {
+    if (process.env.NODE_ENV === "development") return 'devel';
+
+    return "EDMONTON"
 }

src/modules/data/Schema.ts

@@ -0,0 +1,15 @@
+export const Schema = {
+  "master_screener": {
+    index: "questions",
+    type: "screener",
+
+  },
+  "queries": {
+    index: "master_screener",
+    type: "queries",
+  },
+  "programs": {
+    index: "programs",
+    type: "user_facing",
+  }
+};

src/modules/data/backup.service.ts

@@ -0,0 +1,81 @@
+import {Component} from '@nestjs/common';
+import {ClientService} from '../db.elasticsearch/client.service';
+import { Client } from "elasticsearch"
+import {Schema} from './Schema';
+
+@Component()
+export class BackupService {
+    private client: Client;
+    private readonly PAGE_SIZE = 10000;
+
+    constructor(private clientService: ClientService){
+        this.client = this.clientService.client
+    }
+
+    async execute(): Promise<Object> {
+        const requests = [
+            this.client.search({
+                index: Schema.programs.index,
+                type: Schema.programs.type,
+                size: this.PAGE_SIZE,
+                body: { query: { match_all: {} } }
+            }),
+
+            this.client.indices.getMapping({
+                index: Schema.programs.index,
+                type: Schema.programs.type
+            }),
+
+            this.client.search({
+                index: Schema.queries.index,
+                type: Schema.queries.type,
+                size: this.PAGE_SIZE,
+                body: { query: { match_all: {} } }
+            }),
+
+            this.client.indices.getMapping({
+                index: Schema.queries.index,
+                type: Schema.queries.type
+            }),
+
+            this.client.search( {
+                index: Schema.master_screener.index,
+                type: Schema.master_screener.type,
+                size: this.PAGE_SIZE,
+                body: { query: { match_all: {} } }
+            }),
+
+            this.client.indices.getMapping({
+                index: Schema.master_screener.index,
+                type: Schema.master_screener.type
+            }),
+        ];
+
+        const [
+            programs,
+            programMappings,
+            queries,
+            queryMappings,
+            master_screener,
+            screenerMappings
+        ] = await Promise.all(requests);
+
+        return {
+            programs: this.filterSource(programs),
+            queries:  this.filterSource(queries),
+            screener: this.getRecentScreener(this.filterSource(master_screener)),
+            programMappings,
+            queryMappings,
+            screenerMappings
+        };
+    }
+
+    private filterSource(result): any[] {
+        return result.hits.hits.map(h => h._source);
+    }
+
+    private getRecentScreener(results: any[]): any {
+        const max = Math.max.apply(Math, results.map(screener => screener.created));
+        return results.find(r => r.created  === max) || {}
+    }
+}

src/modules/data/data.controller.ts

@@ -0,0 +1,28 @@
+import {Body, Controller, Get, Post} from '@nestjs/common';
+import {InitService} from './init.service';
+import {BackupService} from './backup.service';
+import {UploadService} from './upload.service';
+
+@Controller('data')
+export class DataController {
+    constructor(
+        private initService: InitService,
+        private backupService: BackupService,
+        private uploadService: UploadService
+    ){}
+
+    @Get('/backup')
+    downloadData(): Promise<any> {
+        return this.backupService.execute()
+    }
+
+    @Post('/init')
+    init(@Body() body): Promise<any> {
+        return this.initService.initialize(body.force)
+    }
+
+    @Post('/upload')
+    upload(@Body() body): Promise<any> {
+        return this.uploadService.execute(body)
+    }
+}

src/modules/data/data.module.ts

@@ -0,0 +1,21 @@
+import { Module } from '@nestjs/common';
+import { DbElasticsearchModule } from '../db.elasticsearch/db.elasticsearch.module';
+import {DataController} from './data.controller';
+import {InitService} from './init.service';
+import {BackupService} from './backup.service';
+import {UploadService} from './upload.service';
+
+@Module({
+    modules: [
+        DbElasticsearchModule,
+    ],
+    controllers: [
+        DataController
+    ],
+    components: [
+        InitService,
+        BackupService,
+        UploadService
+    ],
+})
+export class DataModule {}

src/modules/data/init.service.ts

@@ -0,0 +1,128 @@
+import { Component } from '@nestjs/common';
+import { Client } from "elasticsearch"
+import { ClientService } from '../db.elasticsearch/client.service';
+
+@Component()
+export class InitService {
+    private client: Client;
+
+    constructor(private clientService: ClientService){
+        this.client = this.clientService.client
+    }
+
+
+    async initialize(force = false): Promise<any> {
+        const masterScreenerExists = await this.client.indices.exists({ index: 'master_screener'});
+        const questionsExists = await this.client.indices.exists({ index: 'questions'});
+        const programsExists = await this.client.indices.exists({ index: 'programs'});
+
+        const hasBeenInitialized = masterScreenerExists || questionsExists || programsExists;
+
+        if (hasBeenInitialized && !force) {
+            throw new Error("Database has already been initialized.");
+        }
+
+        if (masterScreenerExists) {
+            await this.client.indices.delete({ index: 'master_screener'});
+        }
+
+        if (questionsExists) {
+            await this.client.indices.delete({ index: 'questions'});
+        }
+
+        if (programsExists) {
+            await this.client.indices.delete({ index: 'programs'});
+        }
+
+        await this.client.indices.create({ index: 'master_screener'});
+        const masterScreenerPutMapping = await this.client.indices.putMapping({
+            index: 'master_screener',
+            type: 'queries',
+            body: { properties: { ...PERCOLATOR_MAPPING } }
+        });
+
+        await this.client.indices.create({ index: 'questions'});
+        const questionScreenerMapping = await this.client.indices.putMapping({
+            index: 'questions',
+            type: 'screener',
+            body: { properties: { ...SCREENER_MAPPING } }
+        });
+
+        await this.client.indices.create({ index: 'programs'});
+        const programsMapping = await this.client.indices.putMapping({
+            index: 'programs',
+            type: 'user_facing',
+            body: { properties: { ...PROGRAM_MAPPING } }
+        });
+
+        return [
+            [ masterScreenerExists, masterScreenerPutMapping],
+            [ questionsExists, questionScreenerMapping ],
+            [ programsExists, programsMapping ]
+        ]
+    }
+}
+
+const PERCOLATOR_MAPPING = {
+    "query": { "type": "percolator" },
+    "meta":{
+        "properties":{
+            "id":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},
+            "program_guid":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}}
+        }
+    }
+};
+
+const PROGRAM_MAPPING = {
+    "created":{"type":"date"},
+    "description":{"type":"text", "fields":{"keyword":{"type":"keyword","ignore_above":256}}},
+    "detailLinks":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},
+    "details":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},
+    "externalLink":{"type":"text"},
+    "guid":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},
+    "tags":{"type":"keyword"},
+    "title":{"type":"text"}
+};
+
+const SCREENER_MAPPING = {
+    "conditionalQuestions":{
+        "properties":{
+            "controlType":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},
+            "expandable":{"type":"boolean"},
+            "id":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},
+            "index":{"type":"long"},
+            "key":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},
+            "label":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},
+            "multiSelectOptions":{
+                "properties":{
+                    "key":{
+                        "properties":{
+                            "name":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},
+                            "type":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}}
+                        }
+                    },
+                    "text":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}}}},
+            "options":{"type":"long"}}},
+    "created":{"type":"long"},
+    "questions":{
+        "properties":{
+            "conditionalQuestions":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},
+            "controlType":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},
+            "expandable":{"type":"boolean"},
+            "id":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},
+            "index":{"type":"long"},
+            "key":{"type":"text", "fields":{"keyword":{"type":"keyword","ignore_above":256}}},
+            "label":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},
+            "multiSelectOptions":{
+                "properties":{
+                    "key":{
+                        "properties":{
+                            "name":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},
+                            "type":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}}}
+                    },
+                    "text":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}}
+                }
+            }
+        }
+    }
+};