From bfc3071d1459a9b860442cb1431027e4fa34cd0c Mon Sep 17 00:00:00 2001 From: jannisvisser Date: Wed, 29 Nov 2023 16:46:55 +0100 Subject: [PATCH] feat: dynamic point data entity + gauge layer AB#25124 --- .../IBF-dashboard/src/app/types/ibf-layer.ts | 1 + .../1701157179286-DynamicPointData.ts | 47 +++++++++++ .../dynamic-point-data_water-level.json | 14 ++++ .../dto/upload-asset-exposure-status.dto.ts | 36 +++++++++ .../api/point-data/dto/upload-gauge.dto.ts | 20 +++++ .../point-data/dynamic-point-data.entity.ts | 34 ++++++++ .../api/point-data/point-data.controller.ts | 18 ++++- .../src/api/point-data/point-data.entity.ts | 10 ++- .../src/api/point-data/point-data.module.ts | 7 +- .../src/api/point-data/point-data.service.ts | 77 +++++++++++++++++-- .../standard-point-layers/gauges_MWI.csv | 4 + .../src/scripts/json/layer-metadata.json | 8 ++ .../src/scripts/scripts.service.ts | 34 +++++++- .../src/scripts/seed-point-data.ts | 1 + 14 files changed, 300 insertions(+), 11 deletions(-) create mode 100644 services/API-service/migration/1701157179286-DynamicPointData.ts create mode 100644 services/API-service/src/api/point-data/dto/example/MWI/flash-floods/dynamic-point-data_water-level.json create mode 100644 services/API-service/src/api/point-data/dto/upload-gauge.dto.ts create mode 100644 services/API-service/src/api/point-data/dynamic-point-data.entity.ts create mode 100644 services/API-service/src/scripts/git-lfs/standard-point-layers/gauges_MWI.csv diff --git a/interfaces/IBF-dashboard/src/app/types/ibf-layer.ts b/interfaces/IBF-dashboard/src/app/types/ibf-layer.ts index d71d9581e..55b63d079 100644 --- a/interfaces/IBF-dashboard/src/app/types/ibf-layer.ts +++ b/interfaces/IBF-dashboard/src/app/types/ibf-layer.ts @@ -123,6 +123,7 @@ export enum IbfLayerName { schools = 'schools', waterpointsInternal = 'waterpoints_internal', roads = 'roads', + gauges = 'gauges', } export enum IbfLayerLabel { diff --git a/services/API-service/migration/1701157179286-DynamicPointData.ts b/services/API-service/migration/1701157179286-DynamicPointData.ts new file mode 100644 index 000000000..1df918513 --- /dev/null +++ b/services/API-service/migration/1701157179286-DynamicPointData.ts @@ -0,0 +1,47 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class DynamicPointData1701157179286 implements MigrationInterface { + name = 'DynamicPointData1701157179286'; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `CREATE TABLE "IBF-app"."dynamic-point-data" ("dynamicPointDataId" uuid NOT NULL DEFAULT uuid_generate_v4(), "timestamp" TIMESTAMP NOT NULL, "key" character varying NOT NULL, "value" character varying, "pointPointDataId" uuid, "leadTime" character varying, CONSTRAINT "PK_4f51e8c4a1091afa35e3cc51b34" PRIMARY KEY ("dynamicPointDataId"))`, + ); + await queryRunner.query( + `ALTER TABLE "IBF-app"."dynamic-point-data" ADD CONSTRAINT "FK_9aaa9be5dbe82b610209bcb456b" FOREIGN KEY ("pointPointDataId") REFERENCES "IBF-app"."point-data"("pointDataId") ON DELETE NO ACTION ON UPDATE NO ACTION`, + ); + await queryRunner.query( + `ALTER TABLE "IBF-app"."dynamic-point-data" ADD CONSTRAINT "FK_289a1f52e25e270d9a28bd9d35a" FOREIGN KEY ("leadTime") REFERENCES "IBF-app"."lead-time"("leadTimeName") ON DELETE NO ACTION ON UPDATE NO ACTION`, + ); + + // NOTE: Do not prioritize this, but leave the code here for future reference + // await queryRunner.query(`INSERT INTO "IBF-app"."point-data" + // ("pointDataId", "countryCodeISO3", "pointDataCategory", "attributes", geom, "referenceId") + // select id + // ,"countryCodeISO3" + // ,'glofas_stations' + // ,(cast('{"stationName":"' as varchar) || "stationName" || cast('","stationCode":"' || "stationCode" || '"}' as varchar))::json + // ,geom + // ,null + // from "IBF-app"."glofas-station" `); + + // await queryRunner.query(`INSERT INTO "IBF-app"."dynamic-point-data" + // ("dynamicPointDataId", "timestamp", "key", value, "pointPointDataId") + // select uuid_generate_v4() + // ,date + // ,unnest(array['forecastLevel','forecastReturnPeriod','triggerLevel','eapAlertClass']) as key + // ,unnest(array[cast("forecastLevel" as varchar),cast("forecastReturnPeriod" as varchar),cast("triggerLevel" as varchar),"eapAlertClass"]) as value + // ,"glofasStationId" + // from "IBF-app"."glofas-station-forecast" gsf `); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `ALTER TABLE "IBF-app"."dynamic-point-data" DROP CONSTRAINT "FK_289a1f52e25e270d9a28bd9d35a"`, + ); + await queryRunner.query( + `ALTER TABLE "IBF-app"."dynamic-point-data" DROP CONSTRAINT "FK_9aaa9be5dbe82b610209bcb456b"`, + ); + await queryRunner.query(`DROP TABLE "IBF-app"."dynamic-point-data"`); + } +} diff --git a/services/API-service/src/api/point-data/dto/example/MWI/flash-floods/dynamic-point-data_water-level.json b/services/API-service/src/api/point-data/dto/example/MWI/flash-floods/dynamic-point-data_water-level.json new file mode 100644 index 000000000..142ab1010 --- /dev/null +++ b/services/API-service/src/api/point-data/dto/example/MWI/flash-floods/dynamic-point-data_water-level.json @@ -0,0 +1,14 @@ +[ + { + "fid": 1, + "value": 100 + }, + { + "fid": 2, + "value": 100 + }, + { + "fid": 3, + "value": 100 + } +] diff --git a/services/API-service/src/api/point-data/dto/upload-asset-exposure-status.dto.ts b/services/API-service/src/api/point-data/dto/upload-asset-exposure-status.dto.ts index 550fe0b3a..265a68d57 100644 --- a/services/API-service/src/api/point-data/dto/upload-asset-exposure-status.dto.ts +++ b/services/API-service/src/api/point-data/dto/upload-asset-exposure-status.dto.ts @@ -4,11 +4,13 @@ import { IsNotEmpty, IsOptional, IsString, + ValidateNested, } from 'class-validator'; import { ApiProperty } from '@nestjs/swagger'; import { LeadTime } from '../../admin-area-dynamic-data/enum/lead-time.enum'; import { DisasterType } from '../../disaster/disaster-type.enum'; import { PointDataEnum } from '../point-data.entity'; +import { Type } from 'class-transformer'; export class UploadAssetExposureStatusDto { @ApiProperty({ example: ['123', '234'] }) @@ -39,3 +41,37 @@ export class UploadAssetExposureStatusDto { @IsEnum(PointDataEnum) public pointDataCategory: PointDataEnum; } + +export class UploadDynamicPointDataDto { + @ApiProperty({ example: LeadTime.hour1 }) + @IsOptional() + @IsString() + public leadTime: LeadTime; + + @ApiProperty({ example: new Date() }) + @IsOptional() + public date: Date; + + @ApiProperty({ example: DisasterType.FlashFloods }) + @IsNotEmpty() + @IsEnum(DisasterType) + public disasterType: DisasterType; + + @ApiProperty({ example: 'waterLevel' }) + public key: string; + + @ApiProperty({ example: [{ fid: 1, value: 100 }] }) + @IsArray() + @ValidateNested() + @Type(() => DynamicPointData) + public dynamicPointData: DynamicPointData[]; +} + +export class DynamicPointData { + @ApiProperty() + @IsNotEmpty() + public fid: string; + + @ApiProperty() + public value: string; +} diff --git a/services/API-service/src/api/point-data/dto/upload-gauge.dto.ts b/services/API-service/src/api/point-data/dto/upload-gauge.dto.ts new file mode 100644 index 000000000..b3df92ffa --- /dev/null +++ b/services/API-service/src/api/point-data/dto/upload-gauge.dto.ts @@ -0,0 +1,20 @@ +import { IsNotEmpty, IsOptional, IsString } from 'class-validator'; +import { ApiProperty } from '@nestjs/swagger'; + +export class GaugeDto { + @ApiProperty({ example: 'name' }) + @IsString() + public name: string = undefined; + + @ApiProperty({ example: 1234 }) + @IsOptional() + public fid: number = undefined; + + @ApiProperty({ example: 0 }) + @IsNotEmpty() + public lat: number; + + @ApiProperty({ example: 0 }) + @IsNotEmpty() + public lon: number; +} diff --git a/services/API-service/src/api/point-data/dynamic-point-data.entity.ts b/services/API-service/src/api/point-data/dynamic-point-data.entity.ts new file mode 100644 index 000000000..0d5ad54d0 --- /dev/null +++ b/services/API-service/src/api/point-data/dynamic-point-data.entity.ts @@ -0,0 +1,34 @@ +import { + Entity, + PrimaryGeneratedColumn, + Column, + ManyToOne, + JoinColumn, +} from 'typeorm'; +import { PointDataEntity } from './point-data.entity'; +import { LeadTimeEntity } from '../lead-time/lead-time.entity'; + +@Entity('dynamic-point-data') +export class DynamicPointDataEntity { + @PrimaryGeneratedColumn('uuid') + public dynamicPointDataId: string; + + @ManyToOne( + (): typeof PointDataEntity => PointDataEntity, + (point): DynamicPointDataEntity[] => point.dynamicData, + ) + public point: PointDataEntity; + + @ManyToOne((): typeof LeadTimeEntity => LeadTimeEntity) + @JoinColumn({ name: 'leadTime', referencedColumnName: 'leadTimeName' }) + public leadTime: string; + + @Column({ type: 'timestamp' }) + public timestamp: Date; + + @Column() + public key: string; + + @Column({ nullable: true }) + public value: string; +} diff --git a/services/API-service/src/api/point-data/point-data.controller.ts b/services/API-service/src/api/point-data/point-data.controller.ts index cdd706dfe..aa7e0b42b 100644 --- a/services/API-service/src/api/point-data/point-data.controller.ts +++ b/services/API-service/src/api/point-data/point-data.controller.ts @@ -26,7 +26,10 @@ import { RolesGuard } from '../../roles.guard'; import { GeoJson } from '../../shared/geo.model'; import { UserRole } from '../user/user-role.enum'; import { PointDataService } from './point-data.service'; -import { UploadAssetExposureStatusDto } from './dto/upload-asset-exposure-status.dto'; +import { + UploadAssetExposureStatusDto, + UploadDynamicPointDataDto, +} from './dto/upload-asset-exposure-status.dto'; import { FILE_UPLOAD_API_FORMAT } from '../../shared/file-upload-api-format'; @ApiBearerAuth() @@ -132,4 +135,17 @@ export class PointDataController { ): Promise { return await this.pointDataService.uploadAssetExposureStatus(assetFids); } + + @UseGuards(RolesGuard) + @ApiOperation({ summary: 'Upload dynamic point data' }) + @ApiResponse({ + status: 201, + description: 'Uploaded dynamic point data.', + }) + @Post('dynamic') + public async uploadDynamicPointData( + @Body() dynamicPointData: UploadDynamicPointDataDto, + ): Promise { + return await this.pointDataService.uploadDynamicPointData(dynamicPointData); + } } diff --git a/services/API-service/src/api/point-data/point-data.entity.ts b/services/API-service/src/api/point-data/point-data.entity.ts index 498e2a1b1..98f3801fe 100644 --- a/services/API-service/src/api/point-data/point-data.entity.ts +++ b/services/API-service/src/api/point-data/point-data.entity.ts @@ -1,4 +1,5 @@ -import { Entity, PrimaryGeneratedColumn, Column } from 'typeorm'; +import { Entity, PrimaryGeneratedColumn, Column, OneToMany } from 'typeorm'; +import { DynamicPointDataEntity } from './dynamic-point-data.entity'; export enum PointDataEnum { evacuationCenters = 'evacuation_centers', @@ -8,6 +9,7 @@ export enum PointDataEnum { communityNotifications = 'community_notifications', schools = 'schools', waterpointsInternal = 'waterpoints_internal', + gauges = 'gauges', } @Entity('point-data') @@ -29,4 +31,10 @@ export class PointDataEntity { @Column('json', { nullable: true }) public geom: JSON; + + @OneToMany( + (): typeof DynamicPointDataEntity => DynamicPointDataEntity, + (dynamicData): PointDataEntity => dynamicData.point, + ) + public dynamicData: DynamicPointDataEntity[]; } diff --git a/services/API-service/src/api/point-data/point-data.module.ts b/services/API-service/src/api/point-data/point-data.module.ts index 8a4b86696..798d4b9f5 100644 --- a/services/API-service/src/api/point-data/point-data.module.ts +++ b/services/API-service/src/api/point-data/point-data.module.ts @@ -8,12 +8,17 @@ import { PointDataEntity } from './point-data.entity'; import { PointDataService } from './point-data.service'; import { PointDataDynamicStatusEntity } from './point-data-dynamic-status.entity'; import { HttpModule } from '@nestjs/axios'; +import { DynamicPointDataEntity } from './dynamic-point-data.entity'; @Module({ imports: [ HttpModule, UserModule, - TypeOrmModule.forFeature([PointDataEntity, PointDataDynamicStatusEntity]), + TypeOrmModule.forFeature([ + PointDataEntity, + PointDataDynamicStatusEntity, + DynamicPointDataEntity, + ]), WhatsappModule, ], providers: [PointDataService, HelperService], diff --git a/services/API-service/src/api/point-data/point-data.service.ts b/services/API-service/src/api/point-data/point-data.service.ts index 9f41695bb..5c1d5c6bb 100644 --- a/services/API-service/src/api/point-data/point-data.service.ts +++ b/services/API-service/src/api/point-data/point-data.service.ts @@ -3,7 +3,7 @@ import { InjectRepository } from '@nestjs/typeorm'; import { validate } from 'class-validator'; import { GeoJson } from '../../shared/geo.model'; import { HelperService } from '../../shared/helper.service'; -import { MoreThanOrEqual, Repository } from 'typeorm'; +import { IsNull, MoreThanOrEqual, Repository } from 'typeorm'; import { EvacuationCenterDto } from './dto/upload-evacuation-centers.dto'; import { PointDataEntity, PointDataEnum } from './point-data.entity'; import { DamSiteDto } from './dto/upload-dam-sites.dto'; @@ -13,9 +13,14 @@ import { CommunityNotificationDto } from './dto/upload-community-notifications.d import { WhatsappService } from '../notification/whatsapp/whatsapp.service'; import { SchoolDto } from './dto/upload-schools.dto'; import { WaterpointDto } from './dto/upload-waterpoint.dto'; -import { UploadAssetExposureStatusDto } from './dto/upload-asset-exposure-status.dto'; +import { + UploadAssetExposureStatusDto, + UploadDynamicPointDataDto, +} from './dto/upload-asset-exposure-status.dto'; import { PointDataDynamicStatusEntity } from './point-data-dynamic-status.entity'; import { DisasterType } from '../disaster/disaster-type.enum'; +import { GaugeDto } from './dto/upload-gauge.dto'; +import { DynamicPointDataEntity } from './dynamic-point-data.entity'; @Injectable() export class PointDataService { @@ -23,6 +28,8 @@ export class PointDataService { private readonly pointDataRepository: Repository; @InjectRepository(PointDataDynamicStatusEntity) private readonly pointDataDynamicStatusRepo: Repository; + @InjectRepository(DynamicPointDataEntity) + private readonly dynamicPointDataRepository: Repository; public constructor( private readonly helperService: HelperService, @@ -47,20 +54,38 @@ export class PointDataService { selectColumns.push('geom'); selectColumns.push('"pointDataId"'); + const recentDate = await this.helperService.getRecentDate( + countryCodeISO3, + disasterType, + ); + const pointDataQuery = this.pointDataRepository .createQueryBuilder('point') .select(selectColumns) .where({ pointDataCategory: pointDataCategory, countryCodeISO3: countryCodeISO3, - }); + }) + .leftJoin( + (subquery) => { + return subquery + .select([ + 'dynamic."pointPointDataId"', + 'json_object_agg("key",value) as "dynamicData"', + ]) + .from(DynamicPointDataEntity, 'dynamic') + .where('dynamic.timestamp = :modelTimestamp', { + modelTimestamp: recentDate.timestamp, + }) + .groupBy('dynamic."pointPointDataId"'); + }, + 'dynamic', + 'dynamic."pointPointDataId" = point."pointDataId"', + ) + .addSelect('dynamic."dynamicData" as "dynamicData"'); // TO DO: hard-code for now if (disasterType === DisasterType.FlashFloods) { - const recentDate = await this.helperService.getRecentDate( - countryCodeISO3, - disasterType, - ); pointDataQuery .leftJoin( PointDataDynamicStatusEntity, @@ -95,6 +120,8 @@ export class PointDataService { return new SchoolDto(); case PointDataEnum.waterpointsInternal: return new WaterpointDto(); + case PointDataEnum.gauges: + return new GaugeDto(); default: throw new HttpException( 'Not a known point layer', @@ -259,4 +286,40 @@ export class PointDataService { } await this.pointDataDynamicStatusRepo.save(assetForecasts); } + + async uploadDynamicPointData(dynamicPointData: UploadDynamicPointDataDto) { + const dynamicPointDataArray: DynamicPointDataEntity[] = []; + + for (const point of dynamicPointData.dynamicPointData) { + const asset = await this.pointDataRepository.findOne({ + where: { + referenceId: Number(point.fid), // TODO: make sure this is unique + }, + }); + if (!asset) { + continue; + } + + // Delete existing entries + await this.dynamicPointDataRepository.delete({ + point: { pointDataId: asset.pointDataId }, + leadTime: dynamicPointData.leadTime || IsNull(), + timestamp: MoreThanOrEqual( + this.helperService.getUploadCutoffMoment( + dynamicPointData.disasterType, + dynamicPointData.date || new Date(), + ), + ), + }); + + const dynamicPoint = new DynamicPointDataEntity(); + dynamicPoint.key = dynamicPointData.key; + dynamicPoint.leadTime = dynamicPointData.leadTime; + dynamicPoint.timestamp = dynamicPointData.date || new Date(); + dynamicPoint.value = point.value; + dynamicPoint.point = asset; + dynamicPointDataArray.push(dynamicPoint); + } + await this.dynamicPointDataRepository.save(dynamicPointDataArray); + } } diff --git a/services/API-service/src/scripts/git-lfs/standard-point-layers/gauges_MWI.csv b/services/API-service/src/scripts/git-lfs/standard-point-layers/gauges_MWI.csv new file mode 100644 index 000000000..7b65a6109 --- /dev/null +++ b/services/API-service/src/scripts/git-lfs/standard-point-layers/gauges_MWI.csv @@ -0,0 +1,4 @@ +id,fid,name,Country,lat,lon +1,1,Kariba,MWI,-9.940606036,33.92438203 +2,2,Tugwi Mukosi,MWI,-9.938750283,33.92635764 +3,3,Mutirikwi,MWI,-11.89861174,33.59389079 \ No newline at end of file diff --git a/services/API-service/src/scripts/json/layer-metadata.json b/services/API-service/src/scripts/json/layer-metadata.json index 8ec4b6783..12ad17469 100644 --- a/services/API-service/src/scripts/json/layer-metadata.json +++ b/services/API-service/src/scripts/json/layer-metadata.json @@ -469,6 +469,14 @@ "active": "no", "description": { "MWI": { "flash-floods": "TBD" } } }, + { + "name": "gauges", + "label": "River gauges", + "type": "point", + "leadTimeDependent": false, + "active": "no", + "description": { "MWI": { "flash-floods": "TBD" } } + }, { "name": "roads", "label": "Roads", diff --git a/services/API-service/src/scripts/scripts.service.ts b/services/API-service/src/scripts/scripts.service.ts index 8c0fc56f9..3c7437444 100644 --- a/services/API-service/src/scripts/scripts.service.ts +++ b/services/API-service/src/scripts/scripts.service.ts @@ -28,7 +28,10 @@ import { LinesDataService } from '../api/lines-data/lines-data.service'; import { UploadLinesExposureStatusDto } from '../api/lines-data/dto/upload-asset-exposure-status.dto'; import { LinesDataEnum } from '../api/lines-data/lines-data.entity'; import { PointDataEnum } from '../api/point-data/point-data.entity'; -import { UploadAssetExposureStatusDto } from '../api/point-data/dto/upload-asset-exposure-status.dto'; +import { + UploadAssetExposureStatusDto, + UploadDynamicPointDataDto, +} from '../api/point-data/dto/upload-asset-exposure-status.dto'; import { PointDataService } from '../api/point-data/point-data.service'; @Injectable() @@ -192,6 +195,11 @@ export class ScriptsService { mockInput.triggered, date, ); + await this.mockDynamicPointData( + selectedCountry.countryCodeISO3, + mockInput.disasterType, + date, + ); } if (mockInput.disasterType === DisasterType.Typhoon) { @@ -1018,6 +1026,30 @@ export class ScriptsService { } } + private async mockDynamicPointData( + countryCodeISO3: string, + disasterType: DisasterType, + date: Date, + ) { + if (countryCodeISO3 !== 'MWI') { + return; + } + + const keys = ['water-level']; + for (const key of keys) { + const payload = new UploadDynamicPointDataDto(); + payload.key = key; + payload.leadTime = null; + payload.date = date || new Date(); + payload.disasterType = disasterType; + const filename = `./src/api/point-data/dto/example/${countryCodeISO3}/${DisasterType.FlashFloods}/dynamic-point-data_${key}.json`; + const dynamicPointData = JSON.parse(fs.readFileSync(filename, 'utf-8')); + payload.dynamicPointData = dynamicPointData; + + await this.pointDataService.uploadDynamicPointData(payload); + } + } + private async mockRasterFile( selectedCountry, disasterType: DisasterType, diff --git a/services/API-service/src/scripts/seed-point-data.ts b/services/API-service/src/scripts/seed-point-data.ts index 236efe678..f4350b44c 100644 --- a/services/API-service/src/scripts/seed-point-data.ts +++ b/services/API-service/src/scripts/seed-point-data.ts @@ -29,6 +29,7 @@ export class SeedPointData implements InterfaceScript { this.seedPointData(PointDataEnum.dams, country); this.seedPointData(PointDataEnum.schools, country); this.seedPointData(PointDataEnum.waterpointsInternal, country); + this.seedPointData(PointDataEnum.gauges, country); return; } else { return Promise.resolve();