From 5190cb6af5b8de254700c935b5410537db4eca4c Mon Sep 17 00:00:00 2001 From: jannisvisser Date: Wed, 29 Nov 2023 16:47:59 +0100 Subject: [PATCH] Revert "feat: dynamic point data entity + gauge layer AB#25124" --- .../IBF-dashboard/src/app/types/ibf-layer.ts | 1 - .../1701157179286-DynamicPointData.ts | 47 ----------- .../dynamic-point-data_water-level.json | 14 ---- .../dto/upload-asset-exposure-status.dto.ts | 36 --------- .../api/point-data/dto/upload-gauge.dto.ts | 20 ----- .../point-data/dynamic-point-data.entity.ts | 34 -------- .../api/point-data/point-data.controller.ts | 18 +---- .../src/api/point-data/point-data.entity.ts | 10 +-- .../src/api/point-data/point-data.module.ts | 7 +- .../src/api/point-data/point-data.service.ts | 77 ++----------------- .../standard-point-layers/gauges_MWI.csv | 4 - .../src/scripts/json/layer-metadata.json | 8 -- .../src/scripts/scripts.service.ts | 34 +------- .../src/scripts/seed-point-data.ts | 1 - 14 files changed, 11 insertions(+), 300 deletions(-) delete mode 100644 services/API-service/migration/1701157179286-DynamicPointData.ts delete mode 100644 services/API-service/src/api/point-data/dto/example/MWI/flash-floods/dynamic-point-data_water-level.json delete mode 100644 services/API-service/src/api/point-data/dto/upload-gauge.dto.ts delete mode 100644 services/API-service/src/api/point-data/dynamic-point-data.entity.ts delete mode 100644 services/API-service/src/scripts/git-lfs/standard-point-layers/gauges_MWI.csv diff --git a/interfaces/IBF-dashboard/src/app/types/ibf-layer.ts b/interfaces/IBF-dashboard/src/app/types/ibf-layer.ts index 55b63d079..d71d9581e 100644 --- a/interfaces/IBF-dashboard/src/app/types/ibf-layer.ts +++ b/interfaces/IBF-dashboard/src/app/types/ibf-layer.ts @@ -123,7 +123,6 @@ export enum IbfLayerName { schools = 'schools', waterpointsInternal = 'waterpoints_internal', roads = 'roads', - gauges = 'gauges', } export enum IbfLayerLabel { diff --git a/services/API-service/migration/1701157179286-DynamicPointData.ts b/services/API-service/migration/1701157179286-DynamicPointData.ts deleted file mode 100644 index 1df918513..000000000 --- a/services/API-service/migration/1701157179286-DynamicPointData.ts +++ /dev/null @@ -1,47 +0,0 @@ -import { MigrationInterface, QueryRunner } from 'typeorm'; - -export class DynamicPointData1701157179286 implements MigrationInterface { - name = 'DynamicPointData1701157179286'; - - public async up(queryRunner: QueryRunner): Promise { - await queryRunner.query( - `CREATE TABLE "IBF-app"."dynamic-point-data" ("dynamicPointDataId" uuid NOT NULL DEFAULT uuid_generate_v4(), "timestamp" TIMESTAMP NOT NULL, "key" character varying NOT NULL, "value" character varying, "pointPointDataId" uuid, "leadTime" character varying, CONSTRAINT "PK_4f51e8c4a1091afa35e3cc51b34" PRIMARY KEY ("dynamicPointDataId"))`, - ); - await queryRunner.query( - `ALTER TABLE "IBF-app"."dynamic-point-data" ADD CONSTRAINT "FK_9aaa9be5dbe82b610209bcb456b" FOREIGN KEY ("pointPointDataId") REFERENCES "IBF-app"."point-data"("pointDataId") ON DELETE NO ACTION ON UPDATE NO ACTION`, - ); - await queryRunner.query( - `ALTER TABLE "IBF-app"."dynamic-point-data" ADD CONSTRAINT "FK_289a1f52e25e270d9a28bd9d35a" FOREIGN KEY ("leadTime") REFERENCES "IBF-app"."lead-time"("leadTimeName") ON DELETE NO ACTION ON UPDATE NO ACTION`, - ); - - // NOTE: Do not prioritize this, but leave the code here for future reference - // await queryRunner.query(`INSERT INTO "IBF-app"."point-data" - // ("pointDataId", "countryCodeISO3", "pointDataCategory", "attributes", geom, "referenceId") - // select id - // ,"countryCodeISO3" - // ,'glofas_stations' - // ,(cast('{"stationName":"' as varchar) || "stationName" || cast('","stationCode":"' || "stationCode" || '"}' as varchar))::json - // ,geom - // ,null - // from "IBF-app"."glofas-station" `); - - // await queryRunner.query(`INSERT INTO "IBF-app"."dynamic-point-data" - // ("dynamicPointDataId", "timestamp", "key", value, "pointPointDataId") - // select uuid_generate_v4() - // ,date - // ,unnest(array['forecastLevel','forecastReturnPeriod','triggerLevel','eapAlertClass']) as key - // ,unnest(array[cast("forecastLevel" as varchar),cast("forecastReturnPeriod" as varchar),cast("triggerLevel" as varchar),"eapAlertClass"]) as value - // ,"glofasStationId" - // from "IBF-app"."glofas-station-forecast" gsf `); - } - - public async down(queryRunner: QueryRunner): Promise { - await queryRunner.query( - `ALTER TABLE "IBF-app"."dynamic-point-data" DROP CONSTRAINT "FK_289a1f52e25e270d9a28bd9d35a"`, - ); - await queryRunner.query( - `ALTER TABLE "IBF-app"."dynamic-point-data" DROP CONSTRAINT "FK_9aaa9be5dbe82b610209bcb456b"`, - ); - await queryRunner.query(`DROP TABLE "IBF-app"."dynamic-point-data"`); - } -} diff --git a/services/API-service/src/api/point-data/dto/example/MWI/flash-floods/dynamic-point-data_water-level.json b/services/API-service/src/api/point-data/dto/example/MWI/flash-floods/dynamic-point-data_water-level.json deleted file mode 100644 index 142ab1010..000000000 --- a/services/API-service/src/api/point-data/dto/example/MWI/flash-floods/dynamic-point-data_water-level.json +++ /dev/null @@ -1,14 +0,0 @@ -[ - { - "fid": 1, - "value": 100 - }, - { - "fid": 2, - "value": 100 - }, - { - "fid": 3, - "value": 100 - } -] diff --git a/services/API-service/src/api/point-data/dto/upload-asset-exposure-status.dto.ts b/services/API-service/src/api/point-data/dto/upload-asset-exposure-status.dto.ts index 265a68d57..550fe0b3a 100644 --- a/services/API-service/src/api/point-data/dto/upload-asset-exposure-status.dto.ts +++ b/services/API-service/src/api/point-data/dto/upload-asset-exposure-status.dto.ts @@ -4,13 +4,11 @@ import { IsNotEmpty, IsOptional, IsString, - ValidateNested, } from 'class-validator'; import { ApiProperty } from '@nestjs/swagger'; import { LeadTime } from '../../admin-area-dynamic-data/enum/lead-time.enum'; import { DisasterType } from '../../disaster/disaster-type.enum'; import { PointDataEnum } from '../point-data.entity'; -import { Type } from 'class-transformer'; export class UploadAssetExposureStatusDto { @ApiProperty({ example: ['123', '234'] }) @@ -41,37 +39,3 @@ export class UploadAssetExposureStatusDto { @IsEnum(PointDataEnum) public pointDataCategory: PointDataEnum; } - -export class UploadDynamicPointDataDto { - @ApiProperty({ example: LeadTime.hour1 }) - @IsOptional() - @IsString() - public leadTime: LeadTime; - - @ApiProperty({ example: new Date() }) - @IsOptional() - public date: Date; - - @ApiProperty({ example: DisasterType.FlashFloods }) - @IsNotEmpty() - @IsEnum(DisasterType) - public disasterType: DisasterType; - - @ApiProperty({ example: 'waterLevel' }) - public key: string; - - @ApiProperty({ example: [{ fid: 1, value: 100 }] }) - @IsArray() - @ValidateNested() - @Type(() => DynamicPointData) - public dynamicPointData: DynamicPointData[]; -} - -export class DynamicPointData { - @ApiProperty() - @IsNotEmpty() - public fid: string; - - @ApiProperty() - public value: string; -} diff --git a/services/API-service/src/api/point-data/dto/upload-gauge.dto.ts b/services/API-service/src/api/point-data/dto/upload-gauge.dto.ts deleted file mode 100644 index b3df92ffa..000000000 --- a/services/API-service/src/api/point-data/dto/upload-gauge.dto.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { IsNotEmpty, IsOptional, IsString } from 'class-validator'; -import { ApiProperty } from '@nestjs/swagger'; - -export class GaugeDto { - @ApiProperty({ example: 'name' }) - @IsString() - public name: string = undefined; - - @ApiProperty({ example: 1234 }) - @IsOptional() - public fid: number = undefined; - - @ApiProperty({ example: 0 }) - @IsNotEmpty() - public lat: number; - - @ApiProperty({ example: 0 }) - @IsNotEmpty() - public lon: number; -} diff --git a/services/API-service/src/api/point-data/dynamic-point-data.entity.ts b/services/API-service/src/api/point-data/dynamic-point-data.entity.ts deleted file mode 100644 index 0d5ad54d0..000000000 --- a/services/API-service/src/api/point-data/dynamic-point-data.entity.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { - Entity, - PrimaryGeneratedColumn, - Column, - ManyToOne, - JoinColumn, -} from 'typeorm'; -import { PointDataEntity } from './point-data.entity'; -import { LeadTimeEntity } from '../lead-time/lead-time.entity'; - -@Entity('dynamic-point-data') -export class DynamicPointDataEntity { - @PrimaryGeneratedColumn('uuid') - public dynamicPointDataId: string; - - @ManyToOne( - (): typeof PointDataEntity => PointDataEntity, - (point): DynamicPointDataEntity[] => point.dynamicData, - ) - public point: PointDataEntity; - - @ManyToOne((): typeof LeadTimeEntity => LeadTimeEntity) - @JoinColumn({ name: 'leadTime', referencedColumnName: 'leadTimeName' }) - public leadTime: string; - - @Column({ type: 'timestamp' }) - public timestamp: Date; - - @Column() - public key: string; - - @Column({ nullable: true }) - public value: string; -} diff --git a/services/API-service/src/api/point-data/point-data.controller.ts b/services/API-service/src/api/point-data/point-data.controller.ts index aa7e0b42b..cdd706dfe 100644 --- a/services/API-service/src/api/point-data/point-data.controller.ts +++ b/services/API-service/src/api/point-data/point-data.controller.ts @@ -26,10 +26,7 @@ import { RolesGuard } from '../../roles.guard'; import { GeoJson } from '../../shared/geo.model'; import { UserRole } from '../user/user-role.enum'; import { PointDataService } from './point-data.service'; -import { - UploadAssetExposureStatusDto, - UploadDynamicPointDataDto, -} from './dto/upload-asset-exposure-status.dto'; +import { UploadAssetExposureStatusDto } from './dto/upload-asset-exposure-status.dto'; import { FILE_UPLOAD_API_FORMAT } from '../../shared/file-upload-api-format'; @ApiBearerAuth() @@ -135,17 +132,4 @@ export class PointDataController { ): Promise { return await this.pointDataService.uploadAssetExposureStatus(assetFids); } - - @UseGuards(RolesGuard) - @ApiOperation({ summary: 'Upload dynamic point data' }) - @ApiResponse({ - status: 201, - description: 'Uploaded dynamic point data.', - }) - @Post('dynamic') - public async uploadDynamicPointData( - @Body() dynamicPointData: UploadDynamicPointDataDto, - ): Promise { - return await this.pointDataService.uploadDynamicPointData(dynamicPointData); - } } diff --git a/services/API-service/src/api/point-data/point-data.entity.ts b/services/API-service/src/api/point-data/point-data.entity.ts index 98f3801fe..498e2a1b1 100644 --- a/services/API-service/src/api/point-data/point-data.entity.ts +++ b/services/API-service/src/api/point-data/point-data.entity.ts @@ -1,5 +1,4 @@ -import { Entity, PrimaryGeneratedColumn, Column, OneToMany } from 'typeorm'; -import { DynamicPointDataEntity } from './dynamic-point-data.entity'; +import { Entity, PrimaryGeneratedColumn, Column } from 'typeorm'; export enum PointDataEnum { evacuationCenters = 'evacuation_centers', @@ -9,7 +8,6 @@ export enum PointDataEnum { communityNotifications = 'community_notifications', schools = 'schools', waterpointsInternal = 'waterpoints_internal', - gauges = 'gauges', } @Entity('point-data') @@ -31,10 +29,4 @@ export class PointDataEntity { @Column('json', { nullable: true }) public geom: JSON; - - @OneToMany( - (): typeof DynamicPointDataEntity => DynamicPointDataEntity, - (dynamicData): PointDataEntity => dynamicData.point, - ) - public dynamicData: DynamicPointDataEntity[]; } diff --git a/services/API-service/src/api/point-data/point-data.module.ts b/services/API-service/src/api/point-data/point-data.module.ts index 798d4b9f5..8a4b86696 100644 --- a/services/API-service/src/api/point-data/point-data.module.ts +++ b/services/API-service/src/api/point-data/point-data.module.ts @@ -8,17 +8,12 @@ import { PointDataEntity } from './point-data.entity'; import { PointDataService } from './point-data.service'; import { PointDataDynamicStatusEntity } from './point-data-dynamic-status.entity'; import { HttpModule } from '@nestjs/axios'; -import { DynamicPointDataEntity } from './dynamic-point-data.entity'; @Module({ imports: [ HttpModule, UserModule, - TypeOrmModule.forFeature([ - PointDataEntity, - PointDataDynamicStatusEntity, - DynamicPointDataEntity, - ]), + TypeOrmModule.forFeature([PointDataEntity, PointDataDynamicStatusEntity]), WhatsappModule, ], providers: [PointDataService, HelperService], diff --git a/services/API-service/src/api/point-data/point-data.service.ts b/services/API-service/src/api/point-data/point-data.service.ts index 5c1d5c6bb..9f41695bb 100644 --- a/services/API-service/src/api/point-data/point-data.service.ts +++ b/services/API-service/src/api/point-data/point-data.service.ts @@ -3,7 +3,7 @@ import { InjectRepository } from '@nestjs/typeorm'; import { validate } from 'class-validator'; import { GeoJson } from '../../shared/geo.model'; import { HelperService } from '../../shared/helper.service'; -import { IsNull, MoreThanOrEqual, Repository } from 'typeorm'; +import { MoreThanOrEqual, Repository } from 'typeorm'; import { EvacuationCenterDto } from './dto/upload-evacuation-centers.dto'; import { PointDataEntity, PointDataEnum } from './point-data.entity'; import { DamSiteDto } from './dto/upload-dam-sites.dto'; @@ -13,14 +13,9 @@ import { CommunityNotificationDto } from './dto/upload-community-notifications.d import { WhatsappService } from '../notification/whatsapp/whatsapp.service'; import { SchoolDto } from './dto/upload-schools.dto'; import { WaterpointDto } from './dto/upload-waterpoint.dto'; -import { - UploadAssetExposureStatusDto, - UploadDynamicPointDataDto, -} from './dto/upload-asset-exposure-status.dto'; +import { UploadAssetExposureStatusDto } from './dto/upload-asset-exposure-status.dto'; import { PointDataDynamicStatusEntity } from './point-data-dynamic-status.entity'; import { DisasterType } from '../disaster/disaster-type.enum'; -import { GaugeDto } from './dto/upload-gauge.dto'; -import { DynamicPointDataEntity } from './dynamic-point-data.entity'; @Injectable() export class PointDataService { @@ -28,8 +23,6 @@ export class PointDataService { private readonly pointDataRepository: Repository; @InjectRepository(PointDataDynamicStatusEntity) private readonly pointDataDynamicStatusRepo: Repository; - @InjectRepository(DynamicPointDataEntity) - private readonly dynamicPointDataRepository: Repository; public constructor( private readonly helperService: HelperService, @@ -54,38 +47,20 @@ export class PointDataService { selectColumns.push('geom'); selectColumns.push('"pointDataId"'); - const recentDate = await this.helperService.getRecentDate( - countryCodeISO3, - disasterType, - ); - const pointDataQuery = this.pointDataRepository .createQueryBuilder('point') .select(selectColumns) .where({ pointDataCategory: pointDataCategory, countryCodeISO3: countryCodeISO3, - }) - .leftJoin( - (subquery) => { - return subquery - .select([ - 'dynamic."pointPointDataId"', - 'json_object_agg("key",value) as "dynamicData"', - ]) - .from(DynamicPointDataEntity, 'dynamic') - .where('dynamic.timestamp = :modelTimestamp', { - modelTimestamp: recentDate.timestamp, - }) - .groupBy('dynamic."pointPointDataId"'); - }, - 'dynamic', - 'dynamic."pointPointDataId" = point."pointDataId"', - ) - .addSelect('dynamic."dynamicData" as "dynamicData"'); + }); // TO DO: hard-code for now if (disasterType === DisasterType.FlashFloods) { + const recentDate = await this.helperService.getRecentDate( + countryCodeISO3, + disasterType, + ); pointDataQuery .leftJoin( PointDataDynamicStatusEntity, @@ -120,8 +95,6 @@ export class PointDataService { return new SchoolDto(); case PointDataEnum.waterpointsInternal: return new WaterpointDto(); - case PointDataEnum.gauges: - return new GaugeDto(); default: throw new HttpException( 'Not a known point layer', @@ -286,40 +259,4 @@ export class PointDataService { } await this.pointDataDynamicStatusRepo.save(assetForecasts); } - - async uploadDynamicPointData(dynamicPointData: UploadDynamicPointDataDto) { - const dynamicPointDataArray: DynamicPointDataEntity[] = []; - - for (const point of dynamicPointData.dynamicPointData) { - const asset = await this.pointDataRepository.findOne({ - where: { - referenceId: Number(point.fid), // TODO: make sure this is unique - }, - }); - if (!asset) { - continue; - } - - // Delete existing entries - await this.dynamicPointDataRepository.delete({ - point: { pointDataId: asset.pointDataId }, - leadTime: dynamicPointData.leadTime || IsNull(), - timestamp: MoreThanOrEqual( - this.helperService.getUploadCutoffMoment( - dynamicPointData.disasterType, - dynamicPointData.date || new Date(), - ), - ), - }); - - const dynamicPoint = new DynamicPointDataEntity(); - dynamicPoint.key = dynamicPointData.key; - dynamicPoint.leadTime = dynamicPointData.leadTime; - dynamicPoint.timestamp = dynamicPointData.date || new Date(); - dynamicPoint.value = point.value; - dynamicPoint.point = asset; - dynamicPointDataArray.push(dynamicPoint); - } - await this.dynamicPointDataRepository.save(dynamicPointDataArray); - } } diff --git a/services/API-service/src/scripts/git-lfs/standard-point-layers/gauges_MWI.csv b/services/API-service/src/scripts/git-lfs/standard-point-layers/gauges_MWI.csv deleted file mode 100644 index 7b65a6109..000000000 --- a/services/API-service/src/scripts/git-lfs/standard-point-layers/gauges_MWI.csv +++ /dev/null @@ -1,4 +0,0 @@ -id,fid,name,Country,lat,lon -1,1,Kariba,MWI,-9.940606036,33.92438203 -2,2,Tugwi Mukosi,MWI,-9.938750283,33.92635764 -3,3,Mutirikwi,MWI,-11.89861174,33.59389079 \ No newline at end of file diff --git a/services/API-service/src/scripts/json/layer-metadata.json b/services/API-service/src/scripts/json/layer-metadata.json index 12ad17469..8ec4b6783 100644 --- a/services/API-service/src/scripts/json/layer-metadata.json +++ b/services/API-service/src/scripts/json/layer-metadata.json @@ -469,14 +469,6 @@ "active": "no", "description": { "MWI": { "flash-floods": "TBD" } } }, - { - "name": "gauges", - "label": "River gauges", - "type": "point", - "leadTimeDependent": false, - "active": "no", - "description": { "MWI": { "flash-floods": "TBD" } } - }, { "name": "roads", "label": "Roads", diff --git a/services/API-service/src/scripts/scripts.service.ts b/services/API-service/src/scripts/scripts.service.ts index 3c7437444..8c0fc56f9 100644 --- a/services/API-service/src/scripts/scripts.service.ts +++ b/services/API-service/src/scripts/scripts.service.ts @@ -28,10 +28,7 @@ import { LinesDataService } from '../api/lines-data/lines-data.service'; import { UploadLinesExposureStatusDto } from '../api/lines-data/dto/upload-asset-exposure-status.dto'; import { LinesDataEnum } from '../api/lines-data/lines-data.entity'; import { PointDataEnum } from '../api/point-data/point-data.entity'; -import { - UploadAssetExposureStatusDto, - UploadDynamicPointDataDto, -} from '../api/point-data/dto/upload-asset-exposure-status.dto'; +import { UploadAssetExposureStatusDto } from '../api/point-data/dto/upload-asset-exposure-status.dto'; import { PointDataService } from '../api/point-data/point-data.service'; @Injectable() @@ -195,11 +192,6 @@ export class ScriptsService { mockInput.triggered, date, ); - await this.mockDynamicPointData( - selectedCountry.countryCodeISO3, - mockInput.disasterType, - date, - ); } if (mockInput.disasterType === DisasterType.Typhoon) { @@ -1026,30 +1018,6 @@ export class ScriptsService { } } - private async mockDynamicPointData( - countryCodeISO3: string, - disasterType: DisasterType, - date: Date, - ) { - if (countryCodeISO3 !== 'MWI') { - return; - } - - const keys = ['water-level']; - for (const key of keys) { - const payload = new UploadDynamicPointDataDto(); - payload.key = key; - payload.leadTime = null; - payload.date = date || new Date(); - payload.disasterType = disasterType; - const filename = `./src/api/point-data/dto/example/${countryCodeISO3}/${DisasterType.FlashFloods}/dynamic-point-data_${key}.json`; - const dynamicPointData = JSON.parse(fs.readFileSync(filename, 'utf-8')); - payload.dynamicPointData = dynamicPointData; - - await this.pointDataService.uploadDynamicPointData(payload); - } - } - private async mockRasterFile( selectedCountry, disasterType: DisasterType, diff --git a/services/API-service/src/scripts/seed-point-data.ts b/services/API-service/src/scripts/seed-point-data.ts index f4350b44c..236efe678 100644 --- a/services/API-service/src/scripts/seed-point-data.ts +++ b/services/API-service/src/scripts/seed-point-data.ts @@ -29,7 +29,6 @@ export class SeedPointData implements InterfaceScript { this.seedPointData(PointDataEnum.dams, country); this.seedPointData(PointDataEnum.schools, country); this.seedPointData(PointDataEnum.waterpointsInternal, country); - this.seedPointData(PointDataEnum.gauges, country); return; } else { return Promise.resolve();