feat: dynamic point data entity + gauge layer AB#25124

interfaces/IBF-dashboard/src/app/types/ibf-layer.ts

@@ -123,6 +123,7 @@ export enum IbfLayerName {
   schools = 'schools',
   waterpointsInternal = 'waterpoints_internal',
   roads = 'roads',
+  gauges = 'gauges',
 }
 
 export enum IbfLayerLabel {

services/API-service/migration/1701157179286-DynamicPointData.ts

@@ -0,0 +1,47 @@
+import { MigrationInterface, QueryRunner } from 'typeorm';
+
+export class DynamicPointData1701157179286 implements MigrationInterface {
+  name = 'DynamicPointData1701157179286';
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `CREATE TABLE "IBF-app"."dynamic-point-data" ("dynamicPointDataId" uuid NOT NULL DEFAULT uuid_generate_v4(), "timestamp" TIMESTAMP NOT NULL, "key" character varying NOT NULL, "value" character varying, "pointPointDataId" uuid, "leadTime" character varying, CONSTRAINT "PK_4f51e8c4a1091afa35e3cc51b34" PRIMARY KEY ("dynamicPointDataId"))`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "IBF-app"."dynamic-point-data" ADD CONSTRAINT "FK_9aaa9be5dbe82b610209bcb456b" FOREIGN KEY ("pointPointDataId") REFERENCES "IBF-app"."point-data"("pointDataId") ON DELETE NO ACTION ON UPDATE NO ACTION`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "IBF-app"."dynamic-point-data" ADD CONSTRAINT "FK_289a1f52e25e270d9a28bd9d35a" FOREIGN KEY ("leadTime") REFERENCES "IBF-app"."lead-time"("leadTimeName") ON DELETE NO ACTION ON UPDATE NO ACTION`,
+    );
+
+    // NOTE: Do not prioritize this, but leave the code here for future reference
+    // await queryRunner.query(`INSERT INTO "IBF-app"."point-data"
+    // ("pointDataId", "countryCodeISO3", "pointDataCategory", "attributes", geom, "referenceId")
+    // select id
+    //   ,"countryCodeISO3"
+    //   ,'glofas_stations'
+    //   ,(cast('{"stationName":"' as varchar) || "stationName" || cast('","stationCode":"' || "stationCode" || '"}' as varchar))::json
+    //   ,geom
+    //   ,null
+    // from "IBF-app"."glofas-station" `);
+
+    // await queryRunner.query(`INSERT INTO "IBF-app"."dynamic-point-data"
+    // ("dynamicPointDataId", "timestamp", "key", value, "pointPointDataId")
+    // select uuid_generate_v4()
+    //   ,date
+    //   ,unnest(array['forecastLevel','forecastReturnPeriod','triggerLevel','eapAlertClass']) as key
+    //   ,unnest(array[cast("forecastLevel" as varchar),cast("forecastReturnPeriod" as varchar),cast("triggerLevel" as varchar),"eapAlertClass"]) as value
+    //   ,"glofasStationId"
+    // from "IBF-app"."glofas-station-forecast" gsf `);
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE "IBF-app"."dynamic-point-data" DROP CONSTRAINT "FK_289a1f52e25e270d9a28bd9d35a"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "IBF-app"."dynamic-point-data" DROP CONSTRAINT "FK_9aaa9be5dbe82b610209bcb456b"`,
+    );
+    await queryRunner.query(`DROP TABLE "IBF-app"."dynamic-point-data"`);
+  }
+}

services/API-service/src/api/point-data/dto/example/MWI/flash-floods/dynamic-point-data_water-level.json

@@ -0,0 +1,14 @@
+[
+  {
+    "fid": 1,
+    "value": 100
+  },
+  {
+    "fid": 2,
+    "value": 100
+  },
+  {
+    "fid": 3,
+    "value": 100
+  }
+]

services/API-service/src/api/point-data/dto/upload-asset-exposure-status.dto.ts

@@ -4,11 +4,13 @@ import {
   IsNotEmpty,
   IsOptional,
   IsString,
+  ValidateNested,
 } from 'class-validator';
 import { ApiProperty } from '@nestjs/swagger';
 import { LeadTime } from '../../admin-area-dynamic-data/enum/lead-time.enum';
 import { DisasterType } from '../../disaster/disaster-type.enum';
 import { PointDataEnum } from '../point-data.entity';
+import { Type } from 'class-transformer';
 
 export class UploadAssetExposureStatusDto {
   @ApiProperty({ example: ['123', '234'] })
@@ -39,3 +41,37 @@ export class UploadAssetExposureStatusDto {
   @IsEnum(PointDataEnum)
   public pointDataCategory: PointDataEnum;
 }
+
+export class UploadDynamicPointDataDto {
+  @ApiProperty({ example: LeadTime.hour1 })
+  @IsOptional()
+  @IsString()
+  public leadTime: LeadTime;
+
+  @ApiProperty({ example: new Date() })
+  @IsOptional()
+  public date: Date;
+
+  @ApiProperty({ example: DisasterType.FlashFloods })
+  @IsNotEmpty()
+  @IsEnum(DisasterType)
+  public disasterType: DisasterType;
+
+  @ApiProperty({ example: 'waterLevel' })
+  public key: string;
+
+  @ApiProperty({ example: [{ fid: 1, value: 100 }] })
+  @IsArray()
+  @ValidateNested()
+  @Type(() => DynamicPointData)
+  public dynamicPointData: DynamicPointData[];
+}
+
+export class DynamicPointData {
+  @ApiProperty()
+  @IsNotEmpty()
+  public fid: string;
+
+  @ApiProperty()
+  public value: string;
+}

services/API-service/src/api/point-data/dto/upload-gauge.dto.ts

@@ -0,0 +1,20 @@
+import { IsNotEmpty, IsOptional, IsString } from 'class-validator';
+import { ApiProperty } from '@nestjs/swagger';
+
+export class GaugeDto {
+  @ApiProperty({ example: 'name' })
+  @IsString()
+  public name: string = undefined;
+
+  @ApiProperty({ example: 1234 })
+  @IsOptional()
+  public fid: number = undefined;
+
+  @ApiProperty({ example: 0 })
+  @IsNotEmpty()
+  public lat: number;
+
+  @ApiProperty({ example: 0 })
+  @IsNotEmpty()
+  public lon: number;
+}

services/API-service/src/api/point-data/dynamic-point-data.entity.ts

@@ -0,0 +1,34 @@
+import {
+  Entity,
+  PrimaryGeneratedColumn,
+  Column,
+  ManyToOne,
+  JoinColumn,
+} from 'typeorm';
+import { PointDataEntity } from './point-data.entity';
+import { LeadTimeEntity } from '../lead-time/lead-time.entity';
+
+@Entity('dynamic-point-data')
+export class DynamicPointDataEntity {
+  @PrimaryGeneratedColumn('uuid')
+  public dynamicPointDataId: string;
+
+  @ManyToOne(
+    (): typeof PointDataEntity => PointDataEntity,
+    (point): DynamicPointDataEntity[] => point.dynamicData,
+  )
+  public point: PointDataEntity;
+
+  @ManyToOne((): typeof LeadTimeEntity => LeadTimeEntity)
+  @JoinColumn({ name: 'leadTime', referencedColumnName: 'leadTimeName' })
+  public leadTime: string;
+
+  @Column({ type: 'timestamp' })
+  public timestamp: Date;
+
+  @Column()
+  public key: string;
+
+  @Column({ nullable: true })
+  public value: string;
+}

services/API-service/src/api/point-data/point-data.controller.ts

@@ -26,7 +26,10 @@ import { RolesGuard } from '../../roles.guard';
 import { GeoJson } from '../../shared/geo.model';
 import { UserRole } from '../user/user-role.enum';
 import { PointDataService } from './point-data.service';
-import { UploadAssetExposureStatusDto } from './dto/upload-asset-exposure-status.dto';
+import {
+  UploadAssetExposureStatusDto,
+  UploadDynamicPointDataDto,
+} from './dto/upload-asset-exposure-status.dto';
 import { FILE_UPLOAD_API_FORMAT } from '../../shared/file-upload-api-format';
 
 @ApiBearerAuth()
@@ -132,4 +135,17 @@ export class PointDataController {
   ): Promise<void> {
     return await this.pointDataService.uploadAssetExposureStatus(assetFids);
   }
+
+  @UseGuards(RolesGuard)
+  @ApiOperation({ summary: 'Upload dynamic point data' })
+  @ApiResponse({
+    status: 201,
+    description: 'Uploaded dynamic point data.',
+  })
+  @Post('dynamic')
+  public async uploadDynamicPointData(
+    @Body() dynamicPointData: UploadDynamicPointDataDto,
+  ): Promise<void> {
+    return await this.pointDataService.uploadDynamicPointData(dynamicPointData);
+  }
 }

services/API-service/src/api/point-data/point-data.entity.ts

@@ -1,4 +1,5 @@
-import { Entity, PrimaryGeneratedColumn, Column } from 'typeorm';
+import { Entity, PrimaryGeneratedColumn, Column, OneToMany } from 'typeorm';
+import { DynamicPointDataEntity } from './dynamic-point-data.entity';
 
 export enum PointDataEnum {
   evacuationCenters = 'evacuation_centers',
@@ -8,6 +9,7 @@ export enum PointDataEnum {
   communityNotifications = 'community_notifications',
   schools = 'schools',
   waterpointsInternal = 'waterpoints_internal',
+  gauges = 'gauges',
 }
 
 @Entity('point-data')
@@ -29,4 +31,10 @@ export class PointDataEntity {
 
   @Column('json', { nullable: true })
   public geom: JSON;
+
+  @OneToMany(
+    (): typeof DynamicPointDataEntity => DynamicPointDataEntity,
+    (dynamicData): PointDataEntity => dynamicData.point,
+  )
+  public dynamicData: DynamicPointDataEntity[];
 }

services/API-service/src/api/point-data/point-data.module.ts

@@ -8,12 +8,17 @@ import { PointDataEntity } from './point-data.entity';
 import { PointDataService } from './point-data.service';
 import { PointDataDynamicStatusEntity } from './point-data-dynamic-status.entity';
 import { HttpModule } from '@nestjs/axios';
+import { DynamicPointDataEntity } from './dynamic-point-data.entity';
 
 @Module({
   imports: [
     HttpModule,
     UserModule,
-    TypeOrmModule.forFeature([PointDataEntity, PointDataDynamicStatusEntity]),
+    TypeOrmModule.forFeature([
+      PointDataEntity,
+      PointDataDynamicStatusEntity,
+      DynamicPointDataEntity,
+    ]),
     WhatsappModule,
   ],
   providers: [PointDataService, HelperService],

services/API-service/src/api/point-data/point-data.service.ts

@@ -3,7 +3,7 @@ import { InjectRepository } from '@nestjs/typeorm';
 import { validate } from 'class-validator';
 import { GeoJson } from '../../shared/geo.model';
 import { HelperService } from '../../shared/helper.service';
-import { MoreThanOrEqual, Repository } from 'typeorm';
+import { IsNull, MoreThanOrEqual, Repository } from 'typeorm';
 import { EvacuationCenterDto } from './dto/upload-evacuation-centers.dto';
 import { PointDataEntity, PointDataEnum } from './point-data.entity';
 import { DamSiteDto } from './dto/upload-dam-sites.dto';
@@ -13,16 +13,23 @@ import { CommunityNotificationDto } from './dto/upload-community-notifications.d
 import { WhatsappService } from '../notification/whatsapp/whatsapp.service';
 import { SchoolDto } from './dto/upload-schools.dto';
 import { WaterpointDto } from './dto/upload-waterpoint.dto';
-import { UploadAssetExposureStatusDto } from './dto/upload-asset-exposure-status.dto';
+import {
+  UploadAssetExposureStatusDto,
+  UploadDynamicPointDataDto,
+} from './dto/upload-asset-exposure-status.dto';
 import { PointDataDynamicStatusEntity } from './point-data-dynamic-status.entity';
 import { DisasterType } from '../disaster/disaster-type.enum';
+import { GaugeDto } from './dto/upload-gauge.dto';
+import { DynamicPointDataEntity } from './dynamic-point-data.entity';
 
 @Injectable()
 export class PointDataService {
   @InjectRepository(PointDataEntity)
   private readonly pointDataRepository: Repository<PointDataEntity>;
   @InjectRepository(PointDataDynamicStatusEntity)
   private readonly pointDataDynamicStatusRepo: Repository<PointDataDynamicStatusEntity>;
+  @InjectRepository(DynamicPointDataEntity)
+  private readonly dynamicPointDataRepository: Repository<DynamicPointDataEntity>;
 
   public constructor(
     private readonly helperService: HelperService,
@@ -47,20 +54,38 @@ export class PointDataService {
     selectColumns.push('geom');
     selectColumns.push('"pointDataId"');
 
+    const recentDate = await this.helperService.getRecentDate(
+      countryCodeISO3,
+      disasterType,
+    );
+
     const pointDataQuery = this.pointDataRepository
       .createQueryBuilder('point')
       .select(selectColumns)
       .where({
         pointDataCategory: pointDataCategory,
         countryCodeISO3: countryCodeISO3,
-      });
+      })
+      .leftJoin(
+        (subquery) => {
+          return subquery
+            .select([
+              'dynamic."pointPointDataId"',
+              'json_object_agg("key",value) as "dynamicData"',
+            ])
+            .from(DynamicPointDataEntity, 'dynamic')
+            .where('dynamic.timestamp = :modelTimestamp', {
+              modelTimestamp: recentDate.timestamp,
+            })
+            .groupBy('dynamic."pointPointDataId"');
+        },
+        'dynamic',
+        'dynamic."pointPointDataId" = point."pointDataId"',
+      )
+      .addSelect('dynamic."dynamicData" as "dynamicData"');
 
     // TO DO: hard-code for now
     if (disasterType === DisasterType.FlashFloods) {
-      const recentDate = await this.helperService.getRecentDate(
-        countryCodeISO3,
-        disasterType,
-      );
       pointDataQuery
         .leftJoin(
           PointDataDynamicStatusEntity,
@@ -95,6 +120,8 @@ export class PointDataService {
         return new SchoolDto();
       case PointDataEnum.waterpointsInternal:
         return new WaterpointDto();
+      case PointDataEnum.gauges:
+        return new GaugeDto();
       default:
         throw new HttpException(
           'Not a known point layer',
@@ -259,4 +286,40 @@ export class PointDataService {
     }
     await this.pointDataDynamicStatusRepo.save(assetForecasts);
   }
+
+  async uploadDynamicPointData(dynamicPointData: UploadDynamicPointDataDto) {
+    const dynamicPointDataArray: DynamicPointDataEntity[] = [];
+
+    for (const point of dynamicPointData.dynamicPointData) {
+      const asset = await this.pointDataRepository.findOne({
+        where: {
+          referenceId: Number(point.fid), // TODO: make sure this is unique
+        },
+      });
+      if (!asset) {
+        continue;
+      }
+
+      // Delete existing entries
+      await this.dynamicPointDataRepository.delete({
+        point: { pointDataId: asset.pointDataId },
+        leadTime: dynamicPointData.leadTime || IsNull(),
+        timestamp: MoreThanOrEqual(
+          this.helperService.getUploadCutoffMoment(
+            dynamicPointData.disasterType,
+            dynamicPointData.date || new Date(),
+          ),
+        ),
+      });
+
+      const dynamicPoint = new DynamicPointDataEntity();
+      dynamicPoint.key = dynamicPointData.key;
+      dynamicPoint.leadTime = dynamicPointData.leadTime;
+      dynamicPoint.timestamp = dynamicPointData.date || new Date();
+      dynamicPoint.value = point.value;
+      dynamicPoint.point = asset;
+      dynamicPointDataArray.push(dynamicPoint);
+    }
+    await this.dynamicPointDataRepository.save(dynamicPointDataArray);
+  }
 }

services/API-service/src/scripts/git-lfs/standard-point-layers/gauges_MWI.csv

@@ -0,0 +1,4 @@
+id,fid,name,Country,lat,lon
+1,1,Kariba,MWI,-9.940606036,33.92438203
+2,2,Tugwi Mukosi,MWI,-9.938750283,33.92635764
+3,3,Mutirikwi,MWI,-11.89861174,33.59389079