From 5d02e042b8512ee9256eff905797070ea0727c82 Mon Sep 17 00:00:00 2001 From: Angel Penchev <26301867+angel-penchev@users.noreply.github.com> Date: Thu, 28 Oct 2021 23:20:07 +0300 Subject: [PATCH] [Feat] NeuralNetworks Created endpoints for viewing NNs and getting them and their metadata. Resolves #23. --- server/.env.template | 1 + server/src/app.module.ts | 6 ++ server/src/configuration.ts | 1 + .../neural-networks.controller.spec.ts | 18 ++++++ .../controllers/neural-networks.controller.ts | 41 +++++++++++++ .../errors/neural-networks.error.codes.ts | 3 + .../errors/neural-networks.error.ts | 6 ++ .../neural-networks/neural-networks.module.ts | 12 ++++ .../service/neural-networks.service.spec.ts | 18 ++++++ .../service/neural-networks.service.ts | 60 +++++++++++++++++++ server/src/users/entities/user.entity.ts | 6 -- server/types/enviourment.d.ts | 1 + 12 files changed, 167 insertions(+), 6 deletions(-) create mode 100644 server/src/neural-networks/controllers/neural-networks.controller.spec.ts create mode 100644 server/src/neural-networks/controllers/neural-networks.controller.ts create mode 100644 server/src/neural-networks/errors/neural-networks.error.codes.ts create mode 100644 server/src/neural-networks/errors/neural-networks.error.ts create mode 100644 server/src/neural-networks/neural-networks.module.ts create mode 100644 server/src/neural-networks/service/neural-networks.service.spec.ts create mode 100644 server/src/neural-networks/service/neural-networks.service.ts diff --git a/server/.env.template b/server/.env.template index e9c6da2..09f0759 100644 --- a/server/.env.template +++ b/server/.env.template @@ -30,3 +30,4 @@ AWS_ACCESS_KEY_ID= AWS_SECRET_ACCESS_KEY= AWS_CODE_SNIPPETS_S3_BUCKET= AWS_RECORDINGS_S3_BUCKET= +AWS_NEURAL_NETWORKS_S3_BUCKET= diff --git a/server/src/app.module.ts b/server/src/app.module.ts index da8df97..13b9a38 100644 --- a/server/src/app.module.ts +++ b/server/src/app.module.ts @@ -18,6 +18,9 @@ import { User } from './users/entities/user.entity'; import { UsersController } from './users/controllers/users.controller'; import { UsersModule } from './users/users.module'; import { UsersService } from './users/services/users.service'; +import { NeuralNetworksModule } from './neural-networks/neural-networks.module'; +import { NeuralNetworksController } from './neural-networks/controllers/neural-networks.controller'; +import { NeuralNetworksService } from './neural-networks/service/neural-networks.service'; import { configuration } from './configuration'; @Module({ @@ -26,6 +29,7 @@ import { configuration } from './configuration'; AuthModule, CodeSnippetsModule, RecordingsModule, + NeuralNetworksModule, ConfigModule.forRoot({ load: [configuration], }), @@ -51,12 +55,14 @@ import { configuration } from './configuration'; AuthController, CodeSnippetsController, RecordingsController, + NeuralNetworksController, ], providers: [ UsersService, AuthService, CodeSnippetsService, RecordingsService, + NeuralNetworksService, ], }) export class AppModule {} diff --git a/server/src/configuration.ts b/server/src/configuration.ts index 39fb05d..f4fe426 100644 --- a/server/src/configuration.ts +++ b/server/src/configuration.ts @@ -70,6 +70,7 @@ const configuration = () => ({ secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY, codeSnippetsS3Bucket: process.env.AWS_CODE_SNIPPETS_S3_BUCKET, recordingsS3Bucket: process.env.AWS_RECORDINGS_S3_BUCKET, + neuralNetworksS3Bucket: process.env.AWS_NEURAL_NETWORKS_S3_BUCKET, }, }); diff --git a/server/src/neural-networks/controllers/neural-networks.controller.spec.ts b/server/src/neural-networks/controllers/neural-networks.controller.spec.ts new file mode 100644 index 0000000..297699b --- /dev/null +++ b/server/src/neural-networks/controllers/neural-networks.controller.spec.ts @@ -0,0 +1,18 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { NeuralNetworksController } from './neural-networks.controller'; + +describe('NeuralNetworksController', () => { + let controller: NeuralNetworksController; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + controllers: [NeuralNetworksController], + }).compile(); + + controller = module.get(NeuralNetworksController); + }); + + it('should be defined', () => { + expect(controller).toBeDefined(); + }); +}); diff --git a/server/src/neural-networks/controllers/neural-networks.controller.ts b/server/src/neural-networks/controllers/neural-networks.controller.ts new file mode 100644 index 0000000..ddb2fe8 --- /dev/null +++ b/server/src/neural-networks/controllers/neural-networks.controller.ts @@ -0,0 +1,41 @@ +import { + Controller, + Get, + HttpCode, + HttpStatus, + Param, + ParseBoolPipe, + Query, +} from '@nestjs/common'; +import { ApiNotFoundResponse, ApiOkResponse } from '@nestjs/swagger'; +import { NeuralNetworksService } from '../service/neural-networks.service'; + +@Controller('/neural-networks') +export class NeuralNetworksController { + constructor(private readonly neuralNetworksService: NeuralNetworksService) {} + + @Get('/') + @HttpCode(HttpStatus.OK) + @ApiOkResponse({ + description: 'List of neural network models on the server.', + }) + @ApiNotFoundResponse({ description: 'No such file or directory.' }) + async listNetworks() { + return await this.neuralNetworksService.listModels(); + } + + @Get(':path(*)') + @HttpCode(HttpStatus.OK) + @ApiOkResponse({ + description: 'Neural network model or its metadata in JSON.', + }) + @ApiNotFoundResponse({ description: 'No such file or directory.' }) + async getNetwork( + @Param('path') path: string, + @Query('metadata', new ParseBoolPipe()) metadata: boolean, + ) { + return metadata + ? await this.neuralNetworksService.getModelMetadata(path) + : await this.neuralNetworksService.getModel(path); + } +} diff --git a/server/src/neural-networks/errors/neural-networks.error.codes.ts b/server/src/neural-networks/errors/neural-networks.error.codes.ts new file mode 100644 index 0000000..023fb07 --- /dev/null +++ b/server/src/neural-networks/errors/neural-networks.error.codes.ts @@ -0,0 +1,3 @@ +export enum NeuralNetworksErrorCode { + INVALID_FILE_PATH = 'Invalid file path.', +} diff --git a/server/src/neural-networks/errors/neural-networks.error.ts b/server/src/neural-networks/errors/neural-networks.error.ts new file mode 100644 index 0000000..25fb5e3 --- /dev/null +++ b/server/src/neural-networks/errors/neural-networks.error.ts @@ -0,0 +1,6 @@ +export default class NeuralNetworksError extends Error { + constructor(message: string) { + super(message); + this.name = 'NeuralNetworksError'; + } +} diff --git a/server/src/neural-networks/neural-networks.module.ts b/server/src/neural-networks/neural-networks.module.ts new file mode 100644 index 0000000..569e24c --- /dev/null +++ b/server/src/neural-networks/neural-networks.module.ts @@ -0,0 +1,12 @@ +import { Module } from '@nestjs/common'; +import { NeuralNetworksService } from './service/neural-networks.service'; +import { NeuralNetworksController } from './controllers/neural-networks.controller'; +import { AwsSdkModule } from 'nest-aws-sdk'; +import { S3 } from 'aws-sdk'; + +@Module({ + imports: [AwsSdkModule.forFeatures([S3])], + providers: [NeuralNetworksService], + controllers: [NeuralNetworksController], +}) +export class NeuralNetworksModule {} diff --git a/server/src/neural-networks/service/neural-networks.service.spec.ts b/server/src/neural-networks/service/neural-networks.service.spec.ts new file mode 100644 index 0000000..8f89c42 --- /dev/null +++ b/server/src/neural-networks/service/neural-networks.service.spec.ts @@ -0,0 +1,18 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { NeuralNetworksService } from './neural-networks.service'; + +describe('NeuralNetworksService', () => { + let service: NeuralNetworksService; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + providers: [NeuralNetworksService], + }).compile(); + + service = module.get(NeuralNetworksService); + }); + + it('should be defined', () => { + expect(service).toBeDefined(); + }); +}); diff --git a/server/src/neural-networks/service/neural-networks.service.ts b/server/src/neural-networks/service/neural-networks.service.ts new file mode 100644 index 0000000..af485aa --- /dev/null +++ b/server/src/neural-networks/service/neural-networks.service.ts @@ -0,0 +1,60 @@ +import { Injectable } from '@nestjs/common'; +import { S3 } from 'aws-sdk'; +import { InjectAwsService } from 'nest-aws-sdk'; +import { configObject } from 'src/configuration'; +import NeuralNetworksError from '../errors/neural-networks.error'; +import { NeuralNetworksErrorCode } from '../errors/neural-networks.error.codes'; + +@Injectable() +export class NeuralNetworksService { + constructor(@InjectAwsService(S3) private readonly s3: S3) {} + + async listModels() { + const queryResponse = await this.s3 + .listObjectsV2({ + Bucket: configObject.aws.neuralNetworksS3Bucket, + Prefix: `models/`, + }) + .promise(); + + const result = queryResponse.Contents.filter( + (item) => !item.Key.endsWith('/'), + ).map((item) => item.Key.replace('models/', '')); + + return result; + } + + async getModel(filepath: string) { + const completeFilepath = `models/${filepath}`; + + if (completeFilepath.endsWith('/')) { + throw new NeuralNetworksError(NeuralNetworksErrorCode.INVALID_FILE_PATH); + } + + const queryResponse = await this.s3 + .getObject({ + Bucket: configObject.aws.neuralNetworksS3Bucket, + Key: completeFilepath, + }) + .promise(); + + return JSON.parse(queryResponse.Body.toString()); + } + + async getModelMetadata(filepath: string) { + const completeFilepath = `metadata/${filepath}`; + + if (completeFilepath.endsWith('/')) { + throw new NeuralNetworksError(NeuralNetworksErrorCode.INVALID_FILE_PATH); + } + + const queryResponse = await this.s3 + .getObject({ + Bucket: configObject.aws.neuralNetworksS3Bucket, + Key: completeFilepath, + }) + .promise(); + + return JSON.parse(queryResponse.Body.toString()); + } +} diff --git a/server/src/users/entities/user.entity.ts b/server/src/users/entities/user.entity.ts index a7ce8bd..06df80f 100644 --- a/server/src/users/entities/user.entity.ts +++ b/server/src/users/entities/user.entity.ts @@ -1,10 +1,4 @@ import { Column, Entity, PrimaryGeneratedColumn } from 'typeorm'; -import { - IsEmail, - IsNotEmpty, - IsOptional, - IsPhoneNumber, -} from 'class-validator'; @Entity({ name: 'users' }) export class User implements IUser { diff --git a/server/types/enviourment.d.ts b/server/types/enviourment.d.ts index 47e8ed4..5070203 100644 --- a/server/types/enviourment.d.ts +++ b/server/types/enviourment.d.ts @@ -34,5 +34,6 @@ declare namespace NodeJS { AWS_SECRET_ACCESS_KEY?: string; AWS_CODE_SNIPPETS_S3_BUCKET?: string; AWS_RECORDINGS_S3_BUCKET?: string; + AWS_NEURAL_NETWORKS_S3_BUCKET?: string; } }