Skip to content

Commit

Permalink
Merge pull request #287 from swisstopo/feature/asset-36-integrate-aws…
Browse files Browse the repository at this point in the history
…-textract

Feature 36: Integrate AWS Textract
  • Loading branch information
daniel-va authored Oct 16, 2024
2 parents b60efd5 + 71dce22 commit 9e75c60
Show file tree
Hide file tree
Showing 15 changed files with 399 additions and 194 deletions.
1 change: 1 addition & 0 deletions .github/workflows/code-quality.yml
Original file line number Diff line number Diff line change
Expand Up @@ -182,6 +182,7 @@ jobs:
DB_PASSWORD: ${{ env.DB_PASSWORD }}
run: |
cd development
touch .env.ocr
dos2unix ./init/elasticsearch/init.sh
chmod +x ./init/elasticsearch/init.sh
chmod +r ./init/elasticsearch/mappings/swissgeol_asset_asset.json
Expand Down
10 changes: 7 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ Be aware that you need to manually insert the `{DB_*}` values beforehand.

```bash
cd development
docker compose exec db sh -c 'pg_dump --dbname=postgresql://{DB_USERNAME}:{DB_PASSWORD}@{DB_HOST}:5432/{DB_DATABASE} --data-only --exclude-table asset_user --exclude-table _prisma_migrations -n public > /dump.sql'
docker compose exec db sh -c 'pg_dump --dbname=postgresql://{DB_USERNAME}:{DB_PASSWORD}@{DB_HOST}:5432/{DB_DATABASE} --data-only --exclude-table asset_user --exclude-table workgroups_on_users --exclude-table _prisma_migrations -n public > /dump.sql'
```

> The export will output warnings related to circular foreign-key constraints.
Expand All @@ -105,10 +105,14 @@ Ensure to start your database service beforehand.
```bash
# Reset the database:
npm run prisma -- migrate reset -f
npm run prisma -- migrate deploy

# Import example data:
# Switch to the directory containing the database's `docker-compose.yml`:
cd development

# Remove the initial workgroup as it will collide with the import:
docker compose exec db sh -c 'psql --dbname=postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@localhost:5432/${POSTGRES_DB} -c "DELETE FROM workgroup"'

# Import example data:
docker compose exec db sh -c 'psql --dbname=postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@localhost:5432/${POSTGRES_DB} -v ON_ERROR_STOP=1 -f /dump.sql'
```

Expand Down
4 changes: 1 addition & 3 deletions apps/server-asset-sg/.env
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,5 @@ OAUTH_SHOW_DEBUG_INFO=true
OAUTH_TOKEN_ENDPOINT=http://localhost:4011/connect/token
OAUTH_AUTHORIZED_GROUPS=assets.swissgeol
ANONYMOUS_MODE=false
OCR_URL=
OCR_CALLBACK_URL=


OCR_SERVICE_URL=http://localhost:5052
149 changes: 149 additions & 0 deletions apps/server-asset-sg/src/app.logger.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,149 @@
import { Injectable, LoggerService, LogLevel } from '@nestjs/common';
import colors from 'colors/safe';

@Injectable()
export class AppLogger implements LoggerService {
levels: Set<LogLevel> | null = null;

log(message: unknown, ...optionalParams: unknown[]) {
this.write(levels.log, message, optionalParams);
}

error(message: unknown, ...optionalParams: unknown[]) {
this.write(levels.error, message, optionalParams);
}

warn(message: unknown, ...optionalParams: unknown[]) {
this.write(levels.warn, message, optionalParams);
}

debug?(message: unknown, ...optionalParams: unknown[]) {
this.write(levels.debug, message, optionalParams);
}

verbose?(message: unknown, ...optionalParams: unknown[]) {
this.write(levels.verbose, message, optionalParams);
}

fatal?(message: unknown, ...optionalParams: unknown[]) {
this.write(levels.fatal, message, optionalParams);
}

setLogLevels?(levels: LogLevel[]) {
this.levels = new Set(levels);
}

private hasLevel(level: LogLevel): boolean {
return this.levels == null || this.levels.has(level);
}

private write(level: Level, message: unknown, params: unknown[]) {
if (!this.hasLevel(level.key)) {
return;
}
const lastParam = params[params.length - 1];
let source = 'main';
if (typeof lastParam === 'string') {
source = lastParam;
params = params.slice(0, params.length - 1);
}

const now = new Date();
const nameSpacer = ' '.repeat(MAX_NAME_LENGTH - level.name.length);
const prefix =
colors.reset(` ${now.toISOString()} `) + nameSpacer + level.bgColor(` ${level.name} `) + ' ' + source;
let output = ' ' + level.color(`${message}`);
if (params.length !== 0) {
output += ' ' + stringify(params, level);
}
console.log(`${prefix} ${output}`);
}
}

const stringify = (value: unknown, level: Level, options: { isNested?: boolean } = {}): string => {
if (Array.isArray(value)) {
return stringifyArray(value, level, options);
}
if (value instanceof Error) {
return JSON.stringify(value.message);
}
if (value != null && typeof value === 'object') {
return stringifyObject(value, level, options);
}
return level.color(JSON.stringify(value));
};

const stringifyArray = (value: unknown[], level: Level, options: { isNested?: boolean } = {}): string => {
let output = '';
for (const element of value) {
if (output.length > 0) {
output += ', ';
}
output += stringify(element, level, { isNested: true });
}
return options.isNested ? `[${output}]` : output;
};

const stringifyObject = (value: object, level: Level, options: { isNested?: boolean } = {}): string => {
const valueString = value.toString();
if (valueString !== '[object Object]') {
return level.color(valueString);
}

let output = '';
for (const [k, v] of Object.entries(value)) {
if (output.length > 0) {
output += ', ';
}
output += `${k}: ${stringify(v, level, { isNested: true })}`;
}
return options.isNested ? `[${output}]` : output;
};

interface Level {
key: LogLevel;
name: string;
color: typeof colors.white;
bgColor: typeof colors.bgWhite;
}

const levels: Record<LogLevel, Level> = {
log: {
key: 'log',
name: 'INFO',
color: colors.green,
bgColor: colors.bgGreen,
},
error: {
key: 'error',
name: 'ERROR',
color: colors.red,
bgColor: colors.bgRed,
},
warn: {
key: 'warn',
name: 'WARN',
color: colors.yellow,
bgColor: colors.bgYellow,
},
debug: {
key: 'debug',
name: 'DEBUG',
color: colors.blue,
bgColor: colors.bgBlue,
},
verbose: {
key: 'verbose',
name: 'VERBOSE',
color: colors.magenta,
bgColor: colors.bgMagenta,
},
fatal: {
key: 'fatal',
name: 'FATAL',
color: colors.red,
bgColor: colors.bgRed,
},
};

const MAX_NAME_LENGTH = Math.max(...Object.values(levels).map((it) => it.name.length));
4 changes: 2 additions & 2 deletions apps/server-asset-sg/src/app.module.ts
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,9 @@ import { ContactRepo } from '@/features/contacts/contact.repo';
import { ContactsController } from '@/features/contacts/contacts.controller';
import { FavoriteRepo } from '@/features/favorites/favorite.repo';
import { FavoritesController } from '@/features/favorites/favorites.controller';
import { FileOcrService } from '@/features/files/file-ocr.service';
import { FileRepo } from '@/features/files/file.repo';
import { FilesController } from '@/features/files/files.controller';
import { OcrController } from '@/features/ocr/ocr.controller';
import { StudiesController } from '@/features/studies/studies.controller';
import { StudyRepo } from '@/features/studies/study.repo';
import { UserRepo } from '@/features/users/user.repo';
Expand All @@ -43,7 +43,6 @@ import { WorkgroupsController } from '@/features/workgroups/workgroups.controlle
ContactsController,
FavoritesController,
FilesController,
OcrController,
StudiesController,
UsersController,
WorkgroupsController,
Expand All @@ -60,6 +59,7 @@ import { WorkgroupsController } from '@/features/workgroups/workgroups.controlle
ContactRepo,
FavoriteRepo,
FileRepo,
FileOcrService,
PrismaService,
StudyRepo,
UserRepo,
Expand Down
Loading

0 comments on commit 9e75c60

Please sign in to comment.