From 41cd0fbbf207f59bc79d0cd66e817538fbb3e34c Mon Sep 17 00:00:00 2001 From: Arthur de Moulins Date: Thu, 21 Dec 2023 17:57:17 +0100 Subject: [PATCH] PS-598 fix indexer rendition definition duplicates --- .../api/migrations/Version20231221125408.php | 34 +++++++ .../RenditionDefinitionInputTransformer.php | 92 +++++++++++++++++++ .../Model/Input/RenditionDefinitionInput.php | 85 +++++++++++++++++ .../RenditionDefinitionCrudController.php | 5 +- .../src/Entity/Core/RenditionDefinition.php | 19 ++++ .../Workspace/WorkspaceDuplicateManager.php | 1 + databox/indexer/package.json | 3 +- databox/indexer/src/alternateUrl.ts | 4 +- databox/indexer/src/command/commandUtil.ts | 8 ++ databox/indexer/src/command/index.ts | 5 +- databox/indexer/src/command/watch.ts | 5 +- databox/indexer/src/configLoader.ts | 9 +- databox/indexer/src/console.ts | 9 +- databox/indexer/src/databox/client.ts | 4 +- .../src/handlers/phraseanet/indexer.ts | 9 +- .../indexer/src/handlers/phraseanet/shared.ts | 4 +- databox/indexer/src/lib/axios.ts | 26 +++++- databox/indexer/src/lib/logger.ts | 24 ++++- databox/indexer/src/types.ts | 3 + databox/indexer/tsconfig.check.json | 30 ++++++ .../src/components/SessionExpireContainer.tsx | 4 +- 21 files changed, 358 insertions(+), 25 deletions(-) create mode 100644 databox/api/migrations/Version20231221125408.php create mode 100644 databox/api/src/Api/InputTransformer/RenditionDefinitionInputTransformer.php create mode 100644 databox/api/src/Api/Model/Input/RenditionDefinitionInput.php create mode 100644 databox/indexer/src/command/commandUtil.ts create mode 100644 databox/indexer/src/types.ts create mode 100644 databox/indexer/tsconfig.check.json diff --git a/databox/api/migrations/Version20231221125408.php b/databox/api/migrations/Version20231221125408.php new file mode 100644 index 000000000..01c6d6638 --- /dev/null +++ b/databox/api/migrations/Version20231221125408.php @@ -0,0 +1,34 @@ +addSql('ALTER TABLE rendition_definition ADD key VARCHAR(150) DEFAULT NULL'); + $this->addSql('CREATE UNIQUE INDEX uniq_rend_def_ws_key ON rendition_definition (workspace_id, key)'); + } + + public function down(Schema $schema): void + { + // this down() migration is auto-generated, please modify it to your needs + $this->addSql('CREATE SCHEMA public'); + $this->addSql('DROP INDEX uniq_rend_def_ws_key'); + $this->addSql('ALTER TABLE rendition_definition DROP key'); + } +} diff --git a/databox/api/src/Api/InputTransformer/RenditionDefinitionInputTransformer.php b/databox/api/src/Api/InputTransformer/RenditionDefinitionInputTransformer.php new file mode 100644 index 000000000..e49e1121d --- /dev/null +++ b/databox/api/src/Api/InputTransformer/RenditionDefinitionInputTransformer.php @@ -0,0 +1,92 @@ +workspace) { + $workspace = $data->workspace; + } + + if ($isNew) { + if (!$workspace instanceof Workspace) { + throw new BadRequestHttpException('Missing workspace'); + } + + if ($data->key) { + $rendDef = $this->em->getRepository(RenditionDefinition::class) + ->findOneBy([ + 'key' => $data->key, + 'workspace' => $workspace->getId() + ]); + + if ($rendDef) { + $isNew = false; + $object = $rendDef; + } + } + } + + if ($isNew) { + $object->setWorkspace($workspace); + $object->setKey($data->key); + } + + if (null !== $data->name) { + $object->setName($data->name); + } + if (null !== $data->class) { + $object->setClass($data->class); + } + + if (null !== $data->download) { + $object->setDownload($data->download); + } + if (null !== $data->pickSourceFile) { + $object->setPickSourceFile($data->pickSourceFile); + } + if (null !== $data->useAsOriginal) { + $object->setUseAsOriginal($data->useAsOriginal); + } + if (null !== $data->useAsPreview) { + $object->setUseAsPreview($data->useAsPreview); + } + if (null !== $data->useAsThumbnail) { + $object->setUseAsThumbnail($data->useAsThumbnail); + } + if (null !== $data->useAsThumbnailActive) { + $object->setUseAsThumbnailActive($data->useAsThumbnailActive); + } + if (null !== $data->definition) { + $object->setDefinition($data->definition); + } + if (null !== $data->priority) { + $object->setPriority($data->priority); + } + + return $object; + } +} diff --git a/databox/api/src/Api/Model/Input/RenditionDefinitionInput.php b/databox/api/src/Api/Model/Input/RenditionDefinitionInput.php new file mode 100644 index 000000000..cc851bf08 --- /dev/null +++ b/databox/api/src/Api/Model/Input/RenditionDefinitionInput.php @@ -0,0 +1,85 @@ + [RenditionDefinition::GROUP_WRITE], ], + input: RenditionDefinitionInput::class, order: ['priority' => 'DESC'], provider: RenditionDefinitionCollectionProvider::class, )] #[ORM\Table] #[ORM\Index(columns: ['workspace_id', 'name'], name: 'rend_def_ws_name')] +#[ORM\UniqueConstraint(name: 'uniq_rend_def_ws_key', columns: ['workspace_id', 'key'])] #[ORM\Entity] class RenditionDefinition extends AbstractUuidEntity implements \Stringable { @@ -90,6 +93,12 @@ class RenditionDefinition extends AbstractUuidEntity implements \Stringable #[Groups(['_'])] protected ?Workspace $workspace = null; + /** + * Unique key by workspace. Used to prevent duplicates. + */ + #[ORM\Column(type: Types::STRING, length: 150, nullable: true)] + private ?string $key = null; + #[Groups([RenditionDefinition::GROUP_LIST, RenditionDefinition::GROUP_READ, RenditionDefinition::GROUP_WRITE])] #[ORM\Column(type: Types::STRING, length: 80)] private ?string $name = null; @@ -255,4 +264,14 @@ public function setPickSourceFile(bool $pickSourceFile): void { $this->pickSourceFile = $pickSourceFile; } + + public function getKey(): ?string + { + return $this->key; + } + + public function setKey(?string $key): void + { + $this->key = $key; + } } diff --git a/databox/api/src/Workspace/WorkspaceDuplicateManager.php b/databox/api/src/Workspace/WorkspaceDuplicateManager.php index af335f39e..6d659e67f 100644 --- a/databox/api/src/Workspace/WorkspaceDuplicateManager.php +++ b/databox/api/src/Workspace/WorkspaceDuplicateManager.php @@ -62,6 +62,7 @@ private function copyRenditionDefinitions(Workspace $from, Workspace $to): void $i->setWorkspace($to); $i->setClass($classMap[$item->getClass()->getId()]); $i->setPriority($item->getPriority()); + $i->setKey($item->getKey()); $i->setUseAsOriginal($item->isUseAsOriginal()); $i->setUseAsPreview($item->isUseAsPreview()); $i->setUseAsThumbnail($item->isUseAsThumbnail()); diff --git a/databox/indexer/package.json b/databox/indexer/package.json index abd6034e4..045be7800 100644 --- a/databox/indexer/package.json +++ b/databox/indexer/package.json @@ -10,7 +10,8 @@ "scripts": { "console": "node dist/console.mjs", "dev": "nodemon", - "build": "rimraf ./dist && vite build", + "validate": "tsc -p ./tsconfig.check.json", + "build": "pnpm validate && rimraf ./dist && vite build", "test": "jest", "sync-databox-types": "generate-api-platform-client --generator typescript http://databox-api src/", "format": "prettier --ignore-path .gitignore --write \"**/*.+(js|ts|json|cjs|tsx|jsx)\"", diff --git a/databox/indexer/src/alternateUrl.ts b/databox/indexer/src/alternateUrl.ts index 759fba95f..e5c9f4a1b 100644 --- a/databox/indexer/src/alternateUrl.ts +++ b/databox/indexer/src/alternateUrl.ts @@ -18,8 +18,8 @@ export function getAlternateUrls( return alternateUrls.map((c): AlternateUrl => { return { type: c.name, - url: c.pathPattern.replace(/\${(.+)}/g, (_m, m1) => { - return dict[m1]; + url: c.pathPattern.replace(/\${(.+)}/g, (_m, m1: string) => { + return dict[m1 as keyof typeof dict] as string; }), }; }); diff --git a/databox/indexer/src/command/commandUtil.ts b/databox/indexer/src/command/commandUtil.ts new file mode 100644 index 000000000..813a58f49 --- /dev/null +++ b/databox/indexer/src/command/commandUtil.ts @@ -0,0 +1,8 @@ +import {setLogLevel} from "../lib/logger"; +import {CommandCommonOptions} from "../types"; + +export function applyCommonOptions(opts: O): void { + if (opts.debug) { + setLogLevel('debug'); + } +} diff --git a/databox/indexer/src/command/index.ts b/databox/indexer/src/command/index.ts index e35d2511b..f9a3b1f38 100644 --- a/databox/indexer/src/command/index.ts +++ b/databox/indexer/src/command/index.ts @@ -4,15 +4,18 @@ import {indexers} from '../indexers.js'; import {getLocation} from '../locations.js'; import {consume} from '../databox/entrypoint.js'; import {runServer} from '../server'; +import {CommandCommonOptions} from "../types"; +import {applyCommonOptions} from "./commandUtil"; export type IndexOptions = { createNewWorkspace?: boolean; -}; +} & CommandCommonOptions; export default async function indexCommand( locationName: string, options: IndexOptions ) { + applyCommonOptions(options); const location = getLocation(locationName); const databoxLogger = createLogger('databox'); diff --git a/databox/indexer/src/command/watch.ts b/databox/indexer/src/command/watch.ts index bbafdf6d7..b13f132c9 100644 --- a/databox/indexer/src/command/watch.ts +++ b/databox/indexer/src/command/watch.ts @@ -4,10 +4,13 @@ import {runServer} from '../server'; import {IndexLocation} from '../types/config'; import {getConfig} from '../configLoader'; import {watchers} from '../watchers'; +import {CommandCommonOptions} from "../types"; +import {applyCommonOptions} from "./commandUtil"; -export type WatchOptions = {}; +export type WatchOptions = {} & CommandCommonOptions; export default async function watchCommand(options: WatchOptions) { + applyCommonOptions(options); const mainLogger = createLogger('app'); const databoxLogger = createLogger('databox'); diff --git a/databox/indexer/src/configLoader.ts b/databox/indexer/src/configLoader.ts index 1f409b56f..4acee3f11 100644 --- a/databox/indexer/src/configLoader.ts +++ b/databox/indexer/src/configLoader.ts @@ -10,7 +10,7 @@ function loadConfig(): object { } function replaceEnv(str: string): string | boolean | number | undefined { - let transform; + let transform: string | undefined; let hasEnv = false; let result: string | undefined = str.replace( /%env\(([^^)]+)\)%/g, @@ -62,8 +62,8 @@ function parseConfig(config: any): any { if (Array.isArray(config)) { return config.map(parseConfig); } else { - const sub = {}; - Object.keys(config).forEach(k => { + const sub: Record = {}; + Object.keys(config).forEach((k: string) => { sub[k] = parseConfig(config[k]); }); return sub; @@ -84,10 +84,11 @@ export function getConfig( let p = root; for (let i = 0; i < parts.length; ++i) { - const k = parts[i]; + const k = parts[i] as string; if (!p.hasOwnProperty(k)) { return defaultValue; } + // @ts-expect-error any p = p[parts[i]]; } diff --git a/databox/indexer/src/console.ts b/databox/indexer/src/console.ts index 21cfabd78..020e8c2e5 100644 --- a/databox/indexer/src/console.ts +++ b/databox/indexer/src/console.ts @@ -1,11 +1,14 @@ -import {Command} from 'commander'; +import {Command, Option} from 'commander'; import indexCommand from './command/index.js'; import listCommand from './command/list'; import watchCommand from './command/watch'; const program = new Command(); -program.name('indexer').description('Databox Indexer').version('1.0.0'); +program.name('console').description('Databox Indexer').version('1.0.0'); + +const debugOption = new Option('--debug', 'Debug mode') + .default(false); program .command('index') @@ -16,12 +19,14 @@ program 'Remove existing workspace and create a new empty one', false ) + .addOption(debugOption) .action(indexCommand); program .command('watch') .description('Watch locations') .option('-l, --location', 'List locations to watch', false) + .addOption(debugOption) .action(watchCommand); program.command('list').description('List locations').action(listCommand); diff --git a/databox/indexer/src/databox/client.ts b/databox/indexer/src/databox/client.ts index 593e5dd3c..02728b1bd 100644 --- a/databox/indexer/src/databox/client.ts +++ b/databox/indexer/src/databox/client.ts @@ -191,7 +191,7 @@ export class DataboxClient { return r; } - async createRenditionClass(data): Promise { + async createRenditionClass(data: object): Promise { const res = await this.client.post(`/rendition-classes`, data); return res.data.id; @@ -207,7 +207,7 @@ export class DataboxClient { return res.data['hydra:member']; } - async createRenditionDefinition(data): Promise { + async createRenditionDefinition(data: object): Promise { await this.client.post(`/rendition-definitions`, data); } diff --git a/databox/indexer/src/handlers/phraseanet/indexer.ts b/databox/indexer/src/handlers/phraseanet/indexer.ts index b5089b8b0..719c88ae2 100644 --- a/databox/indexer/src/handlers/phraseanet/indexer.ts +++ b/databox/indexer/src/handlers/phraseanet/indexer.ts @@ -59,15 +59,17 @@ export const phraseanetIndexer: IndexIterator = await client.getMetaStruct(dm.databoxId) ); for (const m of metaStructure) { - logger.debug(`Creating "${m.name}" attribute definition`); + logger.info(`Creating "${m.name}" attribute definition`); const id = m.id.toString(); if (!attrClassIndex[defaultPublicClass]) { + const name = 'Phraseanet Public'; + logger.info(`Creating "${name}" attribute class`); attrClassIndex[defaultPublicClass] = await databoxClient.createAttributeClass( defaultPublicClass, { - name: 'Phraseanet Public', + name, public: true, editable: true, workspace: `/workspaces/${workspaceId}`, @@ -104,7 +106,7 @@ export const phraseanetIndexer: IndexIterator = const subDefs = await client.getSubDefinitions(dm.databoxId); for (const sd of subDefs) { if (!classIndex[sd.class]) { - logger.debug(`Creating rendition class "${sd.class}" `); + logger.info(`Creating rendition class "${sd.class}" `); classIndex[sd.class] = await databoxClient.createRenditionClass({ name: sd.class, @@ -117,6 +119,7 @@ export const phraseanetIndexer: IndexIterator = ); await databoxClient.createRenditionDefinition({ name: sd.name, + key: `${sd.name}_${sd.type ?? ''}`, class: `/rendition-classes/${classIndex[sd.class]}`, useAsOriginal: sd.name === 'document', useAsPreview: sd.name === 'preview', diff --git a/databox/indexer/src/handlers/phraseanet/shared.ts b/databox/indexer/src/handlers/phraseanet/shared.ts index 18e3a7124..e0800de01 100644 --- a/databox/indexer/src/handlers/phraseanet/shared.ts +++ b/databox/indexer/src/handlers/phraseanet/shared.ts @@ -7,7 +7,7 @@ import { RenditionInput, } from '../../databox/types'; -const renditionDefinitionMapping = { +const renditionDefinitionMapping: Record = { document: 'original', }; const renditionDefinitionBlacklist = ['original']; @@ -82,6 +82,6 @@ export function createAsset( }; } -export const attributeTypesEquivalence = { +export const attributeTypesEquivalence: Record = { string: 'text', }; diff --git a/databox/indexer/src/lib/axios.ts b/databox/indexer/src/lib/axios.ts index db3ef3744..b84be7fcb 100644 --- a/databox/indexer/src/lib/axios.ts +++ b/databox/indexer/src/lib/axios.ts @@ -87,8 +87,12 @@ export function createHttpClient({ }; } + logger.debug( + `Error response headers (${error.config?.url}): ` + + JSON.stringify(error.response.headers, undefined, 2) + ); logger.error( - `Error response (${error.config.url}): ` + + `Error response (${error.config?.url}): ` + JSON.stringify(filtered, undefined, 2) ); } @@ -97,5 +101,25 @@ export function createHttpClient({ } ); + const obfuscate = (str: string) => str.replace(/([\w._-]{5})[\w._-]{30,}/g, '$1***'); + + client.interceptors.request.use( + config => { + if (!config) { + return config; + } + + logger.debug(`${config.method?.toUpperCase()} ${config.url} +${obfuscate(JSON.stringify(config.headers, null, 2))}${ + config.data ? `\n${obfuscate(JSON.stringify(config.data, null, 2))}` : '' + }`); + + return config; + }, + error => { + return Promise.reject(error); + } + ); + return client; } diff --git a/databox/indexer/src/lib/logger.ts b/databox/indexer/src/lib/logger.ts index ebbc97bf5..41adeca1c 100644 --- a/databox/indexer/src/lib/logger.ts +++ b/databox/indexer/src/lib/logger.ts @@ -10,11 +10,31 @@ const myFormat = printf(({context, level, message, timestamp}) => { return `${timestamp} ${context}.${level.toUpperCase()}: ${message}`; }); +type LogLevel = "debug" | "warn" | "info" | "error"; + +const loggerConfig: { + level: LogLevel; +} = { + level: 'info' +}; + +const loggers: Logger[] = []; + +export function setLogLevel(level: LogLevel): void { + loggerConfig.level = level; + + loggers.forEach(l => l.level = level); +} + export function createLogger(context: string): Logger { - return winstonCreateLogger({ - level: 'debug', + const l = winstonCreateLogger({ + level: loggerConfig.level, format: combine(timestamp(), myFormat), defaultMeta: {context}, transports: [new transports.Console()], }); + + loggers.push(l); + + return l; } diff --git a/databox/indexer/src/types.ts b/databox/indexer/src/types.ts new file mode 100644 index 000000000..2e15e5133 --- /dev/null +++ b/databox/indexer/src/types.ts @@ -0,0 +1,3 @@ +export type CommandCommonOptions = { + debug: boolean; +} diff --git a/databox/indexer/tsconfig.check.json b/databox/indexer/tsconfig.check.json new file mode 100644 index 000000000..d5db6522d --- /dev/null +++ b/databox/indexer/tsconfig.check.json @@ -0,0 +1,30 @@ +{ + "compilerOptions": { + "target": "ES2020", + "useDefineForClassFields": true, + "lib": ["ES2020", "DOM", "DOM.Iterable"], + "module": "ESNext", + "skipLibCheck": true, + + /* Bundler mode */ + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "resolveJsonModule": true, + "isolatedModules": true, + "noEmit": true, + + /* Linting */ + "strict": true, + "noUnusedLocals": true, + "noUnusedParameters": true, + "noFallthroughCasesInSwitch": true, + "allowJs": true, + + "types": [ + "node" + ] + }, + "include": [ + "src/*" + ] +} diff --git a/lib/js/react-auth/src/components/SessionExpireContainer.tsx b/lib/js/react-auth/src/components/SessionExpireContainer.tsx index 8c0f72519..7f2f6af35 100644 --- a/lib/js/react-auth/src/components/SessionExpireContainer.tsx +++ b/lib/js/react-auth/src/components/SessionExpireContainer.tsx @@ -18,9 +18,9 @@ export default function SessionExpireContainer({}: Props) { }, []); let delay: number | undefined = undefined; - if (tokens) { + if (tokens?.refreshExpiresAt) { const beforeEnd = 60000; - const end = tokens.expiresAt * 1000 - new Date().getTime(); + const end = tokens.refreshExpiresAt * 1000 - new Date().getTime(); delay = Math.max(end - beforeEnd, 5000) }