feat(backend): add categories database tables and es indices

apps/backend/src/migrations/0016-add-apps-categories.ts

@@ -0,0 +1,48 @@
+import type { Kysely } from 'kysely';
+
+import {
+  addArchivedAtColumns,
+  addIdColumn,
+  addTimestampColumns,
+} from './utils';
+
+export async function up(db: Kysely<any>): Promise<void> {
+  await db.schema
+    .createTable('apps_categories')
+    .$call(addIdColumn)
+    .$call(addTimestampColumns)
+    .$call(addArchivedAtColumns)
+    .addColumn('name', 'text', col => col.notNull())
+    .addColumn('icon', 'text', col => col.notNull())
+    .addColumn('description', 'text')
+    .addColumn('parent_category_id', 'integer', col => col.references('apps_categories.id'))
+    .execute();
+
+  await db.schema
+    .alterTable('apps')
+    .addColumn('category_id', 'integer', col => col.references('apps_categories.id'))
+    .execute();
+
+  const { id } = await db
+    .insertInto('apps_categories')
+    .values([
+      { name: 'Other', icon: 'ellipsis', description: 'Other apps that do not fit into any other category' },
+    ])
+    .returning('id')
+    .executeTakeFirstOrThrow();
+
+  await db
+    .updateTable('apps')
+    .set({ category_id: id })
+    .execute();
+
+  await db.schema
+    .alterTable('apps')
+    .alterColumn('category_id', col => col.setNotNull())
+    .execute();
+}
+
+export async function down(db: Kysely<any>): Promise<void> {
+  await db.schema.alterTable('apps').dropColumn('category_id').execute();
+  await db.schema.dropTable('apps_categories').execute();
+}

apps/backend/src/migrations/index.ts

@@ -14,6 +14,7 @@ import * as addCreatorToMessagesTable from './0012-add-creator-to-messages-table
 import * as addRepliedMessageId from './0013-add-replied-message-id';
 import * as addLastSummarizedMessage from './0014-add-last-summarized-message';
 import * as addAttachedAppIdToMessages from './0015-add-attached-app-id-to-messages';
+import * as addAppsCategories from './0016-add-apps-categories';
 
 export const DB_MIGRATIONS = {
   '0000-add-users-tables': addUsersTables,
@@ -32,4 +33,5 @@ export const DB_MIGRATIONS = {
   '0013-add-replied-message-id': addRepliedMessageId,
   '0014-add-last-summarized-message': addLastSummarizedMessage,
   '0015-add-attached-app-id-to-messages': addAttachedAppIdToMessages,
+  '0016-add-apps-categories': addAppsCategories,
 };

apps/backend/src/modules/apps-categories/apps-categories.firewall.ts

@@ -0,0 +1,46 @@
+import { flow } from 'fp-ts/lib/function';
+
+import type { SdkJwtTokenT } from '@llm/sdk';
+
+import { AuthFirewallService } from '~/modules/auth/firewall';
+
+import type { AppsCategoriesService } from './apps-categories.service';
+
+export class AppsCategoriesFirewall extends AuthFirewallService {
+  constructor(
+    jwt: SdkJwtTokenT,
+    private readonly appsService: AppsCategoriesService,
+  ) {
+    super(jwt);
+  }
+
+  unarchive = flow(
+    this.appsService.unarchive,
+    this.tryTEIfUser.is.root,
+  );
+
+  archive = flow(
+    this.appsService.archive,
+    this.tryTEIfUser.is.root,
+  );
+
+  update = flow(
+    this.appsService.update,
+    this.tryTEIfUser.is.root,
+  );
+
+  create = flow(
+    this.appsService.create,
+    this.tryTEIfUser.is.root,
+  );
+
+  search = flow(
+    this.appsService.search,
+    this.tryTEIfUser.is.root,
+  );
+
+  get = flow(
+    this.appsService.get,
+    this.tryTEIfUser.is.root,
+  );
+}

apps/backend/src/modules/apps-categories/apps-categories.repo.ts

@@ -0,0 +1,66 @@
+import camelcaseKeys from 'camelcase-keys';
+import { array as A, taskEither as TE } from 'fp-ts';
+import { pipe } from 'fp-ts/lib/function';
+import { injectable } from 'tsyringe';
+
+import {
+  createArchiveRecordQuery,
+  createArchiveRecordsQuery,
+  createDatabaseRepo,
+  createUnarchiveRecordQuery,
+  createUnarchiveRecordsQuery,
+  DatabaseError,
+  TableId,
+  TransactionalAttrs,
+  tryReuseTransactionOrSkip,
+} from '~/modules/database';
+
+import { AppTableRowWithRelations } from './apps-categories.tables';
+
+@injectable()
+export class AppsCategoriesRepo extends createDatabaseRepo('apps_categories') {
+  archive = createArchiveRecordQuery(this.queryFactoryAttrs);
+
+  archiveRecords = createArchiveRecordsQuery(this.queryFactoryAttrs);
+
+  unarchive = createUnarchiveRecordQuery(this.queryFactoryAttrs);
+
+  unarchiveRecords = createUnarchiveRecordsQuery(this.queryFactoryAttrs);
+
+  findWithRelationsByIds = ({ forwardTransaction, ids }: TransactionalAttrs<{ ids: TableId[]; }>) => {
+    const transaction = tryReuseTransactionOrSkip({ db: this.db, forwardTransaction });
+
+    return pipe(
+      transaction(
+        async qb =>
+          qb
+            .selectFrom(`${this.table} as category`)
+            .where('category.id', 'in', ids)
+            .leftJoin('apps_categories as parent_categories', 'parent_categories.id', 'category.parent_category_id')
+            .selectAll('category')
+            .select([
+              'parent_categories.id as parent_category_id',
+              'parent_categories.name as parent_category_name',
+            ])
+            .limit(ids.length)
+            .execute(),
+      ),
+      DatabaseError.tryTask,
+      TE.map(
+        A.map(({
+          parent_category_id: parentId,
+          parent_category_name: parentName,
+          ...item
+        }): AppTableRowWithRelations => ({
+          ...camelcaseKeys(item),
+          parentCategory: parentId
+            ? {
+                id: parentId,
+                name: parentName || '',
+              }
+            : null,
+        })),
+      ),
+    );
+  };
+}

apps/backend/src/modules/apps-categories/apps-categories.service.ts

@@ -0,0 +1,83 @@
+import { taskEither as TE } from 'fp-ts';
+import { pipe } from 'fp-ts/lib/function';
+import { inject, injectable } from 'tsyringe';
+
+import type {
+  SdkCreateAppCategoryInputT,
+  SdkJwtTokenT,
+  SdkTableRowIdT,
+  SdkUpdateAppInputT,
+} from '@llm/sdk';
+
+import {
+  asyncIteratorToVoidPromise,
+  runTaskAsVoid,
+  tapAsyncIterator,
+  tryOrThrowTE,
+} from '@llm/commons';
+
+import type { WithAuthFirewall } from '../auth';
+import type { TableId, TableRowWithId } from '../database';
+
+import { AppsCategoriesFirewall } from './apps-categories.firewall';
+import { AppsCategoriesRepo } from './apps-categories.repo';
+import { AppsCategoriesEsIndexRepo, AppsCategoriesEsSearchRepo } from './elasticsearch';
+
+@injectable()
+export class AppsCategoriesService implements WithAuthFirewall<AppsCategoriesFirewall> {
+  constructor(
+    @inject(AppsCategoriesRepo) private readonly repo: AppsCategoriesRepo,
+    @inject(AppsCategoriesEsSearchRepo) private readonly esSearchRepo: AppsCategoriesEsSearchRepo,
+    @inject(AppsCategoriesEsIndexRepo) private readonly esIndexRepo: AppsCategoriesEsIndexRepo,
+  ) {}
+
+  asUser = (jwt: SdkJwtTokenT) => new AppsCategoriesFirewall(jwt, this);
+
+  get = this.esSearchRepo.get;
+
+  archiveSeqStream = (stream: AsyncIterableIterator<TableId[]>) => async () =>
+    pipe(
+      stream,
+      tapAsyncIterator<TableId[], void>(async ids =>
+        pipe(
+          this.repo.archiveRecords({
+            where: [
+              ['id', 'in', ids],
+              ['archived', '=', false],
+            ],
+          }),
+          TE.tap(() => this.esIndexRepo.findAndIndexDocumentsByIds(ids)),
+          tryOrThrowTE,
+          runTaskAsVoid,
+        ),
+      ),
+      asyncIteratorToVoidPromise,
+    );
+
+  unarchive = (id: SdkTableRowIdT) => pipe(
+    this.repo.unarchive({ id }),
+    TE.tap(() => this.esIndexRepo.findAndIndexDocumentById(id)),
+  );
+
+  archive = (id: SdkTableRowIdT) => pipe(
+    this.repo.archive({ id }),
+    TE.tap(() => this.esIndexRepo.findAndIndexDocumentById(id)),
+  );
+
+  search = this.esSearchRepo.search;
+
+  create = ({ parentCategory, ...values }: SdkCreateAppCategoryInputT) => pipe(
+    this.repo.create({
+      value: {
+        ...values,
+        parentCategoryId: parentCategory?.id || null,
+      },
+    }),
+    TE.tap(({ id }) => this.esIndexRepo.findAndIndexDocumentById(id)),
+  );
+
+  update = ({ id, ...value }: SdkUpdateAppInputT & TableRowWithId) => pipe(
+    this.repo.update({ id, value }),
+    TE.tap(() => this.esIndexRepo.findAndIndexDocumentById(id)),
+  );
+}

apps/backend/src/modules/apps-categories/apps-categories.tables.ts

@@ -0,0 +1,21 @@
+import type {
+  NormalizeSelectTableRow,
+  TableId,
+  TableRowWithIdName,
+  TableWithArchivedAtColumn,
+  TableWithDefaultColumns,
+} from '../database';
+
+export type AppsCategoriesTable = TableWithDefaultColumns &
+  TableWithArchivedAtColumn & {
+    name: string;
+    description: string | null;
+    icon: string;
+    parent_category_id: TableId | null;
+  };
+
+export type AppCategoryTableRow = NormalizeSelectTableRow<AppsCategoriesTable>;
+
+export type AppTableRowWithRelations = Omit<AppCategoryTableRow, 'parentCategoryId'> & {
+  parentCategory: TableRowWithIdName | null;
+};

apps/backend/src/modules/apps-categories/elasticsearch/apps-categories-es-index.repo.ts

@@ -0,0 +1,73 @@
+import { array as A, taskEither as TE } from 'fp-ts';
+import { pipe } from 'fp-ts/lib/function';
+import snakecaseKeys from 'snakecase-keys';
+import { inject, injectable } from 'tsyringe';
+
+import { tryOrThrowTE } from '@llm/commons';
+import {
+  createArchivedRecordMappings,
+  createAutocompleteFieldAnalyzeSettings,
+  createBaseAutocompleteFieldMappings,
+  createBaseDatedRecordMappings,
+  createElasticsearchIndexRepo,
+  createIdNameObjectMapping,
+  ElasticsearchRepo,
+  type EsDocument,
+} from '~/modules/elasticsearch';
+
+import type { AppTableRowWithRelations } from '../apps-categories.tables';
+
+import { AppsCategoriesRepo } from '../apps-categories.repo';
+
+const AppsAbstractEsIndexRepo = createElasticsearchIndexRepo({
+  indexName: 'dashboard-apps-categories',
+  schema: {
+    mappings: {
+      dynamic: false,
+      properties: {
+        ...createBaseDatedRecordMappings(),
+        ...createBaseAutocompleteFieldMappings(),
+        ...createArchivedRecordMappings(),
+        parent_category: createIdNameObjectMapping(),
+        description: {
+          type: 'text',
+          analyzer: 'folded_lowercase_analyzer',
+        },
+      },
+    },
+    settings: {
+      'index.number_of_replicas': 1,
+      'analysis': createAutocompleteFieldAnalyzeSettings(),
+    },
+  },
+});
+
+export type AppsEsDocument = EsDocument<AppTableRowWithRelations>;
+
+@injectable()
+export class AppsCategoriesEsIndexRepo extends AppsAbstractEsIndexRepo<AppsEsDocument> {
+  constructor(
+    @inject(ElasticsearchRepo) elasticsearchRepo: ElasticsearchRepo,
+    @inject(AppsCategoriesRepo) private readonly appsRepo: AppsCategoriesRepo,
+  ) {
+    super(elasticsearchRepo);
+  }
+
+  protected async findEntities(ids: number[]): Promise<AppsEsDocument[]> {
+    return pipe(
+      this.appsRepo.findWithRelationsByIds({ ids }),
+      TE.map(
+        A.map(entity => ({
+          ...snakecaseKeys(entity, { deep: true }),
+          _id: String(entity.id),
+        })),
+      ),
+      tryOrThrowTE,
+    )();
+  }
+
+  protected createAllEntitiesIdsIterator = () =>
+    this.appsRepo.createIdsIterator({
+      chunkSize: 100,
+    });
+}