Merge pull request #122 from DashHub-ai/feature/summarize-projects

Add automated projects summaries

apps/admin/src/i18n/packs/i18n-lang-en.ts

@@ -127,6 +127,7 @@ export const I18N_PACK_EN = deepmerge(I18N_FORWARDED_EN_PACK, {
           description: {
             label: 'Description',
             placeholder: 'Enter project description',
+            generated: 'Generated by AI',
           },
           organization: {
             label: 'Organization',

apps/admin/src/i18n/packs/i18n-lang-pl.ts

@@ -129,6 +129,7 @@ export const I18N_PACK_PL: I18nLangPack = deepmerge(I18N_FORWARDED_PL_PACK, {
           description: {
             label: 'Opis',
             placeholder: 'Wpisz opis projektu',
+            generated: 'Wygenerowany przez AI',
           },
           organization: {
             label: 'Wybierz organizację',

apps/admin/src/modules/projects/form/shared/project-shared-form-fields.tsx

@@ -1,15 +1,15 @@
 import { controlled, useFormValidatorMessages, type ValidationErrorsListProps } from '@under-control/forms';
 
-import type { SdkProjectT } from '@llm/sdk';
+import type { SdkUpdateProjectInputT } from '@llm/sdk';
 
-import { FormField, Input, TextArea } from '@llm/ui';
+import { Checkbox, FormField, Input, TextArea } from '@llm/ui';
 import { useI18n } from '~/i18n';
 
-type Value = Pick<SdkProjectT, 'name' | 'description'>;
+type Value = Pick<SdkUpdateProjectInputT, 'name' | 'summary'>;
 
 type Props = ValidationErrorsListProps<Value>;
 
-export const ProjectSharedFormFields = controlled<Value, Props>(({ errors, control: { bind } }) => {
+export const ProjectSharedFormFields = controlled<Value, Props>(({ errors, control: { bind, value } }) => {
   const t = useI18n().pack.modules.projects.form;
   const validation = useFormValidatorMessages({ errors });
 
@@ -31,13 +31,33 @@ export const ProjectSharedFormFields = controlled<Value, Props>(({ errors, contr
       <FormField
         className="uk-margin"
         label={t.fields.description.label}
-        {...validation.extract('description')}
+        {...validation.extract('summary.content.value')}
       >
         <TextArea
           name="description"
           placeholder={t.fields.description.placeholder}
-          {...bind.path('description')}
+          {...bind.path('summary.content.value', { input: value => value ?? '' })}
+          disabled={value.summary.content.generated}
         />
+
+        <Checkbox
+          className="block pt-2 uk-text-small"
+          {...bind.path('summary.content.generated', {
+            relatedInputs: ({ newGlobalValue, newControlValue }) => ({
+              ...newGlobalValue,
+              summary: {
+                ...newGlobalValue.summary,
+                content: (
+                  newControlValue
+                    ? { generated: true, value: null }
+                    : { generated: false, value: '' }
+                ),
+              },
+            }),
+          })}
+        >
+          {t.fields.description.generated}
+        </Checkbox>
       </FormField>
     </>
   );

apps/admin/src/modules/projects/form/update/project-update-form-modal.tsx

@@ -32,7 +32,16 @@ export function ProjectUpdateFormModal(
 ) {
   const t = useI18n().pack.modules.projects.form;
   const { handleSubmitEvent, validator, submitState, bind } = useProjectUpdateForm({
-    defaultValue: project,
+    defaultValue: {
+      ...project,
+      summary: {
+        content: (
+          project.summary.content.generated
+            ? { generated: true, value: null }
+            : { generated: false, value: project.summary.content.value! }
+        ),
+      },
+    },
     onAfterSubmit,
   });
 

apps/admin/src/modules/projects/table/projects-table-container.tsx

@@ -64,7 +64,9 @@ export function ProjectsTableContainer() {
       createModal.showAsOptional({
         defaultValue: {
           name: '',
-          description: null,
+          summary: {
+            content: { generated: true, value: null },
+          },
           organization: createFakeSelectItem(),
         },
       }),

apps/admin/src/modules/projects/table/projects-table-row.tsx

@@ -22,7 +22,7 @@ export function ProjectsTableRow({ item, onUpdated }: Props) {
     <tr>
       <td>{item.id}</td>
       <td>{item.name}</td>
-      <td>{item.description || '-'}</td>
+      <td>{item.summary.content.value || '-'}</td>
       <td>
         <Link
           className="uk-link"

apps/backend/package.json

@@ -8,6 +8,7 @@
     "db:migrate": "node dist/run-migrations.cli.mjs",
     "es:reindex:all": "node dist/run-reindex-all.cli.mjs",
     "chats:summarize:all": "node dist/run-summarize-chats.cli.mjs",
+    "projects:summarize:all": "node dist/run-summarize-projects.cli.mjs",
     "dev": "npm run clean && concurrently --kill-others \"vite build --watch --mode production\" \"npm run build:wait && nodemon --stack-trace-limit=1000 --enable-source-maps ./dist/index.mjs\"",
     "lint": "eslint . --config ./eslint.config.mjs",
     "clean": "rm -rf ./dist 2>/dev/null",

apps/backend/src/cli/run-summarize-projects.cli.ts

@@ -0,0 +1,21 @@
+import 'reflect-metadata';
+
+import { pipe } from 'fp-ts/lib/function';
+import { container } from 'tsyringe';
+
+import { runTaskAsVoid, tapTaskEitherTE, tryOrThrowTE } from '@llm/commons';
+import { DatabaseConnectionRepo, ElasticsearchRepo } from '~/modules';
+import { ProjectsSummariesService } from '~/modules/projects-summaries';
+
+const databaseConnectionRepo = container.resolve(DatabaseConnectionRepo);
+const esConnection = container.resolve(ElasticsearchRepo);
+
+const summariesService = container.resolve(ProjectsSummariesService);
+
+void pipe(
+  summariesService.summarizeAllProjects(),
+  tapTaskEitherTE(() => esConnection.close),
+  tapTaskEitherTE(() => databaseConnectionRepo.close),
+  tryOrThrowTE,
+  runTaskAsVoid,
+);

apps/backend/src/migrations/0009-add-chats-tables.ts

@@ -1,7 +1,11 @@
 import { type Kysely, sql } from 'kysely';
 
-import { addArchivedAtColumns, addIdColumn, addTimestampColumns } from './utils';
-import { addUuidColumn } from './utils/add-uuid';
+import {
+  addArchivedAtColumns,
+  addIdColumn,
+  addTimestampColumns,
+  addUuidColumn,
+} from './utils';
 
 export async function up(db: Kysely<any>): Promise<void> {
   await db.schema

apps/backend/src/migrations/0028-add-projects-summarizes.ts

@@ -0,0 +1,41 @@
+import type { Kysely } from 'kysely';
+
+import {
+  addAIGeneratedColumns,
+  addIdColumn,
+  addTimestampColumns,
+} from './utils';
+
+export async function up(db: Kysely<any>): Promise<void> {
+  await db.schema
+    .alterTable('projects')
+    .dropColumn('description')
+    .execute();
+
+  await db.schema
+    .createTable('projects_summaries')
+    .$call(addIdColumn)
+    .$call(addTimestampColumns)
+    .$call(addAIGeneratedColumns('content'))
+    .addColumn('project_id', 'integer', col => col.notNull().references('projects.id').onDelete('restrict'))
+    .addColumn('last_summarized_message_id', 'uuid', col => col.references('messages.id').onDelete('restrict'))
+    .execute();
+
+  await db
+    .insertInto('projects_summaries')
+    .columns(['project_id'])
+    .expression(
+      db.selectFrom('projects')
+        .select(['id as project_id']),
+    )
+    .execute();
+}
+
+export async function down(db: Kysely<any>): Promise<void> {
+  await db.schema
+    .alterTable('projects')
+    .addColumn('description', 'text')
+    .execute();
+
+  await db.schema.dropTable('projects_summaries').execute();
+}

apps/backend/src/migrations/index.ts

@@ -26,6 +26,7 @@ import * as dropUnusedImagesTable from './0024-drop-unused-images-table';
 import * as addInternalProjectsFields from './0025-add-internal-projects-fields';
 import * as addUsersGroupTable from './0026-add-users-groups-table';
 import * as addProjectsPoliciesTable from './0027-add-projects-policies';
+import * as addProjectsSummaries from './0028-add-projects-summarizes';
 
 export const DB_MIGRATIONS = {
   '0000-add-users-tables': addUsersTables,
@@ -56,4 +57,5 @@ export const DB_MIGRATIONS = {
   '0025-add-internal-projects-fields': addInternalProjectsFields,
   '0026-add-users-groups-table': addUsersGroupTable,
   '0027-add-projects-policies': addProjectsPoliciesTable,
+  '0028-add-projects-summarizes': addProjectsSummaries,
 };

apps/backend/src/modules/boot/boot.service.ts

@@ -13,6 +13,7 @@ import {
   ElasticsearchRegistryBootService,
 } from '../elasticsearch/boot';
 import { LoggerService } from '../logger';
+import { ProjectSummariesCronJob } from '../projects-summaries';
 import { UsersBootService } from '../users';
 
 @injectable()
@@ -32,6 +33,7 @@ export class BootService {
     // Cron jobs
     @inject(ElasticsearchAutoReindexJob) private readonly elasticsearchReindexJob: ElasticsearchAutoReindexJob,
     @inject(ChatSummariesCronJob) private readonly chatSummariesCronJob: ChatSummariesCronJob,
+    @inject(ProjectSummariesCronJob) private readonly projectSummariesCronJob: ProjectSummariesCronJob,
   ) {}
 
   /**
@@ -70,6 +72,7 @@ export class BootService {
   private registerCronJobs = () => {
     this.elasticsearchReindexJob.registerCronJob();
     this.chatSummariesCronJob.registerCronJob();
+    this.projectSummariesCronJob.registerCronJob();
 
     this.logger.info('Registered cron jobs!');
   };

apps/backend/src/modules/chats/chats.repo.ts

@@ -137,11 +137,9 @@ export class ChatsRepo extends createProtectedDatabaseRepo('chats') {
               'users.email as creator_email',
 
               'chat_summaries.id as summary_id',
-
               'chat_summaries.content as summary_content',
               'chat_summaries.content_generated as summary_content_generated',
               'chat_summaries.content_generated_at as summary_content_generated_at',
-
               'chat_summaries.name as summary_name',
               'chat_summaries.name_generated as summary_name_generated',
               'chat_summaries.name_generated_at as summary_name_generated_at',

apps/backend/src/modules/config/helpers/config.dto.ts

@@ -7,6 +7,7 @@ import { ChatsSummariesConfigV } from '~/modules/chats-summaries/chats-summaries
 import { DatabaseConfigV } from '~/modules/database/database.config';
 import { DatabaseMigrateConfigV } from '~/modules/database/migrate/database-migrate.config';
 import { ElasticsearchConfigV } from '~/modules/elasticsearch/elasticsearch.config';
+import { ProjectsSummariesConfigV } from '~/modules/projects-summaries/projects-summaries.config';
 import { UsersConfigV } from '~/modules/users/users.config';
 
 export const ConfigV = z.object({
@@ -20,6 +21,7 @@ export const ConfigV = z.object({
   users: UsersConfigV,
   auth: AuthConfigV,
   chatsSummaries: ChatsSummariesConfigV,
+  projectsSummaries: ProjectsSummariesConfigV,
 });
 
 export type ConfigT = z.infer<typeof ConfigV>;

apps/backend/src/modules/config/helpers/try-read-env-or-panic.ts

@@ -48,8 +48,11 @@ export function tryReadEnvOrPanic() {
     JWT_SECRET,
     JWT_EXPIRES_IN,
 
-    // Chats summaries
+    // Chats
     CHATS_SUMMARIES_CRON,
+
+    // Projects
+    PROJECTS_SUMMARIES_CRON,
   } = {
     ...envFile.parsed,
     ...process.env,
@@ -90,6 +93,9 @@ export function tryReadEnvOrPanic() {
     chatsSummaries: {
       cron: CHATS_SUMMARIES_CRON,
     },
+    projectsSummaries: {
+      cron: PROJECTS_SUMMARIES_CRON,
+    },
     users: {
       root: {
         email: USER_ROOT_EMAIL,

apps/backend/src/modules/database/database.tables.ts

@@ -24,6 +24,7 @@ import type {
 import type { ProjectsEmbeddingsTable } from '../projects-embeddings';
 import type { ProjectsFilesTable } from '../projects-files';
 import type { ProjectsPoliciesTable } from '../projects-policies';
+import type { ProjectsSummariesTable } from '../projects-summaries';
 import type {
   S3ResourcesBucketsTable,
   S3ResourcesTable,
@@ -61,6 +62,7 @@ export type DatabaseTables = {
   projects_files: ProjectsFilesTable;
   projects_embeddings: ProjectsEmbeddingsTable;
   projects_policies: ProjectsPoliciesTable;
+  projects_summaries: ProjectsSummariesTable;
 
   // Apps
   apps: AppsTable;

apps/backend/src/modules/projects-summaries/index.ts

@@ -0,0 +1,6 @@
+export * from './projects-summaries.config';
+export * from './projects-summaries.cron';
+export * from './projects-summaries.errors';
+export * from './projects-summaries.repo';
+export * from './projects-summaries.service';
+export * from './projects-summaries.tables';

apps/backend/src/modules/projects-summaries/projects-summaries.config.ts

@@ -0,0 +1,7 @@
+import { z } from 'zod';
+
+export const ProjectsSummariesConfigV = z.object({
+  cron: z.string().default('*/20 * * * * *'),
+});
+
+export type ProjectsSummariesConfigT = z.infer<typeof ProjectsSummariesConfigV>;

apps/backend/src/modules/projects-summaries/projects-summaries.cron.ts

@@ -0,0 +1,46 @@
+import { flow } from 'fp-ts/lib/function';
+import { inject, injectable } from 'tsyringe';
+
+import { runTaskAsVoid, tryOrThrowTE } from '@llm/commons';
+import { ConfigService } from '~/modules/config';
+
+import { CronJob, CronService } from '../cron';
+import { LoggerService } from '../logger';
+import { ProjectsSummariesService } from './projects-summaries.service';
+
+@injectable()
+export class ProjectSummariesCronJob extends CronJob {
+  private readonly logger = LoggerService.of('ProjectSummariesCronJob');
+
+  constructor(
+    @inject(CronService) cronService: CronService,
+    @inject(ConfigService) configService: ConfigService,
+    @inject(ProjectsSummariesService) private readonly projectsSummariesService: ProjectsSummariesService,
+  ) {
+    super(cronService, configService);
+  }
+
+  registerCronJob(): void {
+    const { logger } = this;
+    const { cron } = this.config.projectsSummaries;
+
+    if (!cron) {
+      logger.warn('ProjectSummariesCronJob cron job is disabled!');
+      return;
+    }
+
+    this.cronService.tryRegister(
+      {
+        expression: cron,
+        skipIfPreviousJobNotDone: true,
+      },
+      flow(
+        this.projectsSummariesService.summarizeAllProjects,
+        tryOrThrowTE,
+        runTaskAsVoid,
+      ),
+    );
+
+    logger.info('Registered ProjectSummariesCronJob cron job!');
+  }
+}

apps/backend/src/modules/projects-summaries/projects-summaries.errors.ts

@@ -0,0 +1,7 @@
+import type { SdKSearchMessagesOutputT } from '@llm/sdk';
+
+import { TaggedError } from '@llm/commons';
+
+export class MissingAIModelInProjectError
+  extends TaggedError.ofLiteral<SdKSearchMessagesOutputT>()('MissingAIModelInProject') {
+}