Store PDF documents in db table

apps/spotlight/src/context.ts

@@ -2,6 +2,7 @@ import {
   type FormConfig,
   type FormService,
   createFormService,
+  parsePdf,
 } from '@atj/forms';
 import { defaultFormConfig } from '@atj/forms';
 import { BrowserFormRepository } from '@atj/forms/context';
@@ -43,6 +44,7 @@ const createAppFormService = () => {
       repository,
       config: defaultFormConfig,
       isUserLoggedIn: () => true,
+      parsePdf,
     });
   } else {
     return createTestBrowserFormService();

apps/spotlight/src/features/form-page/store/actions/get-form-session.ts

@@ -43,7 +43,6 @@ export const getFormSession: GetFormSession = async (ctx, opts) => {
           },
         });
       } else {
-        console.log('using session', result.data.data);
         ctx.setState({
           formSessionResponse: {
             status: 'loaded',

packages/common/src/index.ts

@@ -7,6 +7,7 @@ export type Result<T, E = string> = Success<T> | Failure<E>;
 export type VoidResult<E = string> = VoidSuccess | Failure<E>;
 
 export const success = <T>(data: T): Success<T> => ({ success: true, data });
+export const voidSuccess: VoidSuccess = { success: true };
 export const failure = <E>(error: E): Failure<E> => ({ success: false, error });
 
 export { en as enLocale } from './locales/en/app.js';

packages/database/migrations/20241031103354_form_documents_table.mjs

@@ -0,0 +1,21 @@
+/**
+ * @param { import("knex").Knex } knex
+ * @returns { Promise<void> }
+ */
+export async function up(knex) {
+  await knex.schema.createTable('form_documents', table => {
+    table.uuid('id').primary();
+    table.string('type').notNullable();
+    table.string('file_name').notNullable();
+    table.binary('data').notNullable();
+    table.string('extract').notNullable();
+  });
+}
+
+/**
+ * @param { import("knex").Knex } knex
+ * @returns { Promise<void> }
+ */
+export async function down(knex) {
+  await knex.schema.dropTableIfExists('form_documents');
+}

packages/database/src/clients/kysely/types.ts

@@ -1,4 +1,5 @@
 import type {
+  ColumnType,
   Generated,
   Insertable,
   Kysely,
@@ -14,7 +15,9 @@ export interface Database<T extends Engine = Engine> {
   sessions: SessionsTable<T>;
   forms: FormsTable;
   form_sessions: FormSessionsTable;
+  form_documents: FormDocumentsTable;
 }
+export type DatabaseClient = Kysely<Database>;
 
 interface UsersTable {
   id: string;
@@ -48,8 +51,6 @@ export type FormsTableSelectable = Selectable<FormsTable>;
 export type FormsTableInsertable = Insertable<FormsTable>;
 export type FormsTableUpdateable = Updateable<FormsTable>;
 
-export type DatabaseClient = Kysely<Database>;
-
 interface FormSessionsTable {
   id: string;
   form_id: string;
@@ -60,3 +61,14 @@ interface FormSessionsTable {
 export type FormSessionsTableSelectable = Selectable<FormSessionsTable>;
 export type FormSessionsTableInsertable = Insertable<FormSessionsTable>;
 export type FormSessionsTableUpdateable = Updateable<FormSessionsTable>;
+
+interface FormDocumentsTable {
+  id: string;
+  type: string;
+  data: ColumnType<Buffer, Buffer, Buffer>;
+  file_name: string;
+  extract: string;
+}
+export type FormDocumentsTableSelectable = Selectable<FormDocumentsTable>;
+export type FormDocumentsTableInsertable = Insertable<FormDocumentsTable>;
+export type FormDocumentsTableUpdateable = Updateable<FormDocumentsTable>;

packages/design/src/FormManager/FormList/store.ts

@@ -1,6 +1,6 @@
 import { type StateCreator } from 'zustand';
 
-import { BlueprintBuilder } from '@atj/forms';
+import { BlueprintBuilder, uint8ArrayToBase64 } from '@atj/forms';
 import { type FormManagerContext } from '../../FormManager/index.js';
 import { type Result, failure } from '@atj/common';
 
@@ -23,18 +23,17 @@ export const createFormListSlice =
   () => ({
     context,
     createNewFormByPDFUrl: async url => {
-      const data = await fetchUint8Array(`${context.baseUrl}${url}`);
-
-      const builder = new BlueprintBuilder(context.config);
-      builder.setFormSummary({
-        title: url,
-        description: '',
-      });
-      await builder.addDocument({
-        name: url,
-        data,
+      const data = await fetchAsBase64(`${context.baseUrl}${url}`);
+      const result = await context.formService.initializeForm({
+        summary: {
+          title: url,
+          description: '',
+        },
+        document: {
+          fileName: url,
+          data,
+        },
       });
-      const result = await context.formService.addForm(builder.form);
       if (result.success) {
         return {
           success: true,
@@ -51,7 +50,16 @@ export const createFormListSlice =
         description: '',
       });
       await builder.addDocument(fileDetails);
-      const result = await context.formService.addForm(builder.form);
+      const result = await context.formService.initializeForm({
+        summary: {
+          title: fileDetails.name,
+          description: '',
+        },
+        document: {
+          fileName: fileDetails.name,
+          data: await uint8ArrayToBase64(fileDetails.data),
+        },
+      });
       if (result.success) {
         return {
           success: true,
@@ -63,8 +71,9 @@ export const createFormListSlice =
     },
   });
 
-const fetchUint8Array = async (url: string) => {
+const fetchAsBase64 = async (url: string) => {
   const response = await fetch(url);
   const blob = await response.blob();
-  return new Uint8Array(await blob.arrayBuffer());
+  const data = new Uint8Array(await blob.arrayBuffer());
+  return uint8ArrayToBase64(data);
 };

packages/design/src/FormManager/store.tsx

@@ -8,7 +8,7 @@ import {
 import { createContext } from 'zustand-utils';
 
 import { type Result, failure } from '@atj/common';
-import { type FormSession, type Blueprint, BlueprintBuilder } from '@atj/forms';
+import { type FormSession, type Blueprint } from '@atj/forms';
 
 import { type FormListSlice, createFormListSlice } from './FormList/store.js';
 import { type FormEditSlice, createFormEditSlice } from './FormEdit/store.js';
@@ -79,12 +79,12 @@ const createFormManagerSlice =
       inProgress: false,
     },
     createNewForm: async function () {
-      const builder = new BlueprintBuilder(context.config);
-      builder.setFormSummary({
-        title: `My form - ${new Date().toISOString()}`,
-        description: '',
+      const result = await context.formService.initializeForm({
+        summary: {
+          title: `My form - ${new Date().toISOString()}`,
+          description: '',
+        },
       });
-      const result = await context.formService.addForm(builder.form);
       if (!result.success) {
         return failure(result.error.message);
       }

packages/forms/src/blueprint.ts

@@ -71,11 +71,11 @@ export const createForm = (
     patterns: [
       {
         id: 'root',
-        type: 'sequence',
+        type: 'page-set',
         data: {
-          patterns: [],
+          pages: [],
         },
-      } satisfies SequencePattern,
+      } satisfies PageSetPattern,
     ],
     root: 'root',
   }

packages/forms/src/builder/index.ts

@@ -1,4 +1,6 @@
-import { type VoidResult } from '@atj/common';
+import * as z from 'zod';
+
+import { failure, success, type Result, type VoidResult } from '@atj/common';
 import {
   addPageToPageSet,
   addPatternToFieldset,
@@ -9,7 +11,7 @@ import {
   removePatternFromBlueprint,
   updateFormSummary,
 } from '../blueprint.js';
-import { addDocument } from '../documents/document.js';
+import { addDocument, addParsedPdfToForm } from '../documents/document.js';
 import type { FormErrors } from '../error.js';
 import {
   createDefaultPattern,
@@ -23,6 +25,64 @@ import {
 import { type FieldsetPattern } from '../patterns/fieldset/config.js';
 import { type PageSetPattern } from '../patterns/page-set/config.js';
 import type { Blueprint, FormSummary } from '../types.js';
+import type { ParsedPdf } from '../documents/pdf/parsing-api.js';
+
+const createFormSchema = (config: FormConfig) => {
+  return z.object({
+    summary: z.object({
+      title: z.string(),
+      description: z.string(),
+    }),
+    root: z.string(),
+    patterns: z.record(
+      z.string(),
+      z.any().refine(
+        val => {
+          const patternConfig = config.patterns[val?.type];
+          if (!patternConfig) {
+            return false;
+          }
+          const result = patternConfig.parseConfigData(val?.data);
+          if (!result.success) {
+            const message = Object.values(result.error)
+              .map(err => err.message || '')
+              .join(', ');
+            console.error(val?.type, result.error);
+            console.error(`Pattern config error: ${message}`);
+          }
+          return result.success;
+        },
+        {
+          message: 'Invalid pattern',
+        }
+      )
+    ),
+    outputs: z.array(
+      z.object({
+        id: z.string(),
+        path: z.string(),
+        fields: z.record(z.string(), z.any()),
+        formFields: z.record(z.string(), z.string()),
+      })
+    ),
+  });
+};
+
+export const parseForm = (config: FormConfig, obj: any): Result<Blueprint> => {
+  const formSchema = createFormSchema(config);
+  const result = formSchema.safeParse(obj);
+  if (result.error) {
+    return failure(result.error.message);
+  }
+  return success(result.data);
+};
+
+export const parseFormString = (
+  config: FormConfig,
+  json: string
+): Result<Blueprint> => {
+  return parseForm(config, JSON.parse(json));
+};
 
 export class BlueprintBuilder {
   bp: Blueprint;
@@ -47,6 +107,15 @@ export class BlueprintBuilder {
     this.bp = updatedForm;
   }
 
+  async addDocumentRef(opts: { id: string; extract: ParsedPdf }) {
+    const { updatedForm } = await addParsedPdfToForm(this.form, {
+      id: opts.id,
+      label: opts.extract.title,
+      extract: opts.extract,
+    });
+    this.bp = updatedForm;
+  }
+
   addPage() {
     const newPage = createDefaultPattern(this.config, 'page');
     this.bp = addPageToPageSet(this.form, newPage);