feat: wip

apps/web/src/actions/commits/fetchCommitsByProjectAction.ts

@@ -25,7 +25,7 @@ export const fetchCommitsByProjectAction = withProject
       project: ctx.project,
       filterByStatus: input.status,
     })
-    const [rows] = await paginateQuery({
+    const { rows } = await paginateQuery({
       query,
       defaultPaginate: {
         pageSize: ULTRA_LARGE_PAGE_SIZE,

apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/logs/_components/DocumentLogs/index.tsx

@@ -2,7 +2,7 @@
 
 import { useState } from 'react'
 
-import { IPagination } from '@latitude-data/core/lib/buildPagination'
+import { IPagination } from '@latitude-data/core/lib/pagination/buildPagination'
 import { DocumentLogWithMetadata } from '@latitude-data/core/repositories'
 import WebPagination from '$/components/WebPagination'
 import useProviderLogs from '$/stores/providerLogs'

apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/logs/page.tsx

@@ -1,16 +1,12 @@
-import {
-  buildPagination,
-  parsePage,
-} from '@latitude-data/core/lib/buildPagination'
-import { computeDocumentLogsWithMetadata } from '@latitude-data/core/services/documentLogs/computeDocumentLogsWithMetadata'
+import { paginateQuery } from '@latitude-data/core/lib/pagination/paginate'
+import { computeDocumentLogsWithMetadataQuery } from '@latitude-data/core/services/documentLogs/computeDocumentLogsWithMetadata'
 import { TableBlankSlate, TableWithHeader } from '@latitude-data/web-ui'
 import { findCommitCached } from '$/app/(private)/_data-access'
 import { getCurrentUser } from '$/services/auth/getCurrentUser'
 import { ROUTES } from '$/services/routes'
 
 import { DocumentLogs } from './_components/DocumentLogs'
 
-const PAGE_SIZE = 25
 export default async function DocumentPage({
   params,
   searchParams,
@@ -22,23 +18,21 @@ export default async function DocumentPage({
   const projectId = Number(params.projectId)
   const commitUuid = params.commitUuid
   const commit = await findCommitCached({ projectId, uuid: commitUuid })
-  const page = parsePage(searchParams.page)
-  const [rows, count] = await computeDocumentLogsWithMetadata({
-    workspaceId: workspace.id,
-    documentUuid: params.documentUuid,
-    draft: commit,
-    pagination: { page, pageSize: PAGE_SIZE },
-  })
-  const baseUrl = ROUTES.projects
-    .detail({ id: projectId })
-    .commits.detail({ uuid: commitUuid })
-    .documents.detail({ uuid: params.documentUuid }).logs.root
-  const pagination = buildPagination({
-    baseUrl,
-    count,
-    page,
-    pageSize: PAGE_SIZE,
+  const { rows, pagination } = await paginateQuery({
+    pageUrl: {
+      base: ROUTES.projects
+        .detail({ id: projectId })
+        .commits.detail({ uuid: commitUuid })
+        .documents.detail({ uuid: params.documentUuid }).logs.root,
+    },
+    searchParams,
+    query: computeDocumentLogsWithMetadataQuery({
+      workspaceId: workspace.id,
+      documentUuid: params.documentUuid,
+      draft: commit,
+    }),
   })
+
   const title = `${pagination.count} logs (page ${pagination.currentPage} of ${pagination.totalPages})`
   return (
     <div className='flex flex-col w-full h-full overflow-hidden p-6 gap-2 min-w-0'>

apps/web/src/components/WebPagination/index.tsx

@@ -1,4 +1,4 @@
-import { IPagination } from '@latitude-data/core/lib/buildPagination'
+import { IPagination } from '@latitude-data/core/lib/pagination/buildPagination'
 import {
   Pagination,
   PaginationContent,

packages/core/src/lib/pagination/paginate.ts

@@ -1,18 +1,33 @@
-import { and, desc, eq, sql, type ColumnsSelection } from 'drizzle-orm'
+import { sql, type ColumnsSelection } from 'drizzle-orm'
 import type { PgSelect, PgSelectBase } from 'drizzle-orm/pg-core'
 import { SelectMode } from 'drizzle-orm/query-builders/select.types'
 
-import { Commit } from '../../browser'
-import { database } from '../../client'
-import {
-  createEvaluationResultQuery,
-  getCommitFilter,
-} from '../../services/evaluationResults/_createEvaluationResultQuery'
+import { buildPagination, parsePage } from './buildPagination'
 
 type ISearchParamsObject = { [key: string]: string | string[] | undefined }
-export type PaginationArgs = { page?: number; pageSize?: number }
+type PaginationArgs = {
+  page?: number
+  pageSize?: number
+  pageItemsCount?: number
+}
 
-type PaginatatedResult<T> = [rows: Awaited<T>, count: number]
+type PaginatatedResult<T> = { rows: Awaited<T>; count: number }
+
+function queryParamsWithDefaults({
+  defaultPaginate,
+  searchParams,
+}: {
+  defaultPaginate?: Exclude<PaginationArgs, 'page'>
+  searchParams?: ISearchParamsObject | undefined
+}) {
+  return {
+    page: parsePage(searchParams?.page),
+    pageSize: searchParams?.pageSize
+      ? Number(searchParams.pageSize)
+      : (defaultPaginate?.pageSize ?? 25),
+    pageItemsCount: defaultPaginate?.pageItemsCount ?? 10,
+  }
+}
 
 /**
  * This use $dynamic() query
@@ -33,22 +48,34 @@ async function paginateQuerySql<T extends PgSelect>({
   }
   const rows = await query.limit(pageSize).offset((page - 1) * pageSize)
   const count = rows[0]?.__count ? Number(rows[0]?.__count) : 0
-  return [rows, count]
+  return { rows, count }
 }
 
 export async function paginateQuery<
   TTableName extends string | undefined,
   TSelection extends ColumnsSelection,
 >({
   query,
+  pageUrl,
+  searchParams,
+  defaultPaginate,
 }: {
+  pageUrl?: { base?: string; queryParams?: Record<string, unknown> }
   query: Omit<
     PgSelectBase<TTableName, TSelection, SelectMode, any, false, string>,
     'where' | 'orderBy' | 'limit' | 'offset'
   >
   searchParams?: ISearchParamsObject
-  defaultPaginate?: PaginationArgs
+  defaultPaginate?: Exclude<PaginationArgs, 'page'>
 }) {
+  const params = queryParamsWithDefaults({ searchParams, defaultPaginate })
   const dynamic = query.$dynamic()
-  return paginateQuerySql({ query: dynamic })
+  const { rows, count } = await paginateQuerySql({ query: dynamic, ...params })
+  const pagination = buildPagination({
+    baseUrl: pageUrl?.base ?? '',
+    count,
+    ...params,
+  })
+
+  return { rows, pagination }
 }

packages/core/src/repositories/commitsRepository/index.ts

@@ -132,7 +132,10 @@ export class CommitsRepository extends Repository<
   getCommitsByProjectQuery({
     project,
     filterByStatus = CommitStatus.All,
-  }: { project: Project; filterByStatus?: CommitStatus } & PaginationArgs) {
+  }: {
+    project: Project
+    filterByStatus?: CommitStatus
+  }) {
     const filter = filterByStatusQuery({
       scope: this.scope,
       status: filterByStatus,

packages/core/src/repositories/connectedEvaluationsRepository/index.ts

@@ -83,8 +83,7 @@ export class ConnectedEvaluationsRepository extends Repository<
       this.db,
     )
 
-    // Last version of each (merged) document
-    const lastVersionOfEachDocument = this.db
+    const lastVersionOfEachDocument = this.db // Last version of each (merged) document
       .$with('last_version_of_each_document')
       .as(
         this.db
@@ -122,28 +121,19 @@ export class ConnectedEvaluationsRepository extends Repository<
         .where(
           and(
             eq(this.scope.evaluationId, evaluationId),
-            // Only show non-removed documents
-            isNull(lastVersionOfEachDocument.deletedAt),
+            isNull(lastVersionOfEachDocument.deletedAt), // Only show non-removed documents
           ),
         ),
     )
 
-    const PROVIDER_LOGS_COLUMNS = getTableColumns(providerLogs)
-    const DOCUMENT_LOGS_COLUMNS = getTableColumns(documentLogs)
-    DOCUMENT_LOGS_COLUMNS
+    // TODO: figure out proper type for this
     const selectedEvaluationResults = this.db
       .$with('selected_evaluation_results')
-      .as(() => {
-        const baseQueryScope = EvaluationResultsRepository.baseQuery(this.db).as(
-          'selected_evaluation_results_with_documents_and_providers_scope',
-        )
-        const extendedQuery = this.db
-          .select({
-            ...baseQueryScope._.selectedFields,
-            ...DOCUMENT_LOGS_COLUMNS,
-            ...PROVIDER_LOGS_COLUMNS,
-          })
-          .from(baseQueryScope)
+      .as(
+        EvaluationResultsRepository.baseQuery(this.db, {
+          ...getTableColumns(documentLogs),
+          ...getTableColumns(providerLogs),
+        })
           .innerJoin(
             documentLogs,
             eq(documentLogs.id, evaluationResults.documentLogId),
@@ -152,18 +142,20 @@ export class ConnectedEvaluationsRepository extends Repository<
             providerLogs,
             eq(providerLogs.id, evaluationResults.providerLogId),
           )
-          .where(eq(evaluationResults.evaluationId, evaluationId))
-        return extendedQuery
-      })
+          .where(eq(evaluationResults.evaluationId, evaluationId)),
+      )
 
     const aggregatedResults = this.db
       .with(selectedEvaluationResults)
       .select({
+        // @ts-expect-error
         documentUuid: selectedEvaluationResults.documentUuid,
         evaluationLogs: count(selectedEvaluationResults.id).as(
           'evaluation_logs',
         ),
+        // @ts-expect-error
         totalTokens: sum(selectedEvaluationResults.tokens).as('total_tokens'),
+        // @ts-expect-error
         costInMillicents: sum(selectedEvaluationResults.costInMillicents).as(
           'cost_in_millicents',
         ),
@@ -174,6 +166,7 @@ export class ConnectedEvaluationsRepository extends Repository<
         ),
       })
       .from(selectedEvaluationResults)
+      // @ts-expect-error
       .groupBy(selectedEvaluationResults.documentUuid)
       .as('aggregated_results')
 
@@ -191,6 +184,7 @@ export class ConnectedEvaluationsRepository extends Repository<
           selectedEvaluationResults,
           eq(
             aggregatedResults.documentUuid,
+            // @ts-expect-error
             selectedEvaluationResults.documentUuid,
           ),
         )

packages/core/src/repositories/evaluationResultsRepository/index.ts

@@ -1,4 +1,5 @@
 import { eq, getTableColumns, sql } from 'drizzle-orm'
+import { NodePgDatabase } from 'drizzle-orm/node-postgres'
 
 import {
   Commit,
@@ -15,7 +16,6 @@ import {
   evaluations,
 } from '../../schema'
 import Repository from '../repository'
-import { Database } from '../../client'
 
 export const evaluationResultDto = {
   ...getTableColumns(evaluationResults),
@@ -43,10 +43,14 @@ export class EvaluationResultsRepository extends Repository<
   EvaluationResultDto
 > {
   static baseQuery(
-    db: Database,
+    db: NodePgDatabase<any>,
+    selectStatements?: Record<string, any>,
   ) {
     return db
-      .select(evaluationResultDto)
+      .select({
+        ...evaluationResultDto,
+        ...selectStatements,
+      })
       .from(evaluationResults)
       .innerJoin(
         evaluations,

packages/core/src/repositories/repository.ts

@@ -1,4 +1,4 @@
-import { ColumnsSelection, eq, sql } from 'drizzle-orm'
+import { ColumnsSelection, eq } from 'drizzle-orm'
 import { SubqueryWithSelection } from 'drizzle-orm/pg-core'
 
 import { database } from '../client'

packages/core/src/services/documentLogs/computeDocumentLogsWithMetadata.ts

@@ -7,7 +7,7 @@ import {
   getCommitFilter,
 } from './_createDocumentLogQuery'
 
-export async function computeDocumentLogsWithMetadataQuery(
+export function computeDocumentLogsWithMetadataQuery(
   {
     workspaceId,
     documentUuid,