From 7798b6094c8b4158d0852f804736d0af031f30ed Mon Sep 17 00:00:00 2001 From: andresgutgon Date: Thu, 26 Sep 2024 14:10:42 +0200 Subject: [PATCH] Paginate document logs table to improve page performance --- .../src/app/(private)/_data-access/index.ts | 21 - .../editor/import-logs/page.tsx | 4 +- .../DocumentLogs/DocumentLogsTable.tsx | 6 +- .../logs/_components/DocumentLogs/index.tsx | 8 +- .../documents/[documentUuid]/logs/page.tsx | 43 +- .../src/components/WebPagination/index.tsx | 50 ++ packages/core/src/lib/buildPagination.ts | 110 +++ .../repositories/commitsRepository/index.ts | 5 +- .../documentLogsRepository/index.ts | 21 +- packages/core/src/repositories/repository.ts | 16 +- .../computeDocumentLogsWithMetadata.test.ts} | 52 +- .../computeDocumentLogsWithMetadata.ts | 12 +- packages/web-ui/src/ds/atoms/Button/index.tsx | 1 + .../src/ds/molecules/Pagination/index.tsx | 115 +++ packages/web-ui/src/ds/molecules/index.ts | 1 + pnpm-lock.yaml | 670 +++++++++--------- 16 files changed, 706 insertions(+), 429 deletions(-) create mode 100644 apps/web/src/components/WebPagination/index.tsx create mode 100644 packages/core/src/lib/buildPagination.ts rename packages/core/src/{repositories/documentLogsRepository/getDocumentLogsWithMetadata.test.ts => services/documentLogs/computeDocumentLogsWithMetadata.test.ts} (84%) create mode 100644 packages/web-ui/src/ds/molecules/Pagination/index.tsx diff --git a/apps/web/src/app/(private)/_data-access/index.ts b/apps/web/src/app/(private)/_data-access/index.ts index b514352b4..1f1cdabba 100644 --- a/apps/web/src/app/(private)/_data-access/index.ts +++ b/apps/web/src/app/(private)/_data-access/index.ts @@ -7,7 +7,6 @@ import { ApiKeysRepository } from '@latitude-data/core/repositories/apiKeysRepos import { CommitsRepository, ConnectedEvaluationsRepository, - DocumentLogsRepository, DocumentVersionsRepository, EvaluationsRepository, ProjectsRepository, @@ -140,26 +139,6 @@ export const getDocumentByIdCached = cache(async (id: number) => { return document }) -export const getDocumentLogsWithMetadataCached = cache( - async ({ - documentUuid, - commit, - }: { - documentUuid: string - commit: Commit - }) => { - const { workspace } = await getCurrentUser() - const docsScope = new DocumentLogsRepository(workspace.id) - const result = await docsScope.getDocumentLogsWithMetadata({ - documentUuid, - draft: commit, - }) - const logs = result.unwrap() - - return logs - }, -) - export const getDocumentsFromMergedCommitsCache = cache( async (workspaceId: number) => { const docsScope = new DocumentVersionsRepository(workspaceId) diff --git a/apps/web/src/app/(private)/evaluations/(evaluation)/[evaluationUuid]/editor/import-logs/page.tsx b/apps/web/src/app/(private)/evaluations/(evaluation)/[evaluationUuid]/editor/import-logs/page.tsx index 95634cf28..c22545213 100644 --- a/apps/web/src/app/(private)/evaluations/(evaluation)/[evaluationUuid]/editor/import-logs/page.tsx +++ b/apps/web/src/app/(private)/evaluations/(evaluation)/[evaluationUuid]/editor/import-logs/page.tsx @@ -198,7 +198,7 @@ const ProviderLogMessages = ({ ) as ProviderLogDto if (!providerLog) { return ( -
+
Select a log on the table to preview the messages here and import @@ -207,7 +207,7 @@ const ProviderLogMessages = ({ } return ( -
+
Messages
{providerLog.messages.map((message, index) => ( diff --git a/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/logs/_components/DocumentLogs/DocumentLogsTable.tsx b/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/logs/_components/DocumentLogs/DocumentLogsTable.tsx index 385aac26b..fe180536d 100644 --- a/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/logs/_components/DocumentLogs/DocumentLogsTable.tsx +++ b/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/logs/_components/DocumentLogs/DocumentLogsTable.tsx @@ -1,5 +1,3 @@ -'use client' - import { DocumentLogWithMetadata } from '@latitude-data/core/repositories' import { Badge, @@ -33,6 +31,7 @@ export const DocumentLogsTable = ({ Time Version + Origin Custom Identifier Duration Tokens @@ -75,6 +74,9 @@ export const DocumentLogsTable = ({ {documentLog.commit.title}
+ + {documentLog.source} + {documentLog.customIdentifier} diff --git a/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/logs/_components/DocumentLogs/index.tsx b/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/logs/_components/DocumentLogs/index.tsx index f29464659..cc10910e9 100644 --- a/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/logs/_components/DocumentLogs/index.tsx +++ b/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/logs/_components/DocumentLogs/index.tsx @@ -2,7 +2,9 @@ import { useState } from 'react' +import { IPagination } from '@latitude-data/core/lib/buildPagination' import { DocumentLogWithMetadata } from '@latitude-data/core/repositories' +import WebPagination from '$/components/WebPagination' import useProviderLogs from '$/stores/providerLogs' import { DocumentLogInfo } from './DocumentLogInfo' @@ -10,8 +12,10 @@ import { DocumentLogsTable } from './DocumentLogsTable' export function DocumentLogs({ documentLogs, + pagination, }: { documentLogs: DocumentLogWithMetadata[] + pagination: IPagination }) { const [selectedLog, setSelectedLog] = useState< DocumentLogWithMetadata | undefined @@ -22,16 +26,18 @@ export function DocumentLogs({ return (
-
+
+
{selectedLog && (
diff --git a/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/logs/page.tsx b/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/logs/page.tsx index 127c4eedb..12ecc4794 100644 --- a/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/logs/page.tsx +++ b/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/logs/page.tsx @@ -1,34 +1,57 @@ -import { TableBlankSlate, TableWithHeader } from '@latitude-data/web-ui' import { - findCommitCached, - getDocumentLogsWithMetadataCached, -} from '$/app/(private)/_data-access' + buildPagination, + parsePage, +} from '@latitude-data/core/lib/buildPagination' +import { computeDocumentLogsWithMetadata } from '@latitude-data/core/services/documentLogs/computeDocumentLogsWithMetadata' +import { TableBlankSlate, TableWithHeader } from '@latitude-data/web-ui' +import { findCommitCached } from '$/app/(private)/_data-access' +import { getCurrentUser } from '$/services/auth/getCurrentUser' +import { ROUTES } from '$/services/routes' import { DocumentLogs } from './_components/DocumentLogs' +const PAGE_SIZE = 25 export default async function DocumentPage({ params, + searchParams, }: { params: { projectId: string; commitUuid: string; documentUuid: string } + searchParams: { [key: string]: string | string[] | undefined } }) { + const { workspace } = await getCurrentUser() const projectId = Number(params.projectId) const commitUuid = params.commitUuid const commit = await findCommitCached({ projectId, uuid: commitUuid }) - const logs = await getDocumentLogsWithMetadataCached({ + const page = parsePage(searchParams.page) + const [rows, count] = await computeDocumentLogsWithMetadata({ + workspaceId: workspace.id, documentUuid: params.documentUuid, - commit, + draft: commit, + pagination: { page, pageSize: PAGE_SIZE }, }) - + const baseUrl = ROUTES.projects + .detail({ id: projectId }) + .commits.detail({ uuid: commitUuid }) + .documents.detail({ uuid: params.documentUuid }).logs.root + const pagination = buildPagination({ + baseUrl, + count, + page, + pageSize: PAGE_SIZE, + }) + const title = `${pagination.count} logs (page ${pagination.currentPage} of ${pagination.totalPages})` return (
- {!logs.length && ( + {!rows.length && ( )} - {logs.length > 0 && } + {rows.length > 0 && ( + + )} } /> diff --git a/apps/web/src/components/WebPagination/index.tsx b/apps/web/src/components/WebPagination/index.tsx new file mode 100644 index 000000000..603cf5fae --- /dev/null +++ b/apps/web/src/components/WebPagination/index.tsx @@ -0,0 +1,50 @@ +import { IPagination } from '@latitude-data/core/lib/buildPagination' +import { + Pagination, + PaginationContent, + PaginationEllipsis, + PaginationItem, + PaginationLink, + PaginationNext, + PaginationPrevious, +} from '@latitude-data/web-ui' +import Link from 'next/link' + +export default function WebPagination({ + currentPage, + prevPage, + pageItems, + nextPage, +}: IPagination) { + return ( + + + {prevPage && ( + + + + )} + {pageItems.map((item, idx) => { + return ( + + {item.type === 'page' ? ( + + + {item.value} + + + ) : ( + + )} + + ) + })} + {nextPage && ( + + + + )} + + + ) +} diff --git a/packages/core/src/lib/buildPagination.ts b/packages/core/src/lib/buildPagination.ts new file mode 100644 index 000000000..6e790e0eb --- /dev/null +++ b/packages/core/src/lib/buildPagination.ts @@ -0,0 +1,110 @@ +import { isNumber } from 'lodash-es' + +type ItemType = 'page' | 'ellipsis' +type PageItem = T extends 'page' + ? { type: T; value: number; url: string } + : { type: T; value: '...' } +export type IPagination = { + count: number + totalPages: number + currentPage: number + prevPage?: PageItem<'page'> + pageItems: (PageItem<'page'> | PageItem<'ellipsis'>)[] + nextPage?: PageItem<'page'> +} + +function buildUrl({ + baseUrl, + page, + limit, +}: { + baseUrl: string + page: number + limit: number +}) { + return `${baseUrl}?page=${page}&limit=${limit}` +} + +export function buildPagination({ + baseUrl, + count, + page, + pageSize, + pageItemsCount = 10, +}: { + baseUrl: string + count: number + page: number + pageSize: number + pageItemsCount?: number +}) { + const totalPages = Math.ceil(count / pageSize) + const pagination: IPagination = { + count, + totalPages, + currentPage: page, + prevPage: + page > 1 + ? { + type: 'page', + value: page - 1, + url: buildUrl({ baseUrl, page: page - 1, limit: pageSize }), + } + : undefined, + pageItems: [], + nextPage: + page < totalPages + ? { + type: 'page', + value: page + 1, + url: buildUrl({ baseUrl, page: page + 1, limit: pageSize }), + } + : undefined, + } + + for (let i = page - 1; i >= 1 && pagination.pageItems.length < 2; i--) { + pagination.pageItems.unshift({ + type: 'page', + url: buildUrl({ baseUrl, page: i, limit: pageSize }), + value: i, + }) + } + + const prevItem = pagination.pageItems[0] + const prevPage = isNumber(prevItem?.value) ? prevItem.value : undefined + if (pagination.pageItems.length > 0 && prevPage && prevPage < page - 2) { + pagination.pageItems.push({ type: 'ellipsis', value: '...' }) + } + + pagination.pageItems.push({ + type: 'page', + value: page, + url: buildUrl({ baseUrl, page, limit: pageSize }), + }) + + for ( + let i = page + 1; + i <= totalPages && pagination.pageItems.length < pageItemsCount; + i++ + ) { + pagination.pageItems.push({ + type: 'page', + value: i, + url: buildUrl({ baseUrl, page: i, limit: pageSize }), + }) + } + + const nextItem = pagination.pageItems[pagination.pageItems.length - 1] + const nextPage = isNumber(nextItem?.value) ? nextItem.value : undefined + if (pagination.pageItems.length > 0 && nextPage && nextPage > page + 1) { + pagination.pageItems.push({ type: 'ellipsis', value: '...' }) + } + + return pagination +} + +export function parsePage( + page: string | string[] | number | undefined, +): number { + return typeof page === 'string' ? Number(page) : 1 +} diff --git a/packages/core/src/repositories/commitsRepository/index.ts b/packages/core/src/repositories/commitsRepository/index.ts index c4b8e5e42..3ca01be6e 100644 --- a/packages/core/src/repositories/commitsRepository/index.ts +++ b/packages/core/src/repositories/commitsRepository/index.ts @@ -156,12 +156,13 @@ export class CommitsRepository extends Repository< .where(and(eq(this.scope.projectId, project.id), filter)) .orderBy(desc(this.scope.createdAt)) - const result = await Repository.paginateQuery({ + const [rows] = await Repository.paginateQuery({ query: query.$dynamic(), page, pageSize, }) - return Result.ok(result) + + return Result.ok(rows) } async getChanges(id: number, tx = database) { diff --git a/packages/core/src/repositories/documentLogsRepository/index.ts b/packages/core/src/repositories/documentLogsRepository/index.ts index a0dd2bf08..88becccc6 100644 --- a/packages/core/src/repositories/documentLogsRepository/index.ts +++ b/packages/core/src/repositories/documentLogsRepository/index.ts @@ -1,9 +1,8 @@ import { eq, getTableColumns } from 'drizzle-orm' import { Commit, DocumentLog } from '../../browser' -import { NotFoundError, Result, TypedResult } from '../../lib' +import { NotFoundError, Result } from '../../lib' import { commits, documentLogs, projects, workspaces } from '../../schema' -import { computeDocumentLogsWithMetadata } from '../../services/documentLogs' import Repository from '../repository' export type DocumentLogWithMetadata = DocumentLog & { @@ -40,22 +39,4 @@ export class DocumentLogsRepository extends Repository { return Result.ok(result[0]!) } - - // TODO: remove in favor of computeDocumentLogsWithMetadata - async getDocumentLogsWithMetadata({ - documentUuid, - draft, - }: { - documentUuid: string - draft?: Commit - }): Promise> { - return computeDocumentLogsWithMetadata( - { - workspaceId: this.workspaceId, - documentUuid, - draft, - }, - this.db, - ) - } } diff --git a/packages/core/src/repositories/repository.ts b/packages/core/src/repositories/repository.ts index 235713877..beaec22f0 100644 --- a/packages/core/src/repositories/repository.ts +++ b/packages/core/src/repositories/repository.ts @@ -1,4 +1,4 @@ -import { ColumnsSelection, eq } from 'drizzle-orm' +import { ColumnsSelection, eq, sql } from 'drizzle-orm' import { PgSelect, SubqueryWithSelection } from 'drizzle-orm/pg-core' import { database } from '../client' @@ -10,6 +10,8 @@ export type QueryOptions = { offset?: number } +type PaginatatedResult = [rows: Awaited, count: number] + export default abstract class Repository< U extends ColumnsSelection, T extends Record, @@ -32,8 +34,16 @@ export default abstract class Repository< pageSize = 20, }: { query: T - } & PaginationArgs) { - return query.limit(pageSize).offset((page - 1) * pageSize) + } & PaginationArgs): Promise> { + // @ts-ignore + query.config.fields = { + // @ts-ignore + ...query.config.fields, + __count: sql`count(*) over()`, + } + const rows = await query.limit(pageSize).offset((page - 1) * pageSize) + const count = rows[0]?.__count ? Number(rows[0]?.__count) : 0 + return [rows, count] } abstract get scope(): SubqueryWithSelection diff --git a/packages/core/src/repositories/documentLogsRepository/getDocumentLogsWithMetadata.test.ts b/packages/core/src/services/documentLogs/computeDocumentLogsWithMetadata.test.ts similarity index 84% rename from packages/core/src/repositories/documentLogsRepository/getDocumentLogsWithMetadata.test.ts rename to packages/core/src/services/documentLogs/computeDocumentLogsWithMetadata.test.ts index 87e738b1a..5f67c623b 100644 --- a/packages/core/src/repositories/documentLogsRepository/getDocumentLogsWithMetadata.test.ts +++ b/packages/core/src/services/documentLogs/computeDocumentLogsWithMetadata.test.ts @@ -3,7 +3,7 @@ import { describe, expect, it } from 'vitest' import { mergeCommit } from '../../services/commits' import { updateDocument } from '../../services/documents' import * as factories from '../../tests/factories' -import { DocumentLogsRepository } from './index' +import { computeDocumentLogsWithMetadata } from './computeDocumentLogsWithMetadata' describe('getDocumentLogsWithMetadata', () => { it('return all logs from merged commits', async () => { @@ -39,12 +39,12 @@ describe('getDocumentLogsWithMetadata', () => { commit: commit2, }) - const documentsScope = new DocumentLogsRepository(project.workspaceId) - const result = await documentsScope - .getDocumentLogsWithMetadata({ - documentUuid: doc.documentUuid, - }) - .then((r) => r.unwrap()) + const [result] = await computeDocumentLogsWithMetadata({ + workspaceId: project.workspaceId, + documentUuid: doc.documentUuid, + draft: commit2, + pagination: { page: 1, pageSize: 100 }, + }) expect(result.find((l) => l.uuid === log1.uuid)).toBeDefined() expect(result.find((l) => l.uuid === log2.uuid)).toBeDefined() @@ -98,13 +98,12 @@ describe('getDocumentLogsWithMetadata', () => { commit: draft, }) - const documentsScope = new DocumentLogsRepository(project.workspaceId) - const result = await documentsScope - .getDocumentLogsWithMetadata({ - documentUuid: doc.documentUuid, - draft: draft, - }) - .then((r) => r.unwrap()) + const [result] = await computeDocumentLogsWithMetadata({ + workspaceId: project.workspaceId, + documentUuid: doc.documentUuid, + draft, + pagination: { page: 1, pageSize: 100 }, + }) expect(result.find((l) => l.uuid === log1.uuid)).toBeDefined() expect(result.find((l) => l.uuid === log2.uuid)).toBeDefined() @@ -157,13 +156,12 @@ describe('getDocumentLogsWithMetadata', () => { commit: draft2, }) - const documentsScope = new DocumentLogsRepository(project.workspaceId) - const result = await documentsScope - .getDocumentLogsWithMetadata({ - documentUuid: doc.documentUuid, - draft: draft1, - }) - .then((r) => r.unwrap()) + const [result] = await computeDocumentLogsWithMetadata({ + workspaceId: project.workspaceId, + documentUuid: doc.documentUuid, + draft: draft1, + pagination: { page: 1, pageSize: 100 }, + }) expect(result.find((l) => l.uuid === log1.uuid)).toBeDefined() expect(result.find((l) => l.uuid === log2.uuid)).toBeDefined() @@ -188,12 +186,12 @@ describe('getDocumentLogsWithMetadata', () => { commit, }) - const documentsScope = new DocumentLogsRepository(project.workspaceId) - const result = await documentsScope - .getDocumentLogsWithMetadata({ - documentUuid: doc.documentUuid, - }) - .then((r) => r.unwrap()) + const [result] = await computeDocumentLogsWithMetadata({ + workspaceId: project.workspaceId, + documentUuid: doc.documentUuid, + draft: commit, + pagination: { page: 1, pageSize: 100 }, + }) expect(result.find((l) => l.uuid === log.uuid)).toBeDefined() expect(result.find((l) => l.uuid === log.uuid)?.tokens).toBeTypeOf('number') diff --git a/packages/core/src/services/documentLogs/computeDocumentLogsWithMetadata.ts b/packages/core/src/services/documentLogs/computeDocumentLogsWithMetadata.ts index ec6c3de29..84c3aa0c0 100644 --- a/packages/core/src/services/documentLogs/computeDocumentLogsWithMetadata.ts +++ b/packages/core/src/services/documentLogs/computeDocumentLogsWithMetadata.ts @@ -2,8 +2,7 @@ import { and, desc, eq } from 'drizzle-orm' import { Commit } from '../../browser' import { database } from '../../client' -import { Result, TypedResult } from '../../lib' -import { DocumentLogWithMetadata } from '../../repositories/documentLogsRepository' +import Repository, { PaginationArgs } from '../../repositories/repository' import { createDocumentLogQuery, getCommitFilter, @@ -14,18 +13,19 @@ export async function computeDocumentLogsWithMetadata( workspaceId, documentUuid, draft, + pagination, }: { workspaceId: number documentUuid: string draft?: Commit + pagination: PaginationArgs }, db = database, -): Promise> { +) { const { scope, baseQuery } = createDocumentLogQuery(workspaceId, db) - - const result = await baseQuery + const query = baseQuery .where(and(eq(scope.documentUuid, documentUuid), getCommitFilter(draft))) .orderBy(desc(scope.createdAt)) - return Result.ok(result) + return Repository.paginateQuery({ query: query.$dynamic(), ...pagination }) } diff --git a/packages/web-ui/src/ds/atoms/Button/index.tsx b/packages/web-ui/src/ds/atoms/Button/index.tsx index e3bcac6a1..ec462b6b9 100644 --- a/packages/web-ui/src/ds/atoms/Button/index.tsx +++ b/packages/web-ui/src/ds/atoms/Button/index.tsx @@ -74,6 +74,7 @@ const buttonVariants = cva( default: 'py-1.5 px-3', small: 'py-1 px-1.5', none: 'py-0 px-0', + icon: 'h-10 w-10', }, fanciness: { default: '', diff --git a/packages/web-ui/src/ds/molecules/Pagination/index.tsx b/packages/web-ui/src/ds/molecules/Pagination/index.tsx new file mode 100644 index 000000000..de07d822e --- /dev/null +++ b/packages/web-ui/src/ds/molecules/Pagination/index.tsx @@ -0,0 +1,115 @@ +import { ComponentProps, forwardRef } from 'react' +import { ChevronLeft, ChevronRight, MoreHorizontal } from 'lucide-react' + +import { cn } from '../../../lib/utils' +import { ButtonProps, buttonVariants } from '../../atoms/Button' + +const Pagination = ({ className, ...props }: ComponentProps<'nav'>) => ( +