Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feature: count chunks in group route and in UI #2824

Merged
merged 2 commits into from
Nov 21, 2024
Merged

feature: count chunks in group route and in UI #2824

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
4d0ef77
feature: route to get number of chunks in a group with /api/chunk_gro…
cdxker Nov 21, 2024
ef730a5
feature: remove description on groups view in favor of Chunk Count on
cdxker Nov 21, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
114 changes: 112 additions & 2 deletions clients/ts-sdk/openapi.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2187,6 +2187,78 @@
]
}
},
"/api/chunk_group/count": {
"post": {
"tags": [
"Chunk Group"
],
"summary": "Count Chunks in a Group",
"description": "Route to get the number of chunks that is in a group",
"operationId": "count_group_chunks",
"parameters": [
{
"name": "TR-Dataset",
"in": "header",
"description": "The dataset id or tracking_id to use for the request. We assume you intend to use an id if the value is a valid uuid.",
"required": true,
"schema": {
"type": "string",
"format": "uuid"
}
}
],
"requestBody": {
"description": "JSON request payload to add a chunk to a group (bookmark it)",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/GetChunkGroupCountRequest"
}
}
},
"required": true
},
"responses": {
"200": {
"description": "JSON body representing the group with the count",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/GetChunkGroupCountResponse"
}
}
}
},
"400": {
"description": "Service error relating to getting the group with the given tracking id",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponseBody"
}
}
}
},
"404": {
"description": "Group not found",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponseBody"
}
}
}
}
},
"security": [
{
"ApiKey": [
"readonly"
]
}
]
}
},
"/api/chunk_group/group_oriented_search": {
"post": {
"tags": [
Expand Down Expand Up @@ -9476,7 +9548,8 @@
"group_chunks_updated",
"group_chunks_action_failed",
"crawl_completed",
"crawl_failed"
"crawl_failed",
"crawl_started"
]
},
"EventTypes": {
Expand Down Expand Up @@ -10349,6 +10422,42 @@
}
}
},
"GetChunkGroupCountRequest": {
"type": "object",
"properties": {
"group_id": {
"type": "string",
"format": "uuid",
"description": "The Id of the group to get the count for, is not required if group_tracking_id is provided.",
"nullable": true
},
"group_tracking_id": {
"type": "string",
"description": "The tracking id of the group to get the count for, is not required if group_id is provided.",
"nullable": true
}
}
},
"GetChunkGroupCountResponse": {
"type": "object",
"required": [
"group_id",
"count"
],
"properties": {
"count": {
"type": "integer",
"format": "int64",
"description": "The count of chunks in the given group.",
"minimum": 0
},
"group_id": {
"type": "string",
"format": "uuid",
"description": "The Id of the group to get the count for."
}
}
},
"GetChunksData": {
"type": "object",
"required": [
Expand Down Expand Up @@ -10589,7 +10698,8 @@
},
"total_pages": {
"type": "integer",
"format": "int32"
"format": "int32",
"minimum": 0
}
}
},
Expand Down
56 changes: 55 additions & 1 deletion clients/ts-sdk/src/types.gen.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1120,7 +1120,7 @@ export type EventReturn = {
page_count: number;
};

export type EventTypeRequest = 'file_uploaded' | 'file_upload_failed' | 'chunks_uploaded' | 'chunk_action_failed' | 'chunk_updated' | 'bulk_chunks_deleted' | 'dataset_delete_failed' | 'qdrant_upload_failed' | 'bulk_chunk_upload_failed' | 'group_chunks_updated' | 'group_chunks_action_failed' | 'crawl_completed' | 'crawl_failed';
export type EventTypeRequest = 'file_uploaded' | 'file_upload_failed' | 'chunks_uploaded' | 'chunk_action_failed' | 'chunk_updated' | 'bulk_chunks_deleted' | 'dataset_delete_failed' | 'qdrant_upload_failed' | 'bulk_chunk_upload_failed' | 'group_chunks_updated' | 'group_chunks_action_failed' | 'crawl_completed' | 'crawl_failed' | 'crawl_started';

export type EventTypes = {
/**
Expand Down Expand Up @@ -1471,6 +1471,28 @@ export type GetAllTagsResponse = {
total: number;
};

export type GetChunkGroupCountRequest = {
/**
* The Id of the group to get the count for, is not required if group_tracking_id is provided.
*/
group_id?: (string) | null;
/**
* The tracking id of the group to get the count for, is not required if group_id is provided.
*/
group_tracking_id?: (string) | null;
};

export type GetChunkGroupCountResponse = {
/**
* The count of chunks in the given group.
*/
count: number;
/**
* The Id of the group to get the count for.
*/
group_id: string;
};

export type GetChunksData = {
ids: Array<(string)>;
};
Expand Down Expand Up @@ -3548,6 +3570,19 @@ export type GetGroupsForChunksData = {

export type GetGroupsForChunksResponse = (Array<GroupsForChunk>);

export type CountGroupChunksData = {
/**
* JSON request payload to add a chunk to a group (bookmark it)
*/
requestBody: GetChunkGroupCountRequest;
/**
* The dataset id or tracking_id to use for the request. We assume you intend to use an id if the value is a valid uuid.
*/
trDataset: string;
};

export type CountGroupChunksResponse = (GetChunkGroupCountResponse);

export type SearchOverGroupsData = {
/**
* JSON request payload to semantically search over groups
Expand Down Expand Up @@ -4929,6 +4964,25 @@ export type $OpenApiTs = {
};
};
};
'/api/chunk_group/count': {
post: {
req: CountGroupChunksData;
res: {
/**
* JSON body representing the group with the count
*/
200: GetChunkGroupCountResponse;
/**
* Service error relating to getting the group with the given tracking id
*/
400: ErrorResponseBody;
/**
* Group not found
*/
404: ErrorResponseBody;
};
};
};
'/api/chunk_group/group_oriented_search': {
post: {
req: SearchOverGroupsData;
Expand Down
69 changes: 66 additions & 3 deletions frontends/search/src/components/OrgGroupPageView.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
import { FiTrash } from "solid-icons/fi";
import { isChunkGroupPageDTO, type ChunkGroupDTO } from "../utils/apiTypes";
import {
indirectHasOwnProperty,
isChunkGroupPageDTO,
type ChunkGroupDTO,
} from "../utils/apiTypes";
import {
For,
Setter,
Expand All @@ -20,20 +24,76 @@ export interface GroupUserPageViewProps {
setShowConfirmModal: Setter<boolean>;
}

export type GetChunkGroupCountResponse = {
count: number;
group_id: string;
};

export const GroupUserPageView = (props: GroupUserPageViewProps) => {
const apiHost = import.meta.env.VITE_API_HOST as string;
const datasetAndUserContext = useContext(DatasetAndUserContext);

const $dataset = datasetAndUserContext.currentDataset;
const $user = datasetAndUserContext.user;
const [groups, setGroups] = createSignal<ChunkGroupDTO[]>([]);
const [groupCounts, setGroupCounts] = createSignal<
GetChunkGroupCountResponse[]
>([]);
const [groupPage, setGroupPage] = createSignal(1);
const [groupPageCount, setGroupPageCount] = createSignal(1);
const [deleting, setDeleting] = createSignal(false);
const [loading, setLoading] = createSignal(true);

const serverConfig = useDatasetServerConfig();

createEffect(() => {
const currentDataset = $dataset?.();
if (!currentDataset) return;

const all_counts = groups().map(async (group) => {
const response = await fetch(`${apiHost}/chunk_group/count`, {
method: "POST",
credentials: "include",
body: JSON.stringify({ group_id: group.id }),
headers: {
"X-API-version": "2.0",
"TR-Dataset": currentDataset.dataset.id,
"Content-Type": "application/json",
},
});

if (response.ok) {
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
const data = await response.json();
console.log("data", data);
if (
data !== null &&
typeof data === "object" &&
indirectHasOwnProperty(data, "count") &&
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
typeof data.count === "number" &&
indirectHasOwnProperty(data, "group_id") &&
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
typeof data.group_id === "string"
) {
console.log("Invalid response", data);
return {
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access
group_id: data.group_id,
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access
count: data.count,
} as GetChunkGroupCountResponse;
}
}
});

void Promise.all(all_counts).then((counts) => {
const filteredGroupCounts = counts.filter((c) => c !== undefined);
console.log("setGroupCounts", filteredGroupCounts);
setGroupCounts(filteredGroupCounts);
});
});

createEffect(() => {
const userId = $user?.()?.id;
if (userId === undefined) return;
Expand Down Expand Up @@ -149,7 +209,7 @@ export const GroupUserPageView = (props: GroupUserPageViewProps) => {
scope="col"
class="px-3 py-3.5 text-left text-base font-semibold dark:text-white"
>
Description
Chunk Count
</th>
<th
scope="col"
Expand Down Expand Up @@ -181,7 +241,10 @@ export const GroupUserPageView = (props: GroupUserPageViewProps) => {
</a>
</td>
<td class="whitespace-nowrap text-wrap px-3 py-4 text-sm text-gray-900 dark:text-gray-300">
{group.description}
{
groupCounts().find((c) => c.group_id == group.id)
?.count
}
</td>
<td class="whitespace-nowrap px-3 py-4 text-left text-sm text-gray-900 dark:text-gray-300">
{getLocalTime(group.created_at).toLocaleDateString() +
Expand Down
38 changes: 20 additions & 18 deletions frontends/search/src/components/UploadFile.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -138,20 +138,20 @@ export const UploadFile = () => {
});

const requestBodyTemplate: Omit<RequestBody, "base64_file" | "file_name"> =
{
link: link() === "" ? undefined : link(),
tag_set:
tagSet().split(",").length > 0 ? undefined : tagSet().split(","),
split_delimiters: splitDelimiters(),
target_splits_per_chunk: targetSplitsPerChunk(),
rebalance_chunks: rebalanceChunks(),
use_pdf2md_ocr: useGptChunking(),
group_tracking_id:
groupTrackingId() === "" ? undefined : groupTrackingId(),
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
metadata: metadata(),
time_stamp: timestamp() ? timestamp() + " 00:00:00" : undefined,
};
{
link: link() === "" ? undefined : link(),
tag_set:
tagSet().split(",").length > 0 ? undefined : tagSet().split(","),
split_delimiters: splitDelimiters(),
target_splits_per_chunk: targetSplitsPerChunk(),
rebalance_chunks: rebalanceChunks(),
use_pdf2md_ocr: useGptChunking(),
group_tracking_id:
groupTrackingId() === "" ? undefined : groupTrackingId(),
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
metadata: metadata(),
time_stamp: timestamp() ? timestamp() + " 00:00:00" : undefined,
};

const uploadFilePromises = files().map(async (file) => {
let base64File = await toBase64(file);
Expand Down Expand Up @@ -343,10 +343,11 @@ export const UploadFile = () => {
</Show>
<div class="m-1 mb-1 flex flex-row gap-2">
<button
class={`rounded border-2 border-magenta p-2 px-4 font-semibold ${showFileInput()
class={`rounded border-2 border-magenta p-2 px-4 font-semibold ${
showFileInput()
? "bg-magenta-600 text-white"
: "text-magenta hover:bg-magenta-500 hover:text-white"
}`}
}`}
onClick={() => {
setFiles([]);
setShowFileInput(true);
Expand All @@ -356,10 +357,11 @@ export const UploadFile = () => {
Select Files
</button>
<button
class={`rounded border-2 border-magenta p-2 px-4 font-semibold ${showFolderInput()
class={`rounded border-2 border-magenta p-2 px-4 font-semibold ${
showFolderInput()
? "bg-magenta-600 text-white"
: "text-magenta hover:bg-magenta-500 hover:text-white"
}`}
}`}
onClick={() => {
setFiles([]);
setShowFolderInput(true);
Expand Down
Loading