Merge pull request #274 from zilliztech/dynamic_field

support dynamic field

client/src/consts/Milvus.ts

@@ -3,6 +3,8 @@ import { DataTypeEnum } from '@/pages/collections/Types';
 export const MILVUS_URL =
   ((window as any)._env_ && (window as any)._env_.MILVUS_URL) || '';
 
+export const DYNAMIC_FIELD = `$meta`;
+
 export enum METRIC_TYPES_VALUES {
   L2 = 'L2',
   IP = 'IP',

client/src/http/BaseModel.ts

@@ -50,7 +50,7 @@ export default class BaseModel {
     } as any;
     if (timeout) httpConfig.timeout = timeout;
     const res = await http(httpConfig);
-    return res.data.data;
+    return new this(res.data.data || {});
   }
 
   /**

client/src/http/Collection.ts

@@ -19,7 +19,7 @@ import { LOADING_STATE } from '@/consts';
 
 export class CollectionHttp extends BaseModel implements CollectionView {
   private aliases!: string[];
-  private autoID!: string;
+  private autoID!: boolean;
   private collection_name!: string;
   private description!: string;
   private consistency_level!: string;
@@ -30,6 +30,9 @@ export class CollectionHttp extends BaseModel implements CollectionView {
   private createdTime!: string;
   private schema!: {
     fields: Field[];
+    autoID: boolean;
+    description: string;
+    enable_dynamic_field: boolean;
   };
   private replicas!: Replica[];
 
@@ -52,7 +55,7 @@ export class CollectionHttp extends BaseModel implements CollectionView {
     return super.search({
       path: `${this.COLLECTIONS_URL}/${name}`,
       params: {},
-    });
+    }) as Promise<CollectionHttp>;
   }
 
   static createCollection(data: any) {
@@ -233,5 +236,12 @@ export class CollectionHttp extends BaseModel implements CollectionView {
   get _replicas(): Replica[] {
     return this.replicas;
   }
-
+
+  get _enableDynamicField(): boolean {
+    return this.schema.enable_dynamic_field;
+  }
+
+  get _schema() {
+    return this.schema;
+  }
 }

client/src/i18n/en/collection.ts

@@ -2,7 +2,7 @@ const collectionTrans = {
   noLoadData: 'No Loaded Collection',
   noData: 'No Collection',
 
-  rowCount: 'Approx Entity Count',
+  rowCount: 'Approx Count',
 
   create: 'Collection',
   delete: 'delete',
@@ -22,11 +22,13 @@ const collectionTrans = {
   // table
   id: 'ID',
   name: 'Name',
+  features: 'Features',
   nameTip: 'Collection Name',
   status: 'Status',
   desc: 'Description',
   createdTime: 'Created Time',
   maxLength: 'Max Length',
+  dynmaicSchema: 'Dynamic Schema',
 
   // table tooltip
   aliasInfo: 'Alias can be used as collection name in vector search.',
@@ -38,7 +40,7 @@ const collectionTrans = {
   createTitle: 'Create Collection',
   general: 'General information',
   schema: 'Schema',
-  consistency: 'Consistency Level',
+  consistency: 'Consistency',
   consistencyLevel: 'Consistency Level',
   description: 'Description',
   fieldType: 'Type',
@@ -63,6 +65,7 @@ const collectionTrans = {
   partitionKey: 'Partition Key',
   partitionKeyTooltip:
     ' Milvus will store entities in a partition according to the values in the partition key field. Only one Int64 or VarChar field is supported.',
+  enableDynamicSchema: 'Enable Dynamic Schema',
 
   // load dialog
   loadTitle: 'Load Collection',
@@ -116,6 +119,15 @@ const collectionTrans = {
   compact: 'Compact',
   compactDialogInfo: `Compaction is a process that optimizes storage and query performance by organizing segments.  <a href='https://milvus.io/blog/2022-2-21-compact.md' target="_blank">Learn more</a><br /><br />  Please note that this operation may take some time to complete, especially for large datasets. We recommend running compaction during periods of lower system activity or during scheduled maintenance to minimize disruption.
     `,
+
+  // column tooltip
+  autoIDTooltip: `The values of the primary key column are automatically generated by Milvus.`,
+  dynamicSchemaTooltip: `Dynamic schema enables users to insert entities with new fields into a Milvus collection without modifying the existing schema.`,
+  consistencyLevelTooltip: `Consistency in a distributed database specifically refers to the property that ensures every node or replica has the same view of data when writing or reading data at a given time.`,
+  consistencyBoundedTooltip: `It allows data inconsistency during a certain period of time`,
+  consistencyStrongTooltip: `It ensures that users can read the latest version of data.`,
+  consistencySessionTooltip: `It ensures that all data writes can be immediately perceived in reads during the same session.`,
+  consistencyEventuallyTooltip: `There is no guaranteed order of reads and writes, and replicas eventually converge to the same state given that no further write operations are done.`,
 };
 
 export default collectionTrans;

client/src/i18n/en/search.ts

@@ -14,6 +14,7 @@ const searchTrans = {
   vectorValueWarning: 'Vector value should be an array of length {{dimension}}',
   timeTravel: 'Time Travel',
   timeTravelPrefix: 'Data before',
+  dynamicFields: 'Dynamic Fields',
 };
 
 export default searchTrans;

client/src/pages/collections/Collections.tsx

@@ -1,6 +1,6 @@
 import { useCallback, useContext, useEffect, useMemo, useState } from 'react';
 import { Link, useSearchParams } from 'react-router-dom';
-import { makeStyles, Theme } from '@material-ui/core';
+import { makeStyles, Theme, Chip, Tooltip } from '@material-ui/core';
 import { useTranslation } from 'react-i18next';
 import Highlighter from 'react-highlight-words';
 import {
@@ -62,6 +62,10 @@ const useStyles = makeStyles((theme: Theme) => ({
     color: theme.palette.primary.main,
     backgroundColor: 'transparent',
   },
+  chip: {
+    color: theme.palette.text.primary,
+    marginRight: theme.spacing(0.5),
+  },
 }));
 
 const Collections = () => {
@@ -90,6 +94,13 @@ const Collections = () => {
   const InfoIcon = icons.info;
   const SourceIcon = icons.source;
 
+  const consistencyTooltipsMap: Record<string, string> = {
+    Strong: collectionTrans('consistencyStrongTooltip'),
+    Bounded: collectionTrans('consistencyBoundedTooltip'),
+    Session: collectionTrans('consistencySessionTooltip'),
+    Eventually: collectionTrans('consistencyEventuallyTooltip'),
+  };
+
   const fetchData = useCallback(async () => {
     try {
       setLoading(true);
@@ -137,6 +148,47 @@ const Collections = () => {
             />
           </Link>
         ),
+        features: (
+          <>
+            {v._autoId ? (
+              <Tooltip
+                title={collectionTrans('autoIDTooltip')}
+                placement="top"
+                arrow
+              >
+                <Chip
+                  className={classes.chip}
+                  label={collectionTrans('autoID')}
+                  size="small"
+                />
+              </Tooltip>
+            ) : null}
+            {v._enableDynamicField ? (
+              <Tooltip
+                title={collectionTrans('dynamicSchemaTooltip')}
+                placement="top"
+                arrow
+              >
+                <Chip
+                  className={classes.chip}
+                  label={collectionTrans('dynmaicSchema')}
+                  size="small"
+                />
+              </Tooltip>
+            ) : null}
+            <Tooltip
+              title={consistencyTooltipsMap[v._consistencyLevel]}
+              placement="top"
+              arrow
+            >
+              <Chip
+                className={classes.chip}
+                label={v._consistencyLevel}
+                size="small"
+              />
+            </Tooltip>
+          </>
+        ),
         statusElement: (
           <Status status={v._status} percentage={v._loadedPercentage} />
         ),
@@ -360,6 +412,13 @@ const Collections = () => {
       sortBy: '_status',
       label: collectionTrans('status'),
     },
+    {
+      id: 'features',
+      align: 'left',
+      disablePadding: true,
+      sortBy: '_enableDynamicField',
+      label: collectionTrans('features'),
+    },
     {
       id: '_rowCount',
       align: 'left',
@@ -374,19 +433,19 @@ const Collections = () => {
       ),
     },
 
-    {
-      id: 'consistency_level',
-      align: 'left',
-      disablePadding: false,
-      label: (
-        <span className="flex-center">
-          {collectionTrans('consistencyLevel')}
-          <CustomToolTip title={collectionTrans('consistencyLevelInfo')}>
-            <InfoIcon classes={{ root: classes.icon }} />
-          </CustomToolTip>
-        </span>
-      ),
-    },
+    // {
+    //   id: 'consistency_level',
+    //   align: 'left',
+    //   disablePadding: true,
+    //   label: (
+    //     <span className="flex-center">
+    //       {collectionTrans('consistency')}
+    //       <CustomToolTip title={collectionTrans('consistencyLevelInfo')}>
+    //         <InfoIcon classes={{ root: classes.icon }} />
+    //       </CustomToolTip>
+    //     </span>
+    //   ),
+    // },
 
     {
       id: '_desc',
@@ -480,7 +539,7 @@ const Collections = () => {
   ];
 
   if (!isManaged) {
-    colDefinitions.splice(3, 0, {
+    colDefinitions.splice(4, 0, {
       id: '_aliasElement',
       align: 'left',
       disablePadding: false,

client/src/pages/collections/Constants.ts

@@ -18,10 +18,6 @@ export const CONSISTENCY_LEVEL_OPTIONS: KeyValuePair[] = [
     label: 'Eventually',
     value: ConsistencyLevelEnum.Eventually,
   },
-  {
-    label: 'Customized',
-    value: ConsistencyLevelEnum.Customized,
-  },
 ];
 
 export const VECTOR_FIELDS_OPTIONS: KeyValuePair[] = [

client/src/pages/collections/Types.ts

@@ -12,9 +12,11 @@ export interface CollectionData {
   _desc: string;
   _indexState: ChildrenStatusType;
   _fields?: FieldData[];
-  _consistencyLevel?: string;
+  _consistencyLevel: string;
   _aliases: string[];
   _replicas: Replica[];
+  _enableDynamicField: boolean;
+  _autoId: boolean;
 }
 
 export interface Replica {

client/src/pages/dialogs/CreateCollectionDialog.tsx

@@ -1,5 +1,10 @@
-import { makeStyles, Theme } from '@material-ui/core';
-import { FC, useContext, useMemo, useState } from 'react';
+import {
+  makeStyles,
+  Theme,
+  Checkbox,
+  FormControlLabel,
+} from '@material-ui/core';
+import { FC, useContext, useMemo, useState, ChangeEvent } from 'react';
 import { useTranslation } from 'react-i18next';
 import DialogTemplate from '@/components/customDialog/DialogTemplate';
 import CustomInput from '@/components/customInput/CustomInput';
@@ -25,10 +30,10 @@ const useStyles = makeStyles((theme: Theme) => ({
     display: 'flex',
     alignItems: 'center',
     marginBottom: '16px',
-    gap: '8px',
-    '&:nth-last-child(2)': {
+    '&:nth-last-child(3)': {
       flexDirection: 'column',
       alignItems: 'flex-start',
+      marginBottom: '0',
     },
 
     '& legend': {
@@ -64,6 +69,7 @@ const CreateCollectionDialog: FC<CollectionCreateProps> = ({ onCreate }) => {
     collection_name: '',
     description: '',
     autoID: true,
+    enableDynamicField: false,
   });
 
   const [consistencyLevel, setConsistencyLevel] =
@@ -117,6 +123,16 @@ const CreateCollectionDialog: FC<CollectionCreateProps> = ({ onCreate }) => {
     });
   };
 
+  const changeEnableDynamicField = (
+    event: ChangeEvent<any>,
+    value: boolean
+  ) => {
+    setForm({
+      ...form,
+      enableDynamicField: value,
+    });
+  };
+
   const handleInputChange = (key: string, value: string) => {
     setForm(v => ({ ...v, [key]: value }));
   };
@@ -263,6 +279,14 @@ const CreateCollectionDialog: FC<CollectionCreateProps> = ({ onCreate }) => {
             setAutoID={changeIsAutoID}
           />
         </fieldset>
+        <fieldset className={classes.fieldset}>
+          <FormControlLabel
+            checked={form.enableDynamicField}
+            control={<Checkbox />}
+            onChange={changeEnableDynamicField}
+            label={collectionTrans('enableDynamicSchema')}
+          />
+        </fieldset>
 
         <fieldset className={classes.fieldset}>
           <legend>{collectionTrans('consistency')}</legend>