feature: add the ability to decline JSONB prefilter in filter modal

devflowinc · Jun 20, 2024 · e12fc91 · e12fc91
1 parent 7257a5e
commit e12fc91
Show file tree

Hide file tree

Showing 5 changed files with 72 additions and 26 deletions.
diff --git a/search/src/components/FilterModal.tsx b/search/src/components/FilterModal.tsx
@@ -10,6 +10,8 @@ import {
   createMemo,
 } from "solid-js";
 import { DatasetAndUserContext } from "./Contexts/DatasetAndUserContext";
+import { Tooltip } from "./Atoms/Tooltip";
+import { BsQuestionCircle } from "solid-icons/bs";
 
 export interface Filter {
   field: string;
@@ -39,6 +41,7 @@ export interface Filters {
   must: Filter[];
   must_not: Filter[];
   should: Filter[];
+  jsonb_prefilter?: boolean | null;
 }
 
 export interface FilterModalProps {
@@ -57,6 +60,7 @@ export const FilterModal = (props: FilterModalProps) => {
   const [mustFilters, setMustFilters] = createSignal<Filter[]>([]);
   const [mustNotFilters, setMustNotFilters] = createSignal<Filter[]>([]);
   const [shouldFilters, setShouldFilters] = createSignal<Filter[]>([]);
+  const [jsonbPrefilter, setJsonbPrefilter] = createSignal<boolean>(true);
 
   const curDatasetFiltersKey = createMemo(
     () =>
@@ -68,6 +72,7 @@ export const FilterModal = (props: FilterModalProps) => {
       must: mustFilters(),
       must_not: mustNotFilters(),
       should: shouldFilters(),
+      jsonb_prefilter: jsonbPrefilter(),
     };
     localStorage.setItem(curDatasetFiltersKey(), JSON.stringify(filters));
     window.dispatchEvent(new Event("filtersUpdated"));
@@ -86,6 +91,7 @@ export const FilterModal = (props: FilterModalProps) => {
       setMustFilters(parsedFilters.must);
       setMustNotFilters(parsedFilters.must_not);
       setShouldFilters(parsedFilters.should);
+      setJsonbPrefilter(parsedFilters.jsonb_prefilter ?? true);
     }
   }, "");
 
@@ -137,6 +143,26 @@ export const FilterModal = (props: FilterModalProps) => {
           + Add Filter
         </button>
         <div class="flex-1" />
+        <label
+          aria-label="Change JSONB Prefilter"
+          class="flex items-center gap-x-1"
+        >
+          <Tooltip
+            body={
+              <BsQuestionCircle class="h-4 w-4 rounded-full fill-current" />
+            }
+            tooltipText="Only uncheck if on the enterprise plan and you wish to use custom indices for metadata filters."
+          />
+          <span>JSONB Prefilter:</span>
+        </label>
+        <input
+          type="checkbox"
+          class="rounded-md border border-neutral-400 bg-neutral-100 dark:border-neutral-900 dark:bg-neutral-800"
+          onChange={(e) => {
+            setJsonbPrefilter(e.currentTarget.checked);
+          }}
+          checked={jsonbPrefilter()}
+        />
         <button
           class="rounded-md border border-neutral-400 bg-neutral-100 p-1 dark:border-neutral-900 dark:bg-neutral-800"
           onClick={() => {

diff --git a/search/yarn.lock b/search/yarn.lock
@@ -1091,9 +1091,9 @@ camelcase-css@^2.0.1:
   integrity sha512-QOSvevhslijgYwRx6Rv7zKdMF8lbRmx+uQGx2+vDc+KI/eBnsy9kit5aj23AgGu3pa4t9AgwbnXWqS+iOY+2aA==
 
 caniuse-lite@^1.0.30001538, caniuse-lite@^1.0.30001565:
-  version "1.0.30001576"
-  resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001576.tgz#893be772cf8ee6056d6c1e2d07df365b9ec0a5c4"
-  integrity sha512-ff5BdakGe2P3SQsMsiqmt1Lc8221NR1VzHj5jXN5vBny9A6fpze94HiVV/n7XRosOlsShJcvMv5mdnpjOGCEgg==
+  version "1.0.30001636"
+  resolved "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001636.tgz"
+  integrity sha512-bMg2vmr8XBsbL6Lr0UHXy/21m84FTxDLWn2FSqMd5PrlbMxwJlQnC2YWYxVgp66PZE+BBNF2jYQUBKCo1FDeZg==
 
 chalk@^2.4.2:
   version "2.4.2"
@@ -2834,8 +2834,16 @@ source-map-js@^1.0.2:
   resolved "https://registry.yarnpkg.com/source-map-js/-/source-map-js-1.0.2.tgz#adbc361d9c62df380125e7f161f71c826f1e490c"
   integrity sha512-R0XvVJ9WusLiqTCEiGCmICCMplcCkIwwR11mOSD9CR5u+IXYdiseeEuXCVAjS54zqwkLcPNnmU4OeJ6tUrWhDw==
 
-"string-width-cjs@npm:string-width@^4.2.0", string-width@^4.1.0:
-  name string-width-cjs
+"string-width-cjs@npm:string-width@^4.2.0":
+  version "4.2.3"
+  resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010"
+  integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==
+  dependencies:
+    emoji-regex "^8.0.0"
+    is-fullwidth-code-point "^3.0.0"
+    strip-ansi "^6.0.1"
+
+string-width@^4.1.0:
   version "4.2.3"
   resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010"
   integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==
@@ -2880,7 +2888,14 @@ string.prototype.trimstart@^1.0.7:
     define-properties "^1.2.0"
     es-abstract "^1.22.1"
 
-"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1:
+"strip-ansi-cjs@npm:strip-ansi@^6.0.1":
+  version "6.0.1"
+  resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9"
+  integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==
+  dependencies:
+    ansi-regex "^5.0.1"
+
+strip-ansi@^6.0.0, strip-ansi@^6.0.1:
   version "6.0.1"
   resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9"
   integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==
@@ -3242,4 +3257,4 @@ yaml@^2.3.4:
 yocto-queue@^0.1.0:
   version "0.1.0"
   resolved "https://registry.yarnpkg.com/yocto-queue/-/yocto-queue-0.1.0.tgz#0294eb3dee05028d31ee1a5fa2c556a6aaf10a1b"
-  integrity sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==
+  integrity sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==
diff --git a/server/src/data/models.rs b/server/src/data/models.rs
@@ -2765,16 +2765,17 @@ impl FieldCondition {
     pub async fn convert_to_qdrant_condition(
         &self,
         condition_type: &str,
-        pool: web::Data<Pool>,
+        jsonb_prefilter: Option<bool>,
         dataset_id: uuid::Uuid,
+        pool: web::Data<Pool>,
     ) -> Result<Option<qdrant::Condition>, ServiceError> {
         if self.r#match.is_some() && self.range.is_some() {
             return Err(ServiceError::BadRequest(
                 "Cannot have both match and range conditions".to_string(),
             ));
         }
 
-        if self.field.starts_with("metadata.") {
+        if jsonb_prefilter.unwrap_or(true) && self.field.starts_with("metadata.") {
             return Ok(Some(
                 get_metadata_filter_condition(self, dataset_id, pool)
                     .await?

diff --git a/server/src/handlers/chunk_handler.rs b/server/src/handlers/chunk_handler.rs
@@ -850,6 +850,8 @@ pub struct ChunkFilter {
     pub must: Option<Vec<ConditionType>>,
     /// None of these field conditions can match for the chunk to be included in the result set.
     pub must_not: Option<Vec<ConditionType>>,
+    /// JOSNB prefilter tells the server to perform a full scan over the metadata JSONB column instead of using the filtered HNSW. Datasets on the enterprise plan with custom metadata indices will perform better with the filtered HNSW instead. When false, the server will use the filtered HNSW index to filter chunks. When true, the server will perform a full scan over the metadata JSONB column to filter chunks. Default is true.
+    pub jsonb_prefilter: Option<bool>,
 }
 
 #[derive(Serialize, Deserialize, Clone, Debug, ToSchema)]
@@ -863,18 +865,11 @@ pub struct ChunkFilter {
             {
                 "field": "metadata.key1",
                 "match": ["value1", "value2"],
-                "range": {
-                    "gte": 0.0,
-                    "lte": 1.0,
-                    "gt": 0.0,
-                    "lt": 1.0
-                }
             }
         ],
         "must": [
             {
-                "field": "metadata.key2",
-                "match": ["value3", "value4"],
+                "field": "num_value",
                 "range": {
                     "gte": 0.0,
                     "lte": 1.0,
@@ -887,12 +882,6 @@ pub struct ChunkFilter {
             {
                 "field": "metadata.key3",
                 "match": ["value5", "value6"],
-                "range": {
-                    "gte": 0.0,
-                    "lte": 1.0,
-                    "gt": 0.0,
-                    "lt": 1.0
-                }
             }
         ]
     },

diff --git a/server/src/operators/search_operator.rs b/server/src/operators/search_operator.rs
@@ -153,7 +153,12 @@ pub async fn assemble_qdrant_filter(
                                 .await?;
 
                         let qdrant_condition = should_condition
-                            .convert_to_qdrant_condition("should", pool.clone(), dataset_id)
+                            .convert_to_qdrant_condition(
+                                "should",
+                                filters.jsonb_prefilter,
+                                dataset_id,
+                                pool.clone(),
+                            )
                             .await?;
 
                         if let Some(condition) = qdrant_condition {
@@ -178,7 +183,12 @@ pub async fn assemble_qdrant_filter(
                                 .await?;
 
                         let qdrant_condition = must_condition
-                            .convert_to_qdrant_condition("must", pool.clone(), dataset_id)
+                            .convert_to_qdrant_condition(
+                                "must",
+                                filters.jsonb_prefilter,
+                                dataset_id,
+                                pool.clone(),
+                            )
                             .await?;
 
                         if let Some(condition) = qdrant_condition {
@@ -203,7 +213,12 @@ pub async fn assemble_qdrant_filter(
                                 .await?;
 
                         let qdrant_condition = must_not_condition
-                            .convert_to_qdrant_condition("must_not", pool.clone(), dataset_id)
+                            .convert_to_qdrant_condition(
+                                "must_not",
+                                filters.jsonb_prefilter,
+                                dataset_id,
+                                pool.clone(),
+                            )
                             .await?;
 
                         if let Some(condition) = qdrant_condition {