Implement Semantic Search

app.arc

@@ -156,6 +156,8 @@ availabilityZoneCount 3
 volumeSize 10
 dedicatedMasterCount 3
 dedicatedMasterType t3.small.search
+# Use OpenSearch in sandbox mode; default is Elasticsearch.
+sandboxEngine opensearch
 
 @plugins
 plugin-remix

app/routes/_gcn.circulars._archive._index/route.tsx

@@ -38,12 +38,14 @@ import CircularsHeader from './CircularsHeader'
 import CircularsIndex from './CircularsIndex'
 import { DateSelector } from './DateSelectorMenu'
 import Hint from '~/components/Hint'
+import { feature } from '~/lib/env.server'
 import { getFormDataString } from '~/lib/utils'
 import { useFeature } from '~/root'
 
 import searchImg from 'nasawds/src/img/usa-icons-bg/search--white.svg'
 
 export async function loader({ request: { url } }: LoaderFunctionArgs) {
+ const useNLP = feature('CIRCULARS_USE_NLP')
  const { searchParams } = new URL(url)
  const query = searchParams.get('query') || undefined
  if (query) {
@@ -59,6 +61,7 @@ export async function loader({ request: { url } }: LoaderFunctionArgs) {
  limit,
  startDate,
  endDate,
+ useNLP,
  })
 
  return { page, ...results }

app/routes/_gcn.circulars/circulars.server.ts

@@ -127,41 +127,57 @@ export async function search({
  limit,
  startDate,
  endDate,
+ useNLP,
 }: {
  query?: string
  page?: number
  limit?: number
  startDate?: string
  endDate?: string
+ useNLP?: boolean
 }): Promise<{
  items: CircularMetadata[]
  totalPages: number
  totalItems: number
 }> {
  const client = await getSearch()
-
  const [startTime, endTime] = getValidDates(startDate, endDate)
 
- const {
- body: {
- hits: {
- total: { value: totalItems },
- hits,
- },
- },
- } = await client.search({
- index: 'circulars',
- body: {
- query: {
+ const get_model_id_request = {
+ path: '/_ingest/pipeline',
+ }
+
+ let model_id = ''
+ try {
+ const resp = await client.http.get(get_model_id_request)
+
+ if (resp && resp.statusCode == 200) {
+ model_id =
+ resp.body['nlp-ingest-pipeline'].processors[0].text_embedding.model_id
+ } else {
+ console.log(
+ 'Error. Could not deploy model. Returned with response: ',
+ resp
+ )
+ }
+ } catch (e) {
+ console.log('Error: ', e)
+ }
+
+ const nlpSearchQuery = query
+ ? {
  bool: {
- must: query
- ? {
- multi_match: {
- query,
- fields: ['submitter', 'subject', 'body'],
+ must: [
+ {
+ neural: {
+ circular_embedding: {
+ query_text: query,
+ model_id,
+ k: 100,
  },
- }
- : undefined,
+ },
+ },
+ ],
  filter: {
  range: {
  createdOn: {
@@ -171,7 +187,42 @@ export async function search({
  },
  },
  },
+ }
+ : { match_all: {} }
+
+ const searchQuery = {
+ bool: {
+ must: query
+ ? {
+ multi_match: {
+ query,
+ fields: ['submitter', 'subject', 'body'],
+ },
+ }
+ : undefined,
+ filter: {
+ range: {
+ createdOn: {
+ gte: startTime,
+ lte: endTime,
+ },
+ },
+ },
+ },
+ }
+
+ const chosenQuery = useNLP ? nlpSearchQuery : searchQuery
+ const {
+ body: {
+ hits: {
+ total: { value: totalItems },
+ hits,
  },
+ },
+ } = await client.search({
+ index: 'circulars',
+ body: {
+ query: chosenQuery,
  fields: ['subject'],
  _source: false,
  sort: {