Working Examples

packages/semantic-search/package.json

@@ -9,6 +9,10 @@
 		".": {
 			"types": "./src/index.ts",
 			"default": "./dist/index.mjs"
+		},
+		"./lancedb": {
+			"types": "./src/components/lancedb/index.ts",
+			"default": "./dist/components/lancedb/index.mjs"
 		}
 	},
 	"scripts": {
@@ -20,16 +24,16 @@
 		"postinstall": "pnpm build:stub"
 	},
 	"devDependencies": {
-		"@lancedb/lancedb": "0.10.0",
+
 		"@payloadcms/db-mongodb": "^3.0.0-beta.107",
 		"@payloadcms/db-postgres": "^3.0.0-beta.107",
-		"@types/dockerode": "^3.3.31",
-		"apache-arrow": "^17.0.0",
 		"dockerode": "^4.0.2",
-		"mongodb-memory-server": "^10.0.0",
-		"ollama": "^0.5.9"
+		"mongodb-memory-server": "^10.0.0"
 	},
 	"dependencies": {
+		"apache-arrow": "^17.0.0",
+		"ollama": "^0.5.9",
+		"@lancedb/lancedb": "0.10.0",
 		"@workspace/llm-utils": "workspace:*",
 		"defu": "^6.1.4",
 		"pathe": "^1.1.2",

packages/semantic-search/src/components/lancedb/index.ts

@@ -1,8 +1,10 @@
-import { type Table, connect } from '@lancedb/lancedb'
-import { LanceSchema } from '@lancedb/lancedb/embedding'
-import { Utf8 } from 'apache-arrow'
-import type { Identifier, InsertFields, VectorDB } from '../../types'
-import { OllamaEmbeddings } from '../ollama'
+import {connect, type Table} from '@lancedb/lancedb'
+import {getRegistry, LanceSchema} from '@lancedb/lancedb/embedding'
+import {Utf8} from 'apache-arrow'
+import type {Identifier, InsertFields, VectorDB} from '../../types'
+import {OllamaEmbeddings, register} from '../ollama'
+
+register()
 
 export class LanceDB implements VectorDB {
 	public name = 'lancedb'
@@ -23,6 +25,7 @@ export class LanceDB implements VectorDB {
 	}
 
 	static async create(path = './lancedb') {
+
 		const func = new OllamaEmbeddings({
 			host: 'http://100.67.29.127:11434',
 			model: 'nomic-embed-text',

packages/semantic-search/src/components/ollama/index.ts

@@ -1,19 +1,13 @@
-import { embedding } from '@lancedb/lancedb'
-import {
-	EmbeddingFunction,
-	TextEmbeddingFunction,
-	getRegistry,
-} from '@lancedb/lancedb/embedding'
-import type { Float } from 'apache-arrow'
-import { Ollama } from 'ollama'
+import {getRegistry, TextEmbeddingFunction,} from '@lancedb/lancedb/embedding'
+import type {Float} from 'apache-arrow'
+import {Ollama} from 'ollama'
 
 interface Options {
 	model: string
 	timeout: number
 	host: string
 }
-// @ts-ignore
-@embedding.register('ollama')
+
 export class OllamaEmbeddings extends TextEmbeddingFunction<Partial<Options>> {
 	private client: Ollama
 	constructor(private modelOptions: Options) {

packages/semantic-search/src/hooks/afterChangeHook.ts

@@ -1,6 +1,6 @@
-import type { FieldHook, FieldHookArgs } from 'payload'
-import { isObject, isString } from 'radash'
-import { getSemanticSearchCustom } from '../utils/customContext'
+import type {FieldHook, FieldHookArgs} from 'payload'
+import {isObject, isString} from 'radash'
+import {getSemanticSearchCustom} from '../utils/customContext'
 
 export const afterChangeHook: FieldHook = (args) => {
 	if (args.operation === 'create' || args.operation === 'update') {

packages/semantic-search/src/index.ts

@@ -1,9 +1,9 @@
 import defu from 'defu'
-import type { Config, FieldBase } from 'payload'
-import { afterChangeHook } from './hooks/afterChangeHook'
-import type { SemanticSearchPluginConfig } from './types'
-import { setupSemanticSearchCustom } from './utils/customContext'
-import { getField, parseFields } from './utils/fields'
+import type {Config, FieldBase} from 'payload'
+import {afterChangeHook} from './hooks/afterChangeHook'
+import type {SemanticSearchPluginConfig} from './types'
+import {setupSemanticSearchCustom} from './utils/customContext'
+import {getField, parseFields} from './utils/fields'
 
 export const semanticSearchPlugin =
 	(incomingPluginConfig: SemanticSearchPluginConfig) =>
@@ -16,7 +16,6 @@ export const semanticSearchPlugin =
 
 		return setupSemanticSearchCustom(config, {
 			vectorDB: incomingPluginConfig.vectorDB,
-			embeddingFn: incomingPluginConfig.vectorDB,
 		})
 	}
 

packages/semantic-search/src/utils/customContext.ts

@@ -1,10 +1,9 @@
-import type { Config } from 'payload'
-import { isObject } from 'radash'
-import type { SemanticSearchPluginConfig, VectorDB } from '../types'
+import type {Config} from 'payload'
+import {isObject} from 'radash'
+import type {VectorDB} from '../types'
 
 export type SemanticSearchCustomObject = {
 	vectorDB: VectorDB
-	embeddingFn: SemanticSearchPluginConfig['embeddingFn']
 }
 
 export const CUSTOMKEY = 'semantic-search'

packages/semantic-search/src/utils/fields.ts

@@ -1,9 +1,4 @@
-import type {
-	CollectionConfig,
-	Config,
-	TextField,
-	TextareaField,
-} from 'payload'
+import type {CollectionConfig, Config, TextareaField, TextField,} from 'payload'
 
 export interface FieldEntry {
 	collectionConfig: CollectionConfig

playground/.gitignore

@@ -8,3 +8,4 @@ node_modules
 
 # Payload default media upload directory
 public/media/
+db/

playground/next.config.js

@@ -1,11 +1,16 @@
-import { withPayload } from '@payloadcms/next/withPayload'
+import {withPayload} from '@payloadcms/next/withPayload'
 
 import redirects from './redirects.js'
 
 const NEXT_PUBLIC_SERVER_URL = process.env.NEXT_PUBLIC_SERVER_URL || 'http://localhost:3000'
 
 /** @type {import('next').NextConfig} */
 const nextConfig = {
+  webpack(config) {
+    config.externals.push('@lancedb/lancedb')
+    return config;
+  },
+    serverExternalPackages:['@payload-llm-plugins/semantic-search','@lancedb/lancedb'],
   images: {
     remotePatterns: [
       ...[NEXT_PUBLIC_SERVER_URL /* 'https://example.com' */].map((item) => {

playground/src/payload.config.ts

@@ -30,6 +30,7 @@ import {Header} from './Header/config'
 import {revalidateRedirects} from './hooks/revalidateRedirects'
 import {GenerateTitle, GenerateURL} from '@payloadcms/plugin-seo/types'
 import {Page, Post} from './payload-types'
+import {LanceDB} from "@payload-llm-plugins/semantic-search/lancedb";
 
 const filename = fileURLToPath(import.meta.url)
 const dirname = path.dirname(filename)
@@ -188,7 +189,7 @@ export default buildConfig({
       },
     }),
     payloadCloudPlugin(), // storage-adapter-placeholder
-    semanticSearchPlugin({enabled:true, vectorDB: LanceDB.create('')})
+    semanticSearchPlugin({enabled:true, vectorDB: await LanceDB.create('./db/lancedb/'), dimensions:768, indexableFields:['posts.content'] })
   ],
   secret: process.env.PAYLOAD_SECRET!,
   sharp,