0.9.0 changes

.gitignore

@@ -158,8 +158,11 @@ cython_debug/
 # Vector search test files
 tests/siftsmall/*
 tests/siftsmall.tar.gz
+tests/features.conf
+tests/assets/features.conf
 
 # Notes
 notes.txt
 
-public-key.asc
+public-key.asc
+

docs/aio.rst

@@ -1,5 +1,5 @@
-aio Module
-=====================
+Asynchronous Clients leveraging Asyncio
+========================================
 
 This module contains clients with coroutine methods used for asynchronous programming.
 

docs/conf.py

@@ -6,24 +6,24 @@
 # -- Project information -----------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
 
-project = 'aerospike-vector-search'
-copyright = '2024, Dominic Pelini'
-author = 'Dominic Pelini'
-release = '0.6.1'
+project = "aerospike-vector-search"
+copyright = "2024, Dominic Pelini"
+author = "Dominic Pelini"
+release = "0.6.1"
 
 # -- General configuration ---------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
 
 extensions = [
- 'sphinx.ext.autodoc',
- 'sphinx_rtd_theme',
+ "sphinx.ext.autodoc",
+ "sphinx_rtd_theme",
 ]
 
-templates_path = ['_templates']
-exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+templates_path = ["_templates"]
+exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
 
 # -- Options for HTML output -------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
 
-html_theme = 'sphinx_rtd_theme'
-html_static_path = ['_static']
+html_theme = "sphinx_rtd_theme"
+html_static_path = ["_static"]

docs/index.rst

@@ -23,8 +23,7 @@ Please explore the modules below for more information on API usage and details.
  :caption: Contents:
 
  aio
- admin
- client
+ sync
  types
 
 

docs/sync.rst

@@ -0,0 +1,12 @@
+Synchronous Clients
+=====================
+
+This module contains clients with coroutine methods used for asynchronous programming.
+
+
+.. toctree::
+ :maxdepth: 2
+ :caption: Contents:
+
+ admin
+ client

proto/auth.proto

@@ -3,22 +3,24 @@ syntax = "proto3";
 package aerospike.vector;
 
 option go_package = "aerospike.com/vector/protos/";
-option java_package = "com.aerospike.vector.client";
+option java_package = "com.aerospike.vector.client.proto";
 option java_multiple_files = true;
 
-// Auth service
-service AuthService {
- rpc Get(AerospikeAuthRequest) returns (AerospikeAuthResponse) {}
-}
+import "types.proto";
 
 // An auth request to get an access token to perform operations on Aerospike
 // database.
-message AerospikeAuthRequest {
- string username = 1;
- string password = 2;
+message AuthRequest {
+ Credentials credentials = 1;
 }
 
 // An auth token to perform operations on Aerospike database.
-message AerospikeAuthResponse {
+message AuthResponse {
  string token = 1;
 }
+
+// Auth service
+service AuthService {
+ // Request authentication.
+ rpc Authenticate(AuthRequest) returns (AuthResponse) {}
+}

proto/index.proto

@@ -3,7 +3,7 @@ syntax = "proto3";
 package aerospike.vector;
 
 option go_package = "aerospike.com/vector/protos/";
-option java_package = "com.aerospike.vector.client";
+option java_package = "com.aerospike.vector.client.proto";
 option java_multiple_files = true;
 
 import "google/protobuf/empty.proto";
@@ -14,11 +14,32 @@ message IndexStatusResponse {
  int64 unmergedRecordCount = 2;
 }
 
+message GcInvalidVerticesRequest {
+ IndexId indexId = 1;
+
+ // Vertices identified as invalid before cutoff timestamp (Unix timestamp) are garbage collected.
+ int64 cutoffTimestamp = 2;
+}
+
+message IndexUpdateRequest {
+ IndexId indexId = 1;
+
+ // Optional labels associated with the index.
+ map<string, string> labels = 2;
+
+ oneof update {
+ HnswIndexUpdate hnswIndexUpdate = 3;
+ }
+}
+
 // Service to manage indices.
 service IndexService {
  // Create an index.
  rpc Create(IndexDefinition) returns (google.protobuf.Empty) {}
 
+ // Create an index.
+ rpc Update(IndexUpdateRequest) returns (google.protobuf.Empty) {}
+
  // Drop an index.
  rpc Drop(IndexId) returns (google.protobuf.Empty) {}
 
@@ -31,4 +52,7 @@ service IndexService {
  // Query status of an index.
  // NOTE: API is subject to change.
  rpc GetStatus(IndexId) returns (IndexStatusResponse) {}
-}
+
+ // Garbage collect vertices identified as invalid before cutoff timestamp.
+ rpc GcInvalidVertices(GcInvalidVerticesRequest) returns (google.protobuf.Empty) {}
+}

proto/transact.proto

@@ -1,10 +1,9 @@
-
 syntax = "proto3";
 
 package aerospike.vector;
 
 option go_package = "aerospike.com/vector/protos/";
-option java_package = "com.aerospike.vector.client";
+option java_package = "com.aerospike.vector.client.proto";
 option java_multiple_files = true;
 
 import "google/protobuf/empty.proto";
@@ -23,6 +22,14 @@ enum WriteType {
  // Insert / create the record if it does not exists.
  // Fails if the record already exist.
  INSERT_ONLY = 2;
+
+ // Replace all fields in the record if it exists, else create the
+ // record if it does not exists.
+ REPLACE = 3;
+
+ // Replace all fields in the record if it exists.
+ // Fails if the record does not exist.
+ REPLACE_ONLY = 4;
 }
 
 // Put request to insert/update a record.
@@ -35,6 +42,10 @@ message PutRequest {
 
  // The record fields.
  repeated Field fields = 3;
+
+ // Ignore the in-memory queue full error. These records would be written to
+ // storage and later, the index healer would pick for indexing.
+ bool ignoreMemQueueFull = 4;
 }
 
 // Get request to insert/update a record.
@@ -43,7 +54,7 @@ message GetRequest {
  Key key = 1;
 
  // The field selector.
- ProjectionSpec projectionSpec = 2;
+ ProjectionSpec projection = 2;
 }
 
 // Check if a record exists.
@@ -115,7 +126,7 @@ message VectorSearchRequest {
 }
 
 // Record transaction services.
-service Transact {
+service TransactService {
  // Update/insert records.
  rpc Put(PutRequest) returns (google.protobuf.Empty) {}
 
@@ -133,4 +144,4 @@ service Transact {
 
  // Perform a vector nearest neighbor search.
  rpc VectorSearch(VectorSearchRequest) returns (stream Neighbor) {}
-}
+}

proto/types.proto

@@ -1,10 +1,9 @@
-
 syntax = "proto3";
 
 package aerospike.vector;
 
 option go_package = "aerospike.com/vector/protos/";
-option java_package = "com.aerospike.vector.client";
+option java_package = "com.aerospike.vector.client.proto";
 option java_multiple_files = true;
 
 // A record key.
@@ -187,16 +186,77 @@ message HnswParams {
 
  // Configures batching behaviour for batch based index update.
  HnswBatchingParams batchingParams = 4;
+
+ // Maximum size of in-memory queue for inserted/updated vector records.
+ // If the queue is full the record upsert will either be rejected with
+ // a RESOURCE_EXHAUSTED error or written to storage for index healer to
+ // later pick the record for indexing based on the put option.
+ // Defaults to global indexing config configured for the VectorDB.
+ optional uint32 maxMemQueueSize = 5;
+
+ // Configures caching for Hnsw Index.
+ HnswCachingParams cachingParams = 6;
+
+ // Configures index healer params.
+ HnswHealerParams healerParams = 7;
+
+ // Configures merge of batch indices to main index.
+ HnswIndexMergeParams mergeParams = 8;
 }
 
-// Params for the HNSW index search
+// Params for the HNSW index search.
 message HnswSearchParams {
  // The default number of candidate nearest neighbors shortlisted during search.
  // Larger values provide better recall at the cost of longer search times.
  // The default is value set in HnswParams for the index.
  optional uint32 ef = 1;
 }
 
+// Params to configure Hnsw batch index into main index.
+message HnswIndexMergeParams {
+ // The number of vectors merged in parallel from a batch index to main index.
+ // Defaults to global indexing config configured for the VectorDB.
+ optional uint32 parallelism = 1;
+}
+
+// Params to configure Hnsw index cache
+message HnswCachingParams {
+ // Maximum number of entries to cache.
+ // Defaults to the global cache config configured for the VectorDB.
+ optional uint64 maxEntries = 1;
+
+ // A cache entry will expire after this time in milliseconds has
+ // expired after the entry was added to cache.
+ // Defaults to the global cache config configured for the VectorDB.
+ optional uint64 expiry = 2;
+}
+
+// Params to configure Hnsw index cache
+message HnswHealerParams {
+ // Maximum allowed record scan rate per vector db node.
+ // Defaults to the global healer config configured for the VectorDB.
+ optional uint32 maxScanRatePerNode = 1;
+
+ // Maximum number of records in a single scanned page.
+ // Defaults to the global healer config configured for the VectorDB.
+ optional uint32 maxScanPageSize = 2;
+
+ // Percentage of good records randomly selected for reindexing in a healer cycle.
+ // Defaults to the global healer config configured for the VectorDB.
+ optional float reindexPercent = 3;
+
+ // The time delay, in milliseconds, between the termination of a healer run
+ // and the commencement of the next one for an index. It only guarantees
+ // that the next index healer run for an index will not be scheduled before
+ // this time delay.
+ // Defaults to the global healer config configured for the VectorDB.
+ optional uint64 scheduleDelay = 4;
+
+ // Maximum number of records to heal in parallel.
+ // Defaults to the global healer config configured for the VectorDB.
+ optional uint32 parallelism = 5;
+}
+
 // Configures batching behaviour for batch based index update.
 message HnswBatchingParams {
  // Maximum number of records to fit in a batch.
@@ -206,10 +266,28 @@ message HnswBatchingParams {
  // The maximum amount of time in milliseconds to wait before finalizing a batch.
  // The default value is 10000.
  optional uint32 interval = 2;
+}
+
+// Message to update a HNSW index parameters.
+message HnswIndexUpdate {
+ // Configures batching behaviour for batch based index update.
+ optional HnswBatchingParams batchingParams = 1;
 
- // Disables batching for index updates.
- // Default is false meaning batching is enabled.
- optional bool disabled = 3;
+ // Maximum size of in-memory queue for inserted/updated vector records.
+ // If the queue is full the record upsert will either be rejected with
+ // a RESOURCE_EXHAUSTED error or written to storage for index healer to
+ // later pick the record for indexing based on the put option.
+ // Defaults to global indexing config configured for the VectorDB.
+ optional uint32 maxMemQueueSize = 2;
+
+ // Configures caching for Hnsw Index.
+ optional HnswCachingParams cachingParams = 3;
+
+ // Configures index healer params.
+ optional HnswHealerParams healerParams = 4;
+
+ // Configures merge of batch indices to main index.
+ optional HnswIndexMergeParams mergeParams = 5;
 }
 
 // Index storage configuration
@@ -262,7 +340,35 @@ message IndexDefinitionList {
  repeated IndexDefinition indices = 1;
 }
 
+// A role.
+message Role {
+ // Role's unique name/id
+ string id = 1;
+}
+
+// A user.
+message User {
+ // User's username
+ string username = 1;
+
+ // Granted roles
+ repeated string roles = 2;
+}
+
+// Authentication credentials.
+message Credentials {
+ string username = 1;
+ oneof credentials {
+ PasswordCredentials passwordCredentials = 2;
+ }
+}
+
+// Password credentials.
+message PasswordCredentials {
+ string password = 1;
+}
+
 // A boolean type
 message Boolean {
  bool value = 1;
-}
+}