From 9625ad9f7dde0ebb2682ea89cdcccaac18dac428 Mon Sep 17 00:00:00 2001 From: Michael Carroll Date: Fri, 27 Oct 2023 12:35:40 -0400 Subject: [PATCH] feat(knn_config): support knn config in 2.x indices Signed-off-by: Michael Carroll --- README.md | 2 +- docs/resources/index.md | 2 + provider/resource_opensearch_index.go | 23 ++++++++++ provider/resource_opensearch_index_test.go | 52 ++++++++++++++++++++++ 4 files changed, 78 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 1157c9f..7dc52ce 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ Examples of resources can be found in the examples directory. ./script/install-tools export OSS_IMAGE="opensearchproject/opensearch:2" docker-compose up -d -docker-compose ps -a +docker-compose ps -a # Checks that the process is running export OPENSEARCH_URL=http://admin:admin@localhost:9200 export TF_LOG=INFO TF_ACC=1 go test ./... -v -parallel 20 -cover -short diff --git a/docs/resources/index.md b/docs/resources/index.md index 581ba22..a6c5547 100644 --- a/docs/resources/index.md +++ b/docs/resources/index.md @@ -63,6 +63,8 @@ EOF - `gc_deletes` (String) The length of time that a deleted document's version number remains available for further versioned operations. - `highlight_max_analyzed_offset` (String) The maximum number of characters that will be analyzed for a highlight request. A stringified number. - `include_type_name` (String) A string that indicates if and what we should pass to include_type_name parameter. Set to `"false"` when trying to create an index on a v6 cluster without a doc type or set to `"true"` when trying to create an index on a v7 cluster with a doc type. Since mapping updates are not currently supported, this applies only on index create. +- `index_knn` (Boolean) Indicates whether the index should build native library indices for the knn_vector fields. If set to false, the knn_vector fields will be stored in doc values, but Approximate k-NN search functionality will be disabled. +- `index_knn_algo_param_ef_search` (String) The size of the dynamic list used during k-NN searches. Higher values lead to more accurate but slower searches. Only available for `nmslib` implementation. - `index_similarity_default` (String) A JSON string describing the default index similarity config. - `indexing_slowlog_level` (String) Set which logging level to use for the search slow log, can be: `warn`, `info`, `debug`, `trace` - `indexing_slowlog_source` (String) Set the number of characters of the `_source` to include in the slowlog lines, `false` or `0` will skip logging the source entirely and setting it to `true` will log the entire source regardless of size. The original `_source` is reformatted by default to make sure that it fits on a single log line. diff --git a/provider/resource_opensearch_index.go b/provider/resource_opensearch_index.go index 96000aa..247c3c8 100644 --- a/provider/resource_opensearch_index.go +++ b/provider/resource_opensearch_index.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "log" + "strconv" "strings" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" @@ -23,6 +24,7 @@ var ( "shard.check_on_startup", "sort.field", "sort.order", + "index.knn", "index.similarity.default", } dynamicsSettingsKeys = []string{ @@ -66,6 +68,7 @@ var ( "indexing.slowlog.threshold.index.trace", "indexing.slowlog.level", "indexing.slowlog.source", + "index.knn.algo_param.ef_search", } settingsKeys = append(staticSettingsKeys, dynamicsSettingsKeys...) ) @@ -140,6 +143,12 @@ var ( ForceNew: true, Optional: true, }, + "index_knn": { + Type: schema.TypeBool, + Description: "Indicates whether the index should build native library indices for the knn_vector fields. If set to false, the knn_vector fields will be stored in doc values, but Approximate k-NN search functionality will be disabled.", + Optional: true, + ForceNew: true, + }, "index_similarity_default": { Type: schema.TypeString, Description: "A JSON string describing the default index similarity config.", @@ -348,6 +357,11 @@ var ( Description: "Set the number of characters of the `_source` to include in the slowlog lines, `false` or `0` will skip logging the source entirely and setting it to `true` will log the entire source regardless of size. The original `_source` is reformatted by default to make sure that it fits on a single log line.", Optional: true, }, + "index_knn_algo_param_ef_search": { + Type: schema.TypeString, + Description: "The size of the dynamic list used during k-NN searches. Higher values lead to more accurate but slower searches. Only available for nmslib.", + Optional: true, + }, // Other attributes "mappings": { Type: schema.TypeString, @@ -572,6 +586,15 @@ func indexResourceDataFromSettings(settings map[string]interface{}, d *schema.Re } schemaName := strings.Replace(key, ".", "_", -1) + + if configSchema[schemaName].Type == schema.TypeBool { + str := value.(string) + parsed, err := strconv.ParseBool(str) + if err == nil { + value = parsed + } + } + err := d.Set(schemaName, value) if err != nil { log.Printf("[ERROR] indexResourceDataFromSettings: %+v", err) diff --git a/provider/resource_opensearch_index_test.go b/provider/resource_opensearch_index_test.go index d353165..367744a 100644 --- a/provider/resource_opensearch_index_test.go +++ b/provider/resource_opensearch_index_test.go @@ -156,6 +156,26 @@ resource "opensearch_index" "test_similarity_config" { }) } ` + + testAccOpensearchIndexWithKNNConfig = ` +resource "opensearch_index" "test_knn_config" { + name = "terraform-test-update-knn-module" + number_of_shards = 1 + number_of_replicas = 1 + index_knn = true +} +` + + testAccOpensearchIndexWithKNNAlgoParamEfSearchConfig = ` +resource "opensearch_index" "test_knn_algo_param_ef_search_config" { + name = "terraform-test-update-knn-algo-param-ef-search-module" + number_of_shards = 1 + number_of_replicas = 1 + index_knn = true + index_knn_algo_param_ef_search = 600 +} +` + testAccOpensearchIndexRolloverAliasOpendistro = ` resource opensearch_ism_policy "test" { policy_id = "test" @@ -436,6 +456,38 @@ func TestAccOpensearchIndex_similarityConfig(t *testing.T) { }) } +func TestAccOpensearchIndex_knnConfig(t *testing.T) { + resource.Test(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: checkOpensearchIndexDestroy, + Steps: []resource.TestStep{ + { + Config: testAccOpensearchIndexWithKNNConfig, + Check: resource.ComposeTestCheckFunc( + checkOpensearchIndexExists("opensearch_index.test_knn_config"), + ), + }, + }, + }) +} + +func TestAccOpensearchIndex_knnAlgoParamEFSearchConfig(t *testing.T) { + resource.Test(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: checkOpensearchIndexDestroy, + Steps: []resource.TestStep{ + { + Config: testAccOpensearchIndexWithKNNAlgoParamEfSearchConfig, + Check: resource.ComposeTestCheckFunc( + checkOpensearchIndexExists("opensearch_index.test_knn_algo_param_ef_search_config"), + ), + }, + }, + }) +} + func TestAccOpensearchIndex_doctype(t *testing.T) { provider := Provider() diags := provider.Configure(context.Background(), &terraform.ResourceConfig{})