Enable pass query string to model_config in ml inference search response processor #2899

mingshl · 2024-09-05T18:00:26Z

Description

Enable pass query string to model_config in ml inference search response processor

setting cluster

PUT _cluster/settings
{
  "persistent": {
    "plugins": {
      "ml_commons": {
        "only_run_on_ml_node": "false",
        "model_access_control_enabled": "true",
        "native_memory_threshold": "99"
      }
    }
  }
}

register cross-encoders local model

POST /_plugins/_ml/models/_register
{
  "name": "huggingface/cross-encoders/ms-marco-MiniLM-L-6-v2",
  "version": "1.0.2",
  "model_format": "TORCH_SCRIPT"
}

{
  "task_id": "tQ5p1ZEB4iWlnHsIf2Xw",
  "status": "CREATED"
}

get register task status

GET /_plugins/_ml/tasks/tQ5p1ZEB4iWlnHsIf2Xw

{
  "model_id": "tg5p1ZEB4iWlnHsIh2U9",
  "task_type": "REGISTER_MODEL",
  "function_name": "TEXT_SIMILARITY",
  "state": "COMPLETED",
  "worker_node": [
    "AahcbpI9R2OtId7Wnt0cYA"
  ],
  "create_time": 1725862346467,
  "last_update_time": 1725862356009,
  "is_async": true
}

deploy cross-encoders local model

`POST /_plugins/_ml/models/tg5p1ZEB4iWlnHsIh2U9/_deploy

`

{
  "task_id": "tw5q1ZEB4iWlnHsIo2WX",
  "task_type": "DEPLOY_MODEL",
  "status": "CREATED"
}

get deploy task status

GET /_plugins/_ml/tasks/tw5q1ZEB4iWlnHsIo2WX

 {
  "model_id": "tg5p1ZEB4iWlnHsIh2U9",
  "task_type": "DEPLOY_MODEL",
  "function_name": "TEXT_SIMILARITY",
  "state": "RUNNING",
  "worker_node": [
    "AahcbpI9R2OtId7Wnt0cYA"
  ],
  "create_time": 1725862421392,
  "last_update_time": 1725862421555,
  "is_async": true
}

wait until completed

{
  "model_id": "tg5p1ZEB4iWlnHsIh2U9",
  "task_type": "DEPLOY_MODEL",
  "function_name": "TEXT_SIMILARITY",
  "state": "COMPLETED",
  "worker_node": [
    "AahcbpI9R2OtId7Wnt0cYA"
  ],
  "create_time": 1725862421392,
  "last_update_time": 1725862466568,
  "is_async": true
}

test model predict

POST _plugins/_ml/models/tg5p1ZEB4iWlnHsIh2U9/_predict
{
    "query_text": "today is sunny",
    "text_docs": [
        "how are you",
        "today is sunny",
        "today is july fifth",
        "it is winter"
    ]
}

{
  "inference_results": [
    {
      "output": [
        {
          "name": "similarity",
          "data_type": "FLOAT32",
          "shape": [
            1
          ],
          "data": [
            -11.055183
          ],
          "byte_buffer": {
            "array": "COIwwQ==",
            "order": "LITTLE_ENDIAN"
          }
        }
      ]
    },
    {
      "output": [
        {
          "name": "similarity",
          "data_type": "FLOAT32",
          "shape": [
            1
          ],
          "data": [
            8.969885
          ],
          "byte_buffer": {
            "array": "poQPQQ==",
            "order": "LITTLE_ENDIAN"
          }
        }
      ]
    },
    {
      "output": [
        {
          "name": "similarity",
          "data_type": "FLOAT32",
          "shape": [
            1
          ],
          "data": [
            -5.736347
          ],
          "byte_buffer": {
            "array": "KJC3wA==",
            "order": "LITTLE_ENDIAN"
          }
        }
      ]
    },
    {
      "output": [
        {
          "name": "similarity",
          "data_type": "FLOAT32",
          "shape": [
            1
          ],
          "data": [
            -10.0452175
          ],
          "byte_buffer": {
            "array": "NrkgwQ==",
            "order": "LITTLE_ENDIAN"
          }
        }
      ]
    }
  ]
}

upload index

 PUT /demo-index-0/_doc/1
{
  "dairy": "how are you"
}

 PUT /demo-index-0/_doc/2
{
  "dairy": "today is sunny"
}

 PUT /demo-index-0/_doc/3
{
  "dairy": "today is july fifth"
}

{
  "_index": "demo-index-0",
  "_id": "4",
  "_version": 1,
  "result": "created",
  "_shards": {
    "total": 2,
    "successful": 1,
    "failed": 0
  },
  "_seq_no": 3,
  "_primary_term": 1
}

create search pipeline with query text passing in model_config

PUT /_search/pipeline/my_pipeline
{
  "response_processors": [
    {
      "ml_inference": {
        "tag": "ml_inference",
        "description": "This processor is going to run ml inference during search response",
        "model_id": "tg5p1ZEB4iWlnHsIh2U9",
        "model_input":"{ \"text_docs\": ${input_map.text_docs}, \"query_text\": \"${model_config.query_text}\" }",
        "function_name": "TEXT_SIMILARITY",
        "input_map": [
          {
            "text_docs": "dairy"
          }
        ],
        "output_map": [
          {
            "rank_score": "$.inference_results[*].output[*].data"
          }
        ],
        "full_response_path":false,
        "model_config": {
          "query_text": "$.query.term.dairy.value"
        },
        "ignore_missing": false,
        "ignore_failure": false
      }
    }
  ]
}

search with search pipeline, scores are added in the response

GET /demo-index-0/_search?search_pipeline=my_pipeline
{
  "query": {
    "term": {
      "dairy": {
        "value": "today"
      }
    }
  }
}

{
  "took": 400,
  "timed_out": false,
  "_shards": {
    "total": 1,
    "successful": 1,
    "skipped": 0,
    "failed": 0
  },
  "hits": {
    "total": {
      "value": 2,
      "relation": "eq"
    },
    "max_score": 0.71566814,
    "hits": [
      {
        "_index": "demo-index-0",
        "_id": "2",
        "_score": 0.71566814,
        "_source": {
          "dairy": "today is sunny",
          "rank_score": [
            3.6144485
          ]
        }
      },
      {
        "_index": "demo-index-0",
        "_id": "3",
        "_score": 0.6333549,
        "_source": {
          "dairy": "today is july fifth",
          "rank_score": [
            3.6144485
          ]
        }
      }
    ]
  }
}

ToDo

currently ml inference processor only support single tensor for local model, need to support multiple tensor parsing as well.

Related Issues

#2897
#2878

Check List

New functionality includes testing.
New functionality has been documented.
API changes companion pull request created.
Commits are signed per the DCO using --signoff.
Public documentation issue/PR created.

By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
For more information on following Developer Certificate of Origin and signing off your commits, please check here.

ylwu-amzn · 2024-09-05T19:42:52Z

Add more details/examples to description ?

ylwu-amzn · 2024-09-05T19:43:44Z

> Task :opensearch-ml-common:test

StringUtilsTest > testisValidJSONPath_InvalidInputs FAILED
    java.lang.AssertionError at StringUtilsTest.java:476

StringUtilsTest > testisValidJSONPath_EmptyInput FAILED
    java.lang.AssertionError at StringUtilsTest.java:490

1008 tests completed, 2 failed, 2 skipped

mingshl · 2024-09-05T21:31:40Z

#2897

there are more details in the RFC #2897

mingshl · 2024-09-06T18:12:06Z

CI failed

OpenSearchConversationalMemoryHandlerTests > classMethod FAILED
Error: Exception in thread "Thread-4" java.lang.NoClassDefFoundError: Could not initialize class org.opensearch.test.OpenSearchTestCase
	at java.base/java.lang.Thread.run(Thread.java:1583)
	Suppressed: java.lang.IllegalStateException: No context information for thread: Thread[id=29, name=Thread-4, state=RUNNABLE, group=TGRP-ConversationalMemoryHandlerITTests]. Is this thread running under a class com.carrotsearch.randomizedtesting.RandomizedRunner runner context? Add @RunWith(class com.carrotsearch.randomizedtesting.RandomizedRunner.class) to your test class. Make sure your code accesses random contexts within @BeforeClass and @AfterClass boundary (for example, static test class initializers are not permitted to access random contexts).
		at com.carrotsearch.randomizedtesting.RandomizedContext.context(RandomizedContext.java:249)
		at com.carrotsearch.randomizedtesting.RandomizedContext.current(RandomizedContext.java:134)
		at com.carrotsearch.randomizedtesting.RandomizedRunner.augmentStackTrace(RandomizedRunner.java:1885)
		at com.carrotsearch.randomizedtesting.RunnerThreadGroup.uncaughtException(RunnerThreadGroup.java:20)
		at java.base/java.lang.Thread.dispatchUncaughtException(Thread.java:2901)
Caused by: java.lang.ExceptionInInitializerError: Exception java.lang.ExceptionInInitializerError [in thread "SUITE-ConversationalMemoryHandlerITTests-seed#[7B60246E9722A278]"]
	at org.opensearch.test.OpenSearchTestCase.<clinit>(OpenSearchTestCase.java:285)
	at java.base/java.lang.Class.forName0(Native Method)
	at java.base/java.lang.Class.forName(Class.java:534)
	at java.base/java.lang.Class.forName(Class.java:513)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$2.run(RandomizedRunner.java:623)
    java.lang.NoClassDefFoundError at Class.java:-2
        Caused by: java.lang.ExceptionInInitializerError at OpenSearchTestCase.java:285

Signed-off-by: Mingshi Liu <[email protected]>

common/src/main/java/org/opensearch/ml/common/utils/StringUtils.java

dhrubo-os · 2024-09-06T21:27:28Z

I also left a comment in your RFC.

Signed-off-by: Mingshi Liu <[email protected]>

common/src/test/java/org/opensearch/ml/common/utils/StringUtilsTest.java

Signed-off-by: Mingshi Liu <[email protected]>

mingshl · 2024-09-09T21:04:59Z

flaky test

REPRODUCE WITH: ./gradlew ':opensearch-ml-plugin:test' --tests "org.opensearch.ml.rest.RestMLPredictionActionTests.testGetRequest_LocalModelInferenceDisabled" -Dtests.seed=9770CFDD00A92E2A -Dtests.security.manager=false -Dtests.locale=az-AZ -Dtests.timezone=Asia/Sakhalin -Druntime.java=21

RestMLPredictionActionTests > testGetRequest_LocalModelInferenceDisabled FAILED
    java.lang.AssertionError: 
    Expected: (an instance of java.lang.IllegalStateException and exception with message a string containing "Local Model is currently disabled. To enable it, update the setting \"plugins.ml_commons.local_model.enabled\" to true.")
         but: an instance of java.lang.IllegalStateException <java.lang.IllegalArgumentException: Wrong Action Type of models> is a java.lang.IllegalArgumentException
    Stacktrace was: java.lang.IllegalArgumentException: Wrong Action Type of models
    	at org.opensearch.ml.common.connector.ConnectorAction$ActionType.from(ConnectorAction.java:199)
    	at org.opensearch.ml.rest.RestMLPredictionAction.getRequest(RestMLPredictionAction.java:129)
    	at org.opensearch.ml.rest.RestMLPredictionActionTests.testGetRequest_LocalModelInferenceDisabled(RestMLPredictionActionTests.java:146)
    	at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:103)
    	at java.base/java.lang.reflect.Method.invoke(Method.java:580)
    	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
    	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
    	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
    	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
    	at org.junit.rules.ExpectedException$ExpectedExceptionStatement.evaluate(ExpectedException.java:258)
    	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
    	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
    	at org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:48)

ylwu-amzn · 2024-10-03T17:44:24Z

plugin/src/main/java/org/opensearch/ml/processor/MLInferenceSearchResponseProcessor.java

@@ -316,22 +321,42 @@ private void rewriteResponseDocuments(SearchResponse response, ActionListener<Se
 * @param inputMapIndex the index of the input mapping to process
 * @param batchPredictionListener the listener to be notified when the predictions are processed
 * @param hitCountInPredictions a map to keep track of the count of hits that have the required input fields for each round of prediction
+ * @param queryString


minor: add some explanation to this parameter

ylwu-amzn · 2024-10-03T17:45:56Z

plugin/src/main/java/org/opensearch/ml/processor/MLInferenceSearchResponseProcessor.java

+ String modelConfigValue = entry.getValue();
+ if (StringUtils.isValidJSONPath(modelConfigValue)) {
+ Object queryJson = JsonPath.parse(queryString).read("$");
+ Configuration configuration = Configuration


Not efficient to construct same Configuration in a for loop. Move this out of for loop or create a static variable ?

great catch, will move the reading the query string to line 341

ylwu-amzn · 2024-10-03T17:51:59Z

plugin/src/main/java/org/opensearch/ml/processor/MLInferenceSearchResponseProcessor.java

+ String modelConfigKey = entry.getKey();
+ String modelConfigValue = entry.getValue();
+ if (StringUtils.isValidJSONPath(modelConfigValue)) {
+ Object queryJson = JsonPath.parse(queryString).read("$");


Why need to read "$" first ? , Can we just read the json path directly?

JsonPath.parse(queryString).read(modelConfigValue)

mingshl requested review from b4sjoo, dhrubo-os, jngz-es, model-collapse, rbhavna, ylwu-amzn, zane-neo, Zhangxunmt, austintlee, HenryL27, samuel-oci and xinyual as code owners September 5, 2024 18:00

mingshl had a problem deploying to ml-commons-cicd-env September 5, 2024 18:00 — with GitHub Actions Failure

mingshl force-pushed the main_add_query_text_to_response_processor branch from 299aefa to 350ab60 Compare September 5, 2024 22:49

mingshl had a problem deploying to ml-commons-cicd-env September 5, 2024 22:49 — with GitHub Actions Failure

mingshl had a problem deploying to ml-commons-cicd-env September 5, 2024 22:50 — with GitHub Actions Failure

mingshl had a problem deploying to ml-commons-cicd-env September 6, 2024 05:51 — with GitHub Actions Failure

mingshl had a problem deploying to ml-commons-cicd-env September 6, 2024 16:37 — with GitHub Actions Failure

mingshl had a problem deploying to ml-commons-cicd-env September 6, 2024 18:05 — with GitHub Actions Failure

enable add query_text to model_config

c1d112a

Signed-off-by: Mingshi Liu <[email protected]>

mingshl force-pushed the main_add_query_text_to_response_processor branch from 350ab60 to c1d112a Compare September 6, 2024 21:09

mingshl had a problem deploying to ml-commons-cicd-env September 6, 2024 21:09 — with GitHub Actions Failure

dhrubo-os reviewed Sep 6, 2024

View reviewed changes

common/src/main/java/org/opensearch/ml/common/utils/StringUtils.java Outdated Show resolved Hide resolved

dhrubo-os reviewed Sep 6, 2024

View reviewed changes

common/src/main/java/org/opensearch/ml/common/utils/StringUtils.java Outdated Show resolved Hide resolved

dhrubo-os reviewed Sep 6, 2024

View reviewed changes

common/src/main/java/org/opensearch/ml/common/utils/StringUtils.java Show resolved Hide resolved

dhrubo-os reviewed Sep 6, 2024

View reviewed changes

common/src/main/java/org/opensearch/ml/common/utils/StringUtils.java Show resolved Hide resolved

mingshl had a problem deploying to ml-commons-cicd-env September 8, 2024 04:49 — with GitHub Actions Failure

change javadoc

5319937

Signed-off-by: Mingshi Liu <[email protected]>

mingshl force-pushed the main_add_query_text_to_response_processor branch from bbbdae0 to 5319937 Compare September 9, 2024 05:13

mingshl temporarily deployed to ml-commons-cicd-env September 9, 2024 05:13 — with GitHub Actions Inactive

mingshl had a problem deploying to ml-commons-cicd-env September 9, 2024 05:13 — with GitHub Actions Failure

mingshl had a problem deploying to ml-commons-cicd-env September 9, 2024 06:30 — with GitHub Actions Failure

mingshl had a problem deploying to ml-commons-cicd-env September 9, 2024 07:03 — with GitHub Actions Failure

dhrubo-os reviewed Sep 9, 2024

View reviewed changes

common/src/test/java/org/opensearch/ml/common/utils/StringUtilsTest.java Show resolved Hide resolved

add more tests

576c3fa

Signed-off-by: Mingshi Liu <[email protected]>

mingshl temporarily deployed to ml-commons-cicd-env September 9, 2024 18:19 — with GitHub Actions Inactive

mingshl had a problem deploying to ml-commons-cicd-env September 9, 2024 18:20 — with GitHub Actions Failure

mingshl had a problem deploying to ml-commons-cicd-env September 9, 2024 20:18 — with GitHub Actions Failure

mingshl temporarily deployed to ml-commons-cicd-env September 9, 2024 21:05 — with GitHub Actions Inactive

mingshl added the backport 2.17 label Sep 9, 2024

mingshl temporarily deployed to ml-commons-cicd-env September 9, 2024 21:57 — with GitHub Actions Inactive

ylwu-amzn reviewed Oct 3, 2024

View reviewed changes

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Enable pass query string to model_config in ml inference search response processor #2899

Enable pass query string to model_config in ml inference search response processor #2899

mingshl commented Sep 5, 2024 •

edited

Loading

ylwu-amzn commented Sep 5, 2024

ylwu-amzn commented Sep 5, 2024

mingshl commented Sep 5, 2024

mingshl commented Sep 6, 2024

dhrubo-os commented Sep 6, 2024

mingshl commented Sep 9, 2024

ylwu-amzn Oct 3, 2024 •

edited

Loading

ylwu-amzn Oct 3, 2024

mingshl Oct 3, 2024

ylwu-amzn Oct 3, 2024

Enable pass query string to model_config in ml inference search response processor #2899

Are you sure you want to change the base?

Enable pass query string to model_config in ml inference search response processor #2899

Conversation

mingshl commented Sep 5, 2024 • edited Loading

Description

setting cluster

register cross-encoders local model

get register task status

deploy cross-encoders local model

get deploy task status

wait until completed

test model predict

upload index

create search pipeline with query text passing in model_config

search with search pipeline, scores are added in the response

ToDo

Related Issues

Check List

ylwu-amzn commented Sep 5, 2024

ylwu-amzn commented Sep 5, 2024

mingshl commented Sep 5, 2024

mingshl commented Sep 6, 2024

dhrubo-os commented Sep 6, 2024

mingshl commented Sep 9, 2024

ylwu-amzn Oct 3, 2024 • edited Loading

Choose a reason for hiding this comment

ylwu-amzn Oct 3, 2024

Choose a reason for hiding this comment

mingshl Oct 3, 2024

Choose a reason for hiding this comment

ylwu-amzn Oct 3, 2024

Choose a reason for hiding this comment

mingshl commented Sep 5, 2024 •

edited

Loading

ylwu-amzn Oct 3, 2024 •

edited

Loading