Skip to content

Commit

Permalink
Merge branch '2.x' into backport/backport-952-to-2.x
Browse files Browse the repository at this point in the history
Signed-off-by: Junwei Dai <[email protected]>
  • Loading branch information
junweid62 authored Nov 11, 2024
2 parents 404f47f + ccb4e85 commit 42f6b39
Show file tree
Hide file tree
Showing 42 changed files with 2,109 additions and 170 deletions.
1 change: 1 addition & 0 deletions .github/workflows/backport.yml
Original file line number Diff line number Diff line change
Expand Up @@ -37,3 +37,4 @@ jobs:
with:
github_token: ${{ steps.github_app_token.outputs.token }}
head_template: backport/backport-<%= number %>-to-<%= base %>
failure_labels: backport-failed
26 changes: 26 additions & 0 deletions .github/workflows/test-api-consistency.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
name: Daily API Consistency Test

on:
schedule:
- cron: '0 8 * * *' # Runs daily at 8 AM UTC
workflow_dispatch:

jobs:
API-consistency-test:
runs-on: ubuntu-latest
strategy:
matrix:
java: [21]

steps:
- name: Checkout Flow Framework
uses: actions/checkout@v3

- name: Setup Java ${{ matrix.java }}
uses: actions/setup-java@v3
with:
distribution: 'temurin'
java-version: ${{ matrix.java }}

- name: Run API Consistency Tests
run: ./gradlew test --tests "org.opensearch.flowframework.workflow.*"
17 changes: 17 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,17 +8,34 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.1.0/)
### Enhancements
### Bug Fixes
### Infrastructure
- Set Java target compatibility to JDK 21 ([#730](https://github.com/opensearch-project/flow-framework/pull/730))

### Documentation
- Add alert summary agent template ([#873](https://github.com/opensearch-project/flow-framework/pull/873))
- Add alert summary with log pattern agent template ([#945](https://github.com/opensearch-project/flow-framework/pull/945))

### Maintenance
### Refactoring

## [Unreleased 2.x](https://github.com/opensearch-project/flow-framework/compare/2.17...2.x)
### Features
- Add ApiSpecFetcher for Fetching and Comparing API Specifications ([#651](https://github.com/opensearch-project/flow-framework/issues/651))
- Add optional config field to tool step ([#899](https://github.com/opensearch-project/flow-framework/pull/899))
- Add API Consistency Tests with ML-Common and Set Up Daily GitHub Action Trigger([#908](https://github.com/opensearch-project/flow-framework/issues/908))

### Enhancements
- Incrementally remove resources from workflow state during deprovisioning ([#898](https://github.com/opensearch-project/flow-framework/pull/898))

### Bug Fixes
- Remove useCase and defaultParams field in WorkflowRequest ([#758](https://github.com/opensearch-project/flow-framework/pull/758))
- Fixed Template Update Location and Improved Logger Statements in ReprovisionWorkflowTransportAction ([#918](https://github.com/opensearch-project/flow-framework/pull/918))

### Infrastructure
### Documentation
- Add knowledge base alert agent into sample templates ([#874](https://github.com/opensearch-project/flow-framework/pull/874))
- Add query assist data summary agent into sample templates ([#875](https://github.com/opensearch-project/flow-framework/pull/875))
- Add suggest anomaly detector agent into sample templates ([#944](https://github.com/opensearch-project/flow-framework/pull/944))

### Maintenance
### Refactoring
- Update workflow state without using painless script ([#894](https://github.com/opensearch-project/flow-framework/pull/894))
42 changes: 33 additions & 9 deletions build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import java.nio.file.Paths

buildscript {
ext {
opensearch_version = System.getProperty("opensearch.version", "2.17.0-SNAPSHOT")
opensearch_version = System.getProperty("opensearch.version", "2.19.0-SNAPSHOT")
buildVersionQualifier = System.getProperty("build.version_qualifier", "")
isSnapshot = "true" == System.getProperty("build.snapshot", "true")
version_tokens = opensearch_version.tokenize('-')
Expand All @@ -24,7 +24,7 @@ buildscript {
opensearch_no_snapshot = opensearch_build.replace("-SNAPSHOT","")
System.setProperty('tests.security.manager', 'false')
common_utils_version = System.getProperty("common_utils.version", opensearch_build)

swaggerCoreVersion = "2.2.23"
bwcVersionShort = "2.12.0"
bwcVersion = bwcVersionShort + ".0"
bwcOpenSearchFFDownload = 'https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/' + bwcVersionShort + '/latest/linux/x64/tar/builds/' +
Expand All @@ -34,6 +34,10 @@ buildscript {
bwcFlowFrameworkPath = bwcFilePath + "flowframework/"

isSameMajorVersion = opensearch_version.split("\\.")[0] == bwcVersionShort.split("\\.")[0]
swaggerVersion = "2.1.23"
jacksonVersion = "2.18.1"
swaggerCoreVersion = "2.2.25"

}

repositories {
Expand All @@ -52,8 +56,8 @@ buildscript {

plugins {
id "de.undercouch.download" version "5.6.0"
id "org.gradle.test-retry" version "1.5.10" apply false
id "io.github.surpsg.delta-coverage" version "2.4.0"
id "org.gradle.test-retry" version "1.6.0" apply false
id "io.github.surpsg.delta-coverage" version "2.5.0"
}

apply plugin: 'java'
Expand Down Expand Up @@ -164,17 +168,37 @@ configurations {

dependencies {
implementation "org.opensearch:opensearch:${opensearch_version}"
implementation 'org.junit.jupiter:junit-jupiter:5.11.0'
implementation 'org.junit.jupiter:junit-jupiter:5.11.3'
api group: 'org.opensearch', name:'opensearch-ml-client', version: "${opensearch_build}"
api group: 'org.opensearch.client', name: 'opensearch-rest-client', version: "${opensearch_version}"
api group: 'org.slf4j', name: 'slf4j-api', version: '1.7.36'
implementation group: 'org.apache.commons', name: 'commons-lang3', version: '3.17.0'
implementation "org.opensearch:common-utils:${common_utils_version}"
implementation 'com.amazonaws:aws-encryption-sdk-java:2.4.1'
implementation 'org.bouncycastle:bcprov-jdk18on:1.78'
implementation "com.amazonaws:aws-encryption-sdk-java:3.0.1"
implementation "software.amazon.cryptography:aws-cryptographic-material-providers:1.7.0"
implementation "org.dafny:DafnyRuntime:4.9.0"
implementation "software.amazon.smithy.dafny:conversion:0.1.1"
implementation 'org.bouncycastle:bcprov-jdk18on:1.78.1'
implementation "jakarta.json.bind:jakarta.json.bind-api:3.0.1"
implementation "org.glassfish:jakarta.json:2.0.1"
implementation "org.eclipse:yasson:3.0.4"
implementation "com.google.code.gson:gson:2.11.0"
// Swagger-Parser dependencies for API consistency tests
implementation "io.swagger.core.v3:swagger-models:${swaggerCoreVersion}"
implementation "io.swagger.core.v3:swagger-core:${swaggerCoreVersion}"
implementation "io.swagger.parser.v3:swagger-parser-core:${swaggerVersion}"
implementation "io.swagger.parser.v3:swagger-parser:${swaggerVersion}"
implementation "io.swagger.parser.v3:swagger-parser-v3:${swaggerVersion}"
// Declare and force Jackson dependencies for tests
testImplementation("com.fasterxml.jackson.core:jackson-databind") {
version { strictly("${jacksonVersion}") }
}
testImplementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310") {
version { strictly("${jacksonVersion}") }
}
testImplementation("com.fasterxml.jackson.core:jackson-annotations") {
version { strictly("${jacksonVersion}") }
}

// ZipArchive dependencies used for integration tests
zipArchive group: 'org.opensearch.plugin', name:'opensearch-ml-plugin', version: "${opensearch_build}"
Expand All @@ -184,8 +208,8 @@ dependencies {

configurations.all {
resolutionStrategy {
force("com.google.guava:guava:33.3.0-jre") // CVE for 31.1, keep to force transitive dependencies
force("com.fasterxml.jackson.core:jackson-core:2.17.2") // Dependency Jar Hell
force("com.google.guava:guava:33.3.1-jre") // CVE for 31.1, keep to force transitive dependencies
force("com.fasterxml.jackson.core:jackson-core:${jacksonVersion}") // Dependency Jar Hell
}
}
}
Expand Down
4 changes: 2 additions & 2 deletions gradle/wrapper/gradle-wrapper.properties
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
distributionSha256Sum=5b9c5eb3f9fc2c94abaea57d90bd78747ca117ddbbf96c859d3741181a12bf2a
distributionUrl=https\://services.gradle.org/distributions/gradle-8.10-bin.zip
distributionSha256Sum=31c55713e40233a8303827ceb42ca48a47267a0ad4bab9177123121e71524c26
distributionUrl=https\://services.gradle.org/distributions/gradle-8.10.2-bin.zip
networkTimeout=10000
validateDistributionUrl=true
zipStoreBase=GRADLE_USER_HOME
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
## Version 2.17.1.0

Compatible with OpenSearch 2.17.1

### Maintenance
- Fix flaky integ test reprovisioning before template update ([#880](https://github.com/opensearch-project/flow-framework/pull/880))
19 changes: 19 additions & 0 deletions release-notes/opensearch-flow-framework.release-notes-2.18.0.0.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
## Version 2.18.0.0

Compatible with OpenSearch 2.18.0

### Features
- Add ApiSpecFetcher for Fetching and Comparing API Specifications ([#651](https://github.com/opensearch-project/flow-framework/issues/651))
- Add optional config field to tool step ([#899](https://github.com/opensearch-project/flow-framework/pull/899))

### Enhancements
- Incrementally remove resources from workflow state during deprovisioning ([#898](https://github.com/opensearch-project/flow-framework/pull/898))

### Bug Fixes
- Fixed Template Update Location and Improved Logger Statements in ReprovisionWorkflowTransportAction ([#918](https://github.com/opensearch-project/flow-framework/pull/918))

### Documentation
- Add query assist data summary agent into sample templates ([#875](https://github.com/opensearch-project/flow-framework/pull/875))

### Refactoring
- Update workflow state without using painless script ([#894](https://github.com/opensearch-project/flow-framework/pull/894))
94 changes: 94 additions & 0 deletions sample-templates/alert-summary-agent-claude-tested.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
{
"name": "Alert Summary Agent",
"description": "Create Alert Summary Agent using Claude on BedRock",
"use_case": "REGISTER_AGENT",
"version": {
"template": "1.0.0",
"compatibility": ["2.17.0", "3.0.0"]
},
"workflows": {
"provision": {
"user_params": {},
"nodes": [
{
"id": "create_claude_connector",
"type": "create_connector",
"previous_node_inputs": {},
"user_inputs": {
"version": "1",
"name": "Claude instant runtime Connector",
"protocol": "aws_sigv4",
"description": "The connector to BedRock service for Claude model",
"actions": [
{
"headers": {
"x-amz-content-sha256": "required",
"content-type": "application/json"
},
"method": "POST",
"request_body": "{\"prompt\":\"${parameters.prompt}\", \"max_tokens_to_sample\":${parameters.max_tokens_to_sample}, \"temperature\":${parameters.temperature}, \"anthropic_version\":\"${parameters.anthropic_version}\" }",
"action_type": "predict",
"url": "https://bedrock-runtime.us-west-2.amazonaws.com/model/anthropic.claude-instant-v1/invoke"
}
],
"credential": {
"access_key": "<YOUR_ACCESS_KEY>",
"secret_key": "<YOUR_SECRET_KEY>",
"session_token": "<YOUR_SESSION_TOKEN>"
},
"parameters": {
"region": "us-west-2",
"endpoint": "bedrock-runtime.us-west-2.amazonaws.com",
"content_type": "application/json",
"auth": "Sig_V4",
"max_tokens_to_sample": "8000",
"service_name": "bedrock",
"temperature": "0.0001",
"response_filter": "$.completion",
"anthropic_version": "bedrock-2023-05-31"
}
}
},
{
"id": "register_claude_model",
"type": "register_remote_model",
"previous_node_inputs": {
"create_claude_connector": "connector_id"
},
"user_inputs": {
"description": "Claude model",
"deploy": true,
"name": "claude-instant"
}
},
{
"id": "create_alert_summary_ml_model_tool",
"type": "create_tool",
"previous_node_inputs": {
"register_claude_model": "model_id"
},
"user_inputs": {
"parameters": {
"prompt": "You are an OpenSearch Alert Assistant to help summarize the alerts.\n Here is the detail of alert: ${parameters.context};\n The question is: ${parameters.question}."
},
"name": "MLModelTool",
"type": "MLModelTool"
}
},
{
"id": "create_alert_summary_agent",
"type": "register_agent",
"previous_node_inputs": {
"create_alert_summary_ml_model_tool": "tools"
},
"user_inputs": {
"parameters": {},
"type": "flow",
"name": "Alert Summary Agent",
"description": "this is an alert summary agent"
}
}
]
}
}
}
71 changes: 71 additions & 0 deletions sample-templates/alert-summary-agent-claude-tested.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
---
name: Alert Summary Agent
description: Create Alert Summary Agent using Claude on BedRock
use_case: REGISTER_AGENT
version:
template: 1.0.0
compatibility:
- 2.17.0
- 3.0.0
workflows:
provision:
user_params: {}
nodes:
- id: create_claude_connector
type: create_connector
previous_node_inputs: {}
user_inputs:
version: '1'
name: Claude instant runtime Connector
protocol: aws_sigv4
description: The connector to BedRock service for Claude model
actions:
- headers:
x-amz-content-sha256: required
content-type: application/json
method: POST
request_body: '{"prompt":"${parameters.prompt}", "max_tokens_to_sample":${parameters.max_tokens_to_sample},
"temperature":${parameters.temperature}, "anthropic_version":"${parameters.anthropic_version}"
}'
action_type: predict
url: https://bedrock-runtime.us-west-2.amazonaws.com/model/anthropic.claude-instant-v1/invoke
credential:
access_key: "<YOUR_ACCESS_KEY>"
secret_key: "<YOUR_SECRET_KEY>"
session_token: "<YOUR_SESSION_TOKEN>"
parameters:
region: us-west-2
endpoint: bedrock-runtime.us-west-2.amazonaws.com
content_type: application/json
auth: Sig_V4
max_tokens_to_sample: '8000'
service_name: bedrock
temperature: '0.0001'
response_filter: "$.completion"
anthropic_version: bedrock-2023-05-31
- id: register_claude_model
type: register_remote_model
previous_node_inputs:
create_claude_connector: connector_id
user_inputs:
description: Claude model
deploy: true
name: claude-instant
- id: create_alert_summary_ml_model_tool
type: create_tool
previous_node_inputs:
register_claude_model: model_id
user_inputs:
parameters:
prompt: "You are an OpenSearch Alert Assistant to help summarize the alerts.\n Here is the detail of alert: ${parameters.context};\n The question is: ${parameters.question}."
name: MLModelTool
type: MLModelTool
- id: create_alert_summary_agent
type: register_agent
previous_node_inputs:
create_alert_summary_ml_model_tool: tools
user_inputs:
parameters: {}
type: flow
name: Alert Summary Agent
description: this is an alert summary agent
Loading

0 comments on commit 42f6b39

Please sign in to comment.