Merge branch 'main' into reprovision

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,16 @@
+### Description
+[Describe what this change achieves]
+
+### Related Issues
+Resolves #[Issue number to be closed when this PR is merged]
+<!-- List any other related issues here -->
+
+### Check List
+- [ ] New functionality includes testing.
+- [ ] New functionality has been documented.
+- [ ] API changes companion pull request [created](https://github.com/opensearch-project/opensearch-api-specification/blob/main/DEVELOPER_GUIDE.md).
+- [ ] Commits are signed per the DCO using `--signoff`.
+- [ ] Public documentation issue/PR [created](https://github.com/opensearch-project/documentation-website/issues/new/choose).
+
+By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
+For more information on following Developer Certificate of Origin and signing off your commits, please check [here](https://github.com/opensearch-project/flow-framework/blob/main/CONTRIBUTING.md#developer-certificate-of-origin).

.github/workflows/CI.yml

@@ -1,10 +1,12 @@
 name: CI
 
 on:
+  workflow_dispatch:
   push:
     branches-ignore:
       - 'whitesource-remediate/**'
       - 'backport/**'
+      - 'create-pull-request/**'
   pull_request:
     types: [opened, synchronize, reopened]
 
@@ -37,7 +39,7 @@ jobs:
     needs: [spotless, javadoc]
     strategy:
       matrix:
-        os: [ubuntu-latest, macos-13, windows-latest]
+        os: [ubuntu-latest, macos-latest, windows-latest]
         java: [21]
     name: Test JDK${{ matrix.java }}, ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
@@ -63,7 +65,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, macos-13, windows-latest]
+        os: [ubuntu-latest, macos-latest, windows-latest]
         java: [21]
     name: Integ Test JDK${{ matrix.java }}, ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
@@ -82,7 +84,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, macos-13, windows-latest]
+        os: [ubuntu-latest, macos-latest, windows-latest]
         java: [21]
     name: Multi-Node Integ Test JDK${{ matrix.java }}, ${{ matrix.os }}
     runs-on: ${{ matrix.os }}

.github/workflows/test_bwc.yml

@@ -1,11 +1,13 @@
 name: BWC
 on:
+  workflow_dispatch:
   push:
-    branches:
-      - "**"
+    branches-ignore:
+      - 'whitesource-remediate/**'
+      - 'backport/**'
+      - 'create-pull-request/**'
   pull_request:
-    branches:
-      - "**"
+    types: [opened, synchronize, reopened]
 
 jobs:
   Build-ff-linux:

.github/workflows/test_security.yml

@@ -1,12 +1,17 @@
 name: Security test workflow for Flow Framework
 on:
+  workflow_dispatch:
   push:
     branches-ignore:
       - 'whitesource-remediate/**'
       - 'backport/**'
+      - 'create-pull-request/**'
   pull_request:
     types: [opened, synchronize, reopened]
 
+env:
+  ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
+
 jobs:
   Get-CI-Image-Tag:
     uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main

.github/workflows/wrapper.yml

@@ -14,4 +14,4 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - uses: gradle/wrapper-validation-action@v3
+      - uses: gradle/actions/wrapper-validation@v3

CHANGELOG.md

@@ -16,10 +16,16 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.1.0/)
 
 ## [Unreleased 2.x](https://github.com/opensearch-project/flow-framework/compare/2.14...2.x)
 ### Features
+- Support editing of certain workflow fields on a provisioned workflow ([#757](https://github.com/opensearch-project/flow-framework/pull/757))
+- Add allow_delete parameter to Deprovision API ([#763](https://github.com/opensearch-project/flow-framework/pull/763))
+
 ### Enhancements
+- Register system index descriptors through SystemIndexPlugin.getSystemIndexDescriptors ([#750](https://github.com/opensearch-project/flow-framework/pull/750))
+
 ### Bug Fixes
 
 ### Infrastructure
 ### Documentation
 ### Maintenance
 ### Refactoring
+- Improve Template and WorkflowState builders ([#778](https://github.com/opensearch-project/flow-framework/pull/778))

CONTRIBUTING.md

@@ -1,4 +1,4 @@
-- [Contributing to OpenSearch AI Flow Framework](#contributing-to-flow-framework)
+- [Contributing to OpenSearch Flow Framework](#contributing-to-opensearch-flow-framework)
   - [First Things First](#first-things-first)
   - [Ways to Contribute](#ways-to-contribute)
     - [Bug Reports](#bug-reports)
@@ -100,7 +100,7 @@ Adding in the change is two step process -
 
 Any new functionality requires testing. Your PR will trigger an automatic assessment of the code coverage of the lines you've added. You should add unit and/or integration tests to exercise as much of your new code as possible.
 
-If you'd like to preview your coverage before submitting your PR, to identify lines of code which are not tested, you may run `./gradlew diffCoverage` and review the report available in the project build directory at `build/reports/jacoco/diffCoverage/html/index.html`.
+If you'd like to preview your coverage before submitting your PR, to identify lines of code which are not tested, you may run `./gradlew test deltaCoverage` and review the report available in the project build directory at `build/reports/coverage-reports/delta-coverage/html/index.html`.
 
 ## Review Process
 

build.gradle

@@ -47,13 +47,13 @@ buildscript {
     dependencies {
         classpath "org.opensearch.gradle:build-tools:${opensearch_version}"
         classpath "com.diffplug.spotless:spotless-plugin-gradle:6.25.0"
-        classpath "com.github.form-com.diff-coverage-gradle:diff-coverage:0.9.5"
     }
 }
 
 plugins {
     id "de.undercouch.download" version "5.6.0"
-    id "org.gradle.test-retry" version "1.5.9" apply false
+    id "org.gradle.test-retry" version "1.5.10" apply false
+    id "io.github.surpsg.delta-coverage" version "2.3.0"
 }
 
 apply plugin: 'java'
@@ -66,9 +66,6 @@ apply plugin: 'com.diffplug.spotless'
 apply from: 'formatter/formatting.gradle'
 // for javadocs and checks spotless doesn't do
 apply plugin: 'checkstyle'
-// for coverage and diff
-apply plugin: 'jacoco'
-apply plugin: 'com.form.diff-coverage'
 
 def pluginName = 'opensearch-flow-framework'
 def pluginDescription = 'OpenSearch plugin that enables builders to innovate AI apps on OpenSearch'
@@ -167,14 +164,14 @@ configurations {
 
 dependencies {
     implementation "org.opensearch:opensearch:${opensearch_version}"
-    implementation 'org.junit.jupiter:junit-jupiter:5.10.2'
+    implementation 'org.junit.jupiter:junit-jupiter:5.10.3'
     api group: 'org.opensearch', name:'opensearch-ml-client', version: "${opensearch_build}"
     api group: 'org.opensearch.client', name: 'opensearch-rest-client', version: "${opensearch_version}"
-    implementation group: 'org.apache.commons', name: 'commons-lang3', version: '3.14.0'
+    implementation group: 'org.apache.commons', name: 'commons-lang3', version: '3.15.0'
     implementation "org.opensearch:common-utils:${common_utils_version}"
     implementation 'com.amazonaws:aws-encryption-sdk-java:2.4.1'
     implementation 'org.bouncycastle:bcprov-jdk18on:1.78.1'
-    api "org.apache.httpcomponents.core5:httpcore5:5.2.2"
+    api "org.apache.httpcomponents.core5:httpcore5:5.2.5"
     implementation "jakarta.json.bind:jakarta.json.bind-api:3.0.1"
     implementation "org.glassfish:jakarta.json:2.0.1"
     implementation "org.eclipse:yasson:3.0.3"
@@ -188,8 +185,8 @@ dependencies {
     configurations.all {
         resolutionStrategy {
             force("com.google.guava:guava:33.2.1-jre") // CVE for 31.1, keep to force transitive dependencies
-            force("org.eclipse.platform:org.eclipse.core.runtime:3.31.0") // CVE for < 3.29.0
-            force("com.fasterxml.jackson.core:jackson-core:2.17.1") // Dependency Jar Hell
+            force("com.fasterxml.jackson.core:jackson-core:2.17.2") // Dependency Jar Hell
+            force("org.apache.httpcomponents.core5:httpcore5:5.2.5") // Dependency Jar Hell
         }
     }
 }
@@ -394,7 +391,7 @@ testClusters.integTest {
 
     }
 
-    // Install Flow Framwork Plugin on integTest cluster nodes
+    // Install Flow Framework Plugin on integTest cluster nodes
     plugin(project.tasks.bundlePlugin.archiveFile)
 
     // Cluster shrink exception thrown if we try to set numberOfNodes to 1, so only apply if > 1
@@ -683,38 +680,26 @@ task updateVersion {
     }
 }
 
-diffCoverageReport {
-    afterEvaluate {
-        // Get uncommitted files via git diff
-        // https://github.com/form-com/diff-coverage-gradle/issues/73
-        def file = Files.createTempFile(URLEncoder.encode(project.name, 'UTF-8'), '.diff').toFile()
-        def diffBase = 'refs/remotes/origin/main'
-        // Only run locally
-        if (!System.getenv('CI')) {
-            file.withOutputStream { out ->
-                exec {
-                    ignoreExitValue true
-                    commandLine 'git', 'diff', '--no-color', '--minimal', diffBase
-                    standardOutput = out
-                }
-            }
-        }
-        diffSource.file = file
+deltaCoverageReport {
+    diffSource {
+        git.compareWith("refs/remotes/origin/main")
     }
 
-    // View report at build/reports/jacoco/diffCoverage/html/index.html
-    reports {
-        html = true
+    violationRules {
+        failOnViolation.set(true)
+        rule(io.github.surpsg.deltacoverage.gradle.CoverageEntity.LINE) {
+            minCoverageRatio.set(0.75d)
+        }
+        rule(io.github.surpsg.deltacoverage.gradle.CoverageEntity.BRANCH) {
+            minCoverageRatio.set(0.6d)
+        }
     }
 
-    violationRules {
-        minBranches = 0.60
-        minLines = 0.75
-        failOnViolation = true
+    reports {
+        html.set(true)
     }
 }
 
-
 tasks.withType(AbstractPublishToMaven) {
     def predicate = provider {
         publication.name == "pluginZip"

gradle/wrapper/gradle-wrapper.properties

@@ -1,7 +1,7 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionSha256Sum=544c35d6bd849ae8a5ed0bcea39ba677dc40f49df7d1835561582da2009b961d
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.7-bin.zip
+distributionSha256Sum=d725d707bfabd4dfdc958c624003b3c80accc03f7037b5122c4b1d0ef15cecab
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.9-bin.zip
 networkTimeout=10000
 validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME

gradlew

@@ -15,6 +15,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+# SPDX-License-Identifier: Apache-2.0
+#
 
 ##############################################################################
 #
@@ -55,7 +57,7 @@
 #       Darwin, MinGW, and NonStop.
 #
 #   (3) This script is generated from the Groovy template
-#       https://github.com/gradle/gradle/blob/HEAD/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt
+#       https://github.com/gradle/gradle/blob/HEAD/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt
 #       within the Gradle project.
 #
 #       You can find Gradle at https://github.com/gradle/gradle/.
@@ -84,7 +86,8 @@ done
 # shellcheck disable=SC2034
 APP_BASE_NAME=${0##*/}
 # Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036)
-APP_HOME=$( cd "${APP_HOME:-./}" > /dev/null && pwd -P ) || exit
+APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s
+' "$PWD" ) || exit
 
 # Use the maximum available, or set MAX_FD != -1 to use that value.
 MAX_FD=maximum

gradlew.bat

@@ -13,6 +13,8 @@
 @rem See the License for the specific language governing permissions and
 @rem limitations under the License.
 @rem
+@rem SPDX-License-Identifier: Apache-2.0
+@rem
 
 @if "%DEBUG%"=="" @echo off
 @rem ##########################################################################

src/main/java/org/opensearch/flowframework/FlowFrameworkPlugin.java

@@ -59,9 +59,11 @@
 import org.opensearch.flowframework.util.EncryptorUtils;
 import org.opensearch.flowframework.workflow.WorkflowProcessSorter;
 import org.opensearch.flowframework.workflow.WorkflowStepFactory;
+import org.opensearch.indices.SystemIndexDescriptor;
 import org.opensearch.ml.client.MachineLearningNodeClient;
 import org.opensearch.plugins.ActionPlugin;
 import org.opensearch.plugins.Plugin;
+import org.opensearch.plugins.SystemIndexPlugin;
 import org.opensearch.repositories.RepositoriesService;
 import org.opensearch.rest.RestController;
 import org.opensearch.rest.RestHandler;
@@ -75,9 +77,12 @@
 import java.util.List;
 import java.util.function.Supplier;
 
+import static org.opensearch.flowframework.common.CommonValue.CONFIG_INDEX;
 import static org.opensearch.flowframework.common.CommonValue.DEPROVISION_WORKFLOW_THREAD_POOL;
 import static org.opensearch.flowframework.common.CommonValue.FLOW_FRAMEWORK_THREAD_POOL_PREFIX;
+import static org.opensearch.flowframework.common.CommonValue.GLOBAL_CONTEXT_INDEX;
 import static org.opensearch.flowframework.common.CommonValue.PROVISION_WORKFLOW_THREAD_POOL;
+import static org.opensearch.flowframework.common.CommonValue.WORKFLOW_STATE_INDEX;
 import static org.opensearch.flowframework.common.CommonValue.WORKFLOW_THREAD_POOL;
 import static org.opensearch.flowframework.common.FlowFrameworkSettings.FLOW_FRAMEWORK_ENABLED;
 import static org.opensearch.flowframework.common.FlowFrameworkSettings.MAX_WORKFLOWS;
@@ -88,7 +93,7 @@
 /**
  * An OpenSearch plugin that enables builders to innovate AI apps on OpenSearch.
  */
-public class FlowFrameworkPlugin extends Plugin implements ActionPlugin {
+public class FlowFrameworkPlugin extends Plugin implements ActionPlugin, SystemIndexPlugin {
 
     private FlowFrameworkSettings flowFrameworkSettings;
 
@@ -128,13 +133,7 @@ public Collection<Object> createComponents(
             flowFrameworkSettings,
             client
         );
-        WorkflowProcessSorter workflowProcessSorter = new WorkflowProcessSorter(
-            workflowStepFactory,
-            threadPool,
-            clusterService,
-            client,
-            flowFrameworkSettings
-        );
+        WorkflowProcessSorter workflowProcessSorter = new WorkflowProcessSorter(workflowStepFactory, threadPool, flowFrameworkSettings);
 
         return List.of(workflowStepFactory, workflowProcessSorter, encryptorUtils, flowFrameworkIndicesHandler, flowFrameworkSettings);
     }
@@ -210,4 +209,13 @@ public List<ExecutorBuilder<?>> getExecutorBuilders(Settings settings) {
         );
     }
 
+    @Override
+    public Collection<SystemIndexDescriptor> getSystemIndexDescriptors(Settings settings) {
+        return List.of(
+            new SystemIndexDescriptor(CONFIG_INDEX, "Flow Framework Config index"),
+            new SystemIndexDescriptor(GLOBAL_CONTEXT_INDEX, "Flow Framework Global Context index"),
+            new SystemIndexDescriptor(WORKFLOW_STATE_INDEX, "Flow Framework Workflow State index")
+        );
+    }
+
 }

src/main/java/org/opensearch/flowframework/common/CommonValue.java

@@ -68,8 +68,12 @@ private CommonValue() {}
     public static final String WORKFLOW_ID = "workflow_id";
     /** Field name for template validation, the flag to indicate if validation is necessary */
     public static final String VALIDATION = "validation";
+    /** Param name for allow deletion during deprovisioning */
+    public static final String ALLOW_DELETE = "allow_delete";
     /** The field name for provision workflow within a use case template*/
     public static final String PROVISION_WORKFLOW = "provision";
+    /** The param name for update workflow field in create API */
+    public static final String UPDATE_WORKFLOW_FIELDS = "update_fields";
     /** The field name for workflow steps. This field represents the name of the workflow steps to be fetched. */
     public static final String WORKFLOW_STEP = "workflow_step";
     /** The param name for default use case, used by the create workflow API */