From 71123d47eb6d6d3b335b4e4234741d60836d1a07 Mon Sep 17 00:00:00 2001 From: Owais Kazi Date: Tue, 10 Oct 2023 14:38:04 -0700 Subject: [PATCH] Publishes snapshot to maven and adds DEV guide (#68) * Publish snapshot to maven Signed-off-by: Owais Kazi * Updated build.gradle file to publish artifacts correctly Signed-off-by: Owais Kazi * Added DEV guide to the repo Signed-off-by: Owais Kazi * Addressed PR comment Signed-off-by: Owais Kazi --------- Signed-off-by: Owais Kazi --- .github/workflows/publish-snapshots.yml | 38 ++++++++ DEVELOPER_GUIDE.md | 95 +++++++++++++++++++ build.gradle | 72 +++++++++----- .../flowframework/workflow/WorkflowStep.java | 1 + .../resources/rest-api-spec/test/10_basic.yml | 2 +- 5 files changed, 183 insertions(+), 25 deletions(-) create mode 100644 .github/workflows/publish-snapshots.yml create mode 100644 DEVELOPER_GUIDE.md diff --git a/.github/workflows/publish-snapshots.yml b/.github/workflows/publish-snapshots.yml new file mode 100644 index 000000000..1a1516b72 --- /dev/null +++ b/.github/workflows/publish-snapshots.yml @@ -0,0 +1,38 @@ +name: Publish snapshots to maven + +on: + workflow_dispatch: + push: + branches: + - 'main' + - '1.*' + - '2.*' + +jobs: + build-and-publish-snapshots: + strategy: + fail-fast: false + if: github.repository == 'opensearch-project/opensearch-ai-flow-framework' + runs-on: ubuntu-latest + + permissions: + id-token: write + contents: write + + steps: + - uses: actions/setup-java@v3 + with: + distribution: temurin # Temurin is a distribution of adoptium + java-version: 17 + - uses: actions/checkout@v4 + - uses: aws-actions/configure-aws-credentials@v1.7.0 + with: + role-to-assume: ${{ secrets.PUBLISH_SNAPSHOTS_ROLE }} + aws-region: us-east-1 + - name: publish snapshots to maven + run: | + export SONATYPE_USERNAME=$(aws secretsmanager get-secret-value --secret-id maven-snapshots-username --query SecretString --output text) + export SONATYPE_PASSWORD=$(aws secretsmanager get-secret-value --secret-id maven-snapshots-password --query SecretString --output text) + echo "::add-mask::$SONATYPE_USERNAME" + echo "::add-mask::$SONATYPE_PASSWORD" + ./gradlew publishPluginZipPublicationToSnapshotsRepository diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md new file mode 100644 index 000000000..a7cb1a0b6 --- /dev/null +++ b/DEVELOPER_GUIDE.md @@ -0,0 +1,95 @@ +- [Developer Guide](#developer-guide) + - [Forking and Cloning](#forking-and-cloning) + - [Install Prerequisites](#install-prerequisites) + - [JDK 14](#jdk-14) + - [Setup](#setup) + - [Build](#build) + - [Building from the command line](#building-from-the-command-line) + - [Building from the IDE](#building-from-the-ide) + - [Backports](#backports) + - [Publishing](#publishing) + - [Publishing to Maven Local](#publishing-to-maven-local) + - [Generating artifacts](#generating-artifacts) + +## Developer Guide + +### Forking and Cloning + +Fork this repository on GitHub, and clone locally with `git clone`. + +### Install Prerequisites + +#### JDK 14 + +OpenSearch components build using Java 14 at a minimum. This means you must have a JDK 14 installed with the environment variable `JAVA_HOME` referencing the path to Java home for your JDK 14 installation, e.g. `JAVA_HOME=/usr/lib/jvm/jdk-14`. + +### Setup + +1. Clone the repository (see [Forking and Cloning](#forking-and-cloning)) +2. Make sure `JAVA_HOME` is pointing to a Java 14 JDK (see [Install Prerequisites](#install-prerequisites)) +3. Launch Intellij IDEA, Choose Import Project and select the settings.gradle file in the root of this package. + +### Build + +This package uses the [Gradle](https://docs.gradle.org/current/userguide/userguide.html) build system. Gradle comes with excellent documentation that should be your first stop when trying to figure out how to operate or modify the build. we also use the OpenSearch build tools for Gradle. These tools are idiosyncratic and don't always follow the conventions and instructions for building regular Java code using Gradle. Not everything in this package will work the way it's described in the Gradle documentation. If you encounter such a situation, the OpenSearch build tools [source code](https://github.com/opensearch-project/OpenSearch/tree/main/buildSrc/src/main/groovy/org/opensearch/gradle) is your best bet for figuring out what's going on. + +#### Building from the command line + +1. `./gradlew check` builds and tests. +2. `./gradlew :run` runs the plugin. +3. `./gradlew spotlessApply` formats code. And/or import formatting rules in [formatterConfig.xml](formatter/formatterConfig.xml) with IDE. +4. `./gradlew test` to run the complete test suite. + +#### Building from the IDE + +Currently, the only IDE we support is IntelliJ IDEA. It's free, it's open source, it works. The gradle tasks above can also be launched from IntelliJ's Gradle toolbar and the extra parameters can be passed in via the Launch Configurations VM arguments. + +### Backports + +The Github workflow in [`backport.yml`](.github/workflows/backport.yml) creates backport PRs automatically when the +original PR with an appropriate label `backport ` is merged to main with the backport workflow +run successfully on the PR. For example, if a PR on main needs to be backported to `2.x` branch, add a label +`backport 2.x` to the PR and make sure the backport workflow runs on the PR along with other checks. Once this PR is +merged to main, the workflow will create a backport PR to the `2.x` branch. + +### Publishing + +#### Publishing to Maven Local +Run the below command to publish the artifacts to maven local. +```./gradlew publishToMavenLocal``` + +#### Generating artifacts + +To generate the below artifacts on local +``` +snapshots/ +└── org + └── opensearch + └── plugin + └── opensearch-ai-flow-framework + ├── 3.0.0.0-SNAPSHOT + │   ├── maven-metadata.xml + │   ├── maven-metadata.xml.md5 + │   ├── maven-metadata.xml.sha1 + │   ├── maven-metadata.xml.sha256 + │   ├── maven-metadata.xml.sha512 + │   ├── opensearch-ai-flow-framework-3.0.0.0-20231005.170838-1.pom + │   ├── opensearch-ai-flow-framework-3.0.0.0-20231005.170838-1.pom.md5 + │   ├── opensearch-ai-flow-framework-3.0.0.0-20231005.170838-1.pom.sha1 + │   ├── opensearch-ai-flow-framework-3.0.0.0-20231005.170838-1.pom.sha256 + │   ├── opensearch-ai-flow-framework-3.0.0.0-20231005.170838-1.pom.sha512 + │   ├── opensearch-ai-flow-framework-3.0.0.0-20231005.170838-1.zip + │   ├── opensearch-ai-flow-framework-3.0.0.0-20231005.170838-1.zip.md5 + │   ├── opensearch-ai-flow-framework-3.0.0.0-20231005.170838-1.zip.sha1 + │   ├── opensearch-ai-flow-framework-3.0.0.0-20231005.170838-1.zip.sha256 + │   └── opensearch-ai-flow-framework-3.0.0.0-20231005.170838-1.zip.sha512 + ├── maven-metadata.xml + ├── maven-metadata.xml.md5 + ├── maven-metadata.xml.sha1 + ├── maven-metadata.xml.sha256 + └── maven-metadata.xml.sha512 + + +``` +1. Change the url from ``"https://aws.oss.sonatype.org/content/repositories/snapshots"`` to your local path and comment out the credentials under publishing/repositories in build.gradle. +2. Run ```./gradlew publishPluginZipPublicationToSnapshotsRepository```. diff --git a/build.gradle b/build.gradle index 74e09b3bc..758687c8a 100644 --- a/build.gradle +++ b/build.gradle @@ -15,36 +15,13 @@ apply plugin: 'checkstyle' apply plugin: 'jacoco' apply plugin: 'com.form.diff-coverage' -def pluginName = 'flow-framework' +def pluginName = 'opensearch-flow-framework' def pluginDescription = 'OpenSearch plugin that enables builders to innovate AI apps on OpenSearch' def projectPath = 'org.opensearch' def pathToPlugin = 'flowframework' def pluginClassName = 'FlowFrameworkPlugin' group = "org.opensearch.flowframework" -publishing { - publications { - pluginZip(MavenPublication) { publication -> - pom { - name = pluginName - description = pluginDescription - licenses { - license { - name = "The Apache License, Version 2.0" - url = "http://www.apache.org/licenses/LICENSE-2.0.txt" - } - } - developers { - developer { - name = "OpenSearch AI Flow Framework Plugin" - url = "https://github.com/opensearch-project/opensearch-ai-flow-framework" - } - } - } - } - } -} - opensearchplugin { name pluginName description pluginDescription @@ -96,6 +73,49 @@ buildscript { } } + +publishing { + publications { + pluginZip(MavenPublication) { publication -> + pom { + name = pluginName + description = pluginDescription + groupId = "org.opensearch.plugin" + licenses { + license { + name = "The Apache License, Version 2.0" + url = "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + } + developers { + developer { + name = "OpenSearch AI Flow Framework Plugin" + url = "https://github.com/opensearch-project/opensearch-ai-flow-framework" + } + } + } + } + } + + repositories { + maven { + name = "Snapshots" + url = "https://aws.oss.sonatype.org/content/repositories/snapshots" + credentials { + username "$System.env.SONATYPE_USERNAME" + password "$System.env.SONATYPE_PASSWORD" + } + } + } +} + +allprojects { + group = opensearch_group + version = "${opensearch_build}" + targetCompatibility = JavaVersion.VERSION_11 + sourceCompatibility = JavaVersion.VERSION_11 +} + repositories { mavenLocal() maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" } @@ -103,6 +123,10 @@ repositories { maven { url "https://plugins.gradle.org/m2/" } } +configurations { + zipArchive +} + dependencies { implementation "org.opensearch:opensearch:${opensearch_version}" implementation 'org.junit.jupiter:junit-jupiter:5.10.0' diff --git a/src/main/java/org/opensearch/flowframework/workflow/WorkflowStep.java b/src/main/java/org/opensearch/flowframework/workflow/WorkflowStep.java index c7e5a3141..bdbe63caa 100644 --- a/src/main/java/org/opensearch/flowframework/workflow/WorkflowStep.java +++ b/src/main/java/org/opensearch/flowframework/workflow/WorkflowStep.java @@ -21,6 +21,7 @@ public interface WorkflowStep { * Triggers the actual processing of the building block. * @param data representing input params and content, or output content of previous steps. The first element of the list is data (if any) provided from parsing the template, and may be {@link WorkflowData#EMPTY}. * @return A CompletableFuture of the building block. This block should return immediately, but not be completed until the step executes, containing either the step's output data or {@link WorkflowData#EMPTY} which may be passed to follow-on steps. + * @throws IOException on a failure. */ CompletableFuture execute(List data) throws IOException; diff --git a/src/yamlRestTest/resources/rest-api-spec/test/10_basic.yml b/src/yamlRestTest/resources/rest-api-spec/test/10_basic.yml index 467f451ed..e2bbf793f 100644 --- a/src/yamlRestTest/resources/rest-api-spec/test/10_basic.yml +++ b/src/yamlRestTest/resources/rest-api-spec/test/10_basic.yml @@ -5,4 +5,4 @@ h: component - match: - $body: /^flow-framework\n$/ + $body: /^opensearch-flow-framework\n$/