diff --git a/.circleci/continue_config.yml b/.circleci/continue_config.yml index 5c0685c270..cf5fa44060 100644 --- a/.circleci/continue_config.yml +++ b/.circleci/continue_config.yml @@ -993,29 +993,6 @@ jobs: - run: mkdir -p target/wheels && cp target/wheels/* dist/ - run: python -m twine upload --non-interactive --verbose --repository pypi dist/* - publish-javadoc: - working_directory: ~/openlineage/client/java - docker: - - image: cimg/openjdk:11.0 - steps: - - *checkout_project_root - - add_ssh_keys: - fingerprints: - - "7d:bc:78:35:09:c9:25:04:63:f9:eb:4b:f1:f4:d1:91" - - run: ./gradlew --console=plain javadoc - - run: ./release-javadoc.sh - - publish-spec: - working_directory: ~/openlineage - docker: - - image: cimg/base:2021.07 - steps: - - *checkout_project_root - - add_ssh_keys: - fingerprints: - - "7d:bc:78:35:09:c9:25:04:63:f9:eb:4b:f1:f4:d1:91" - - run: spec/release.sh - build-proxy-fluentd: working_directory: ~/openlineage/proxy/fluentd docker: diff --git a/.circleci/workflows/openlineage-integration-publish.yml b/.circleci/workflows/openlineage-integration-publish.yml deleted file mode 100644 index 10ac69c238..0000000000 --- a/.circleci/workflows/openlineage-integration-publish.yml +++ /dev/null @@ -1,11 +0,0 @@ -workflows: - openlineage-integration-publish: - jobs: - - publish-spec: - filters: - branches: - only: main - context: release - - workflow_complete: - requires: - - publish-spec diff --git a/.circleci/workflows/openlineage-java.yml b/.circleci/workflows/openlineage-java.yml index 4ff1484262..750d346c8d 100644 --- a/.circleci/workflows/openlineage-java.yml +++ b/.circleci/workflows/openlineage-java.yml @@ -23,12 +23,4 @@ workflows: requires: - compile-integration-sql-java-linux-arm - compile-integration-sql-java-linux-x86 - - compile-integration-sql-java-macos - - publish-javadoc: - filters: - branches: - only: main - context: release - - workflow_complete: - requires: - - publish-javadoc \ No newline at end of file + - compile-integration-sql-java-macos \ No newline at end of file diff --git a/.github/workflows/docs-deploy.yml b/.github/workflows/docs-deploy.yml new file mode 100644 index 0000000000..ae2e95f072 --- /dev/null +++ b/.github/workflows/docs-deploy.yml @@ -0,0 +1,45 @@ +name: Build & Deploy javadoc + +on: + push: + tags: + - '[0-9]+.[0-9]+.[0-9]+' + branches: + # testing only, not allowed to run on OpenLineage/OpenLineage repo + - test-docs-deploy + +jobs: + generate_javadoc: + if: ${{ (github.event.pull_request.base.ref == 'test-docs-deploy' && github.repository != 'OpenLineage/OpenLineage' ) || }} + name: "Generate Java docs" + steps: + - uses: actions/setup-java@v4 + with: + distribution: 'corretto' + java-version: '11' + - name: Clear existing javadoc + run: rm -rf website/static/apidocs/javadoc + - name: Generate new java doc + run: | + cd client/java + ./gradlew --console=plain javadoc + - name: Copy docs to website directory + run: cp client/java/build/docs/javadoc website/static/apidocs + - name: Commit javadoc to main + run: | + git config user.name github-actions + git config user.email github-actions@github.com + git fetch + git checkout main + git add website/static/apidocs/* + git commit -m "[generated] reloading javadoc" + git push + + netlify_deploy: + if: ${{ (github.event.pull_request.base.ref == 'test-docs-deploy' && github.repository != 'OpenLineage/OpenLineage' ) }} + name: 'Deploy to Netlify' + runs-on: ubuntu-latest + needs: + generate_javadoc + steps: + - uses: actions/nifi-deploy-action@v1 \ No newline at end of file diff --git a/.github/workflows/netlify-deploy-action.yml b/.github/workflows/netlify-deploy-action.yml new file mode 100644 index 0000000000..5c651fc0a3 --- /dev/null +++ b/.github/workflows/netlify-deploy-action.yml @@ -0,0 +1,34 @@ +name: 'Netlify deploy action' +description: 'Deploys website to netlify site' +runs: + using: "composite" + steps: + - name: Repository Checkout + uses: actions/checkout@v4 + - name: Setup NodeJS + uses: actions/setup-node@v4 + with: + node-version: 20 + cache: "npm" + cache-dependency-path: website/package-lock.json + - name: Install Netlify + run: | + cd website + npm install netlify-cli@17.10.1 -g + - name: Install Dependencies + run: | + cd website + yarn + - name: Build project + run: | + cd website + yarn build + - name: Deploy to Netlify + id: netlify_deploy + run: | + cd website/build + netlify deploy \ + --cwd . \ + --site ${{ secrets.NETLIFY_SITE_ID }} \ + --auth ${{ secrets.NETLIFY_TOKEN_SECRET }} \ + --prod \ No newline at end of file diff --git a/.github/workflows/spec-deploy.yml b/.github/workflows/spec-deploy.yml new file mode 100644 index 0000000000..c1446da1ef --- /dev/null +++ b/.github/workflows/spec-deploy.yml @@ -0,0 +1,81 @@ +name: Build & Deploy spec to Netlify + +on: + pull_request: + branches: + - main + - test-docs-deploy # testing only, not allowed to run on OpenLineage/OpenLineage repo + +jobs: + generate_spec: + if: | + ( + github.event.pull_request.base.ref == 'test-docs-deploy' && + github.repository != 'OpenLineage/OpenLineage' + ) || ( + github.event.pull_request.merged == true && + github.event.pull_request.base.ref == 'main' && + github.repository == 'OpenLineage/OpenLineage + ) + name: 'Commit Spec definition' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 2 + - name: Check changes + id: check-changes + run: | + if git diff --name-only --exit-code HEAD^ HEAD -- 'spec/*.json' 'spec/OpenLineage.yml' >> /dev/null; then + echo "no changes in spec detected, skipping publishing spec" + exit 0 + fi + exit -1 + continue-on-error: true + - name: Check & Commit spec changes + if: steps.check-changes.outcome == 'success' + run: | + # Copy changed spec JSON files to target location + git diff --name-only HEAD^ HEAD -- 'spec/*.json' | while read LINE; do + # ignore registry files + if [[ $LINE =~ "registry.json" ]]; then + continue + fi + + # extract target file name from $id field in spec files + URL=$(cat $LINE | jq -r '.["$id"]') + + # extract target location in website repo + LOC="website/static/${URL#*//*/}" + LOC_DIR="${LOC%/*}" + + # create dir if necessary, and copy files + echo "change detected in $LINE" + mkdir -p $LOC_DIR + cp $LINE $LOC + done + - name: Commit changes + if: steps.check-changes.outcome == 'success' + run: | + # verify if there are any changes + if [[ $(git status --porcelain | wc -l) -gt 0 ]]; then + git config user.name github-actions + git config user.email github-actions@github.com + git fetch + git checkout main + git add website/static/spec/* + git commit -m "[generated] adding spec changes" + git push + else + echo "nothing to commit" + fi + + # TODO: openapi build, question should be committed ? + + netlify_deploy: + name: 'Deploy to Netlify' + runs-on: ubuntu-latest + needs: + generate_spec + steps: + - uses: actions/nifi-deploy-action@v1 \ No newline at end of file diff --git a/client/java/release-javadoc.sh b/client/java/release-javadoc.sh deleted file mode 100755 index 9bd9f70aca..0000000000 --- a/client/java/release-javadoc.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright 2018-2024 contributors to the OpenLineage project -# SPDX-License-Identifier: Apache-2.0 -# - -set -e - -function git-website() { - command git --git-dir "$WEBSITE_DIR/.git" --work-tree "$WEBSITE_DIR" $@ -} - -git config --global user.email "openlineage-bot-key@openlineage.io" -git config --global user.name "OpenLineage deploy bot" - -WEBSITE_DIR=${WEBSITE_DIR:-$HOME/build/website} -REPO="git@github.com:OpenLineage/docs" - -if [[ -d $WEBSITE_DIR ]]; then - # Check if we're in git repository and the repository points at website - if [[ $(git-website rev-parse --is-inside-work-tree) == "true" && $(git-website config --get remote.origin.url) == "$REPO" ]]; then - # Make sure we're at the head of the main branch - git checkout main - git reset --hard origin/master - else - echo "$WEBSITE_DIR is not empty - failing" - exit 1 - fi -else - git clone --depth 1 $REPO $WEBSITE_DIR -fi - -# check if there are any changes in javadoc in the latest commit -if [[ $(diff -qr $WEBSITE_DIR/static/apidocs/javadoc './build/docs/javadoc' | wc -l) -eq 0 ]]; then - echo "no changes in javadoc detected, skipping publishing javadoc" - exit 0 -fi - -echo "Changes detected, updating javadoc..." -rm -rf $WEBSITE_DIR/static/apidocs/javadoc -mv ./build/docs/javadoc $WEBSITE_DIR/static/apidocs - -# commit new spec and push -git-website add -A static/apidocs/javadoc -git --git-dir "$WEBSITE_DIR/.git" --work-tree "$WEBSITE_DIR" commit -m "openlineage javadoc update" -git-website push diff --git a/spec/OpenLineage.json b/spec/OpenLineage.json index 58167f7140..d869f65bb8 100644 --- a/spec/OpenLineage.json +++ b/spec/OpenLineage.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://openlineage.io/spec/2-0-2/OpenLineage.json", + "$id": "https://openlineage.io/spec/2-0-/OpenLineage.json", "$defs": { "BaseEvent": { "type": "object", diff --git a/spec/release.sh b/spec/release.sh deleted file mode 100755 index 138f0a6ba6..0000000000 --- a/spec/release.sh +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright 2018-2024 contributors to the OpenLineage project -# SPDX-License-Identifier: Apache-2.0 - -set -e - -function git-website() { - command git --git-dir "$WEBSITE_DIR/.git" --work-tree "$WEBSITE_DIR" $@ -} - -git config --global user.email "openlineage-bot-key@openlineage.io" -git config --global user.name "OpenLineage deploy bot" - -WEBSITE_DIR=${WEBSITE_DIR:-$HOME/build/website} -REPO="git@github.com:OpenLineage/docs" - -if [[ -d $WEBSITE_DIR ]]; then - # Check if we're in git repository and the repository points at website - if [[ $(git-website rev-parse --is-inside-work-tree) == "true" && $(git-website config --get remote.origin.url) == "$REPO" ]]; then - # Make sure we're at the head of the main branch - git checkout main - git reset --hard origin/master - else - echo "$WEBSITE_DIR is not empty - failing" - exit 1 - fi -else - git clone --depth 1 $REPO $WEBSITE_DIR -fi - -WEBSITE_COMMIT_FILE="$WEBSITE_DIR/.last_spec_commit_id" - -# Check on which commit we deployed spec last -if [[ -f $WEBSITE_COMMIT_FILE ]]; then - PREV_SPEC_COMMIT=$(cat "$WEBSITE_COMMIT_FILE") -else - # Before lifecycle state facet - PREV_SPEC_COMMIT="d66c41872f3cc7f7cd5c99664d401e070e60ff48" -fi - -# check if there are any changes in spec in the latest commit -if git diff --name-only --exit-code $PREV_SPEC_COMMIT HEAD 'spec/*.json' 'spec/OpenLineage.yml' >> /dev/null; then - echo "no changes in spec detected, skipping publishing spec" - exit 0 -fi - -echo "Copying spec files from commit $PREV_SPEC_COMMIT" - -# Mark last commit on which we finished copying spec -echo "$CIRCLE_SHA1" > "$WEBSITE_COMMIT_FILE" - -# Copy changed spec YML file to target location -cp spec/OpenLineage.yml ${WEBSITE_DIR}/static/spec/OpenLineage.yml - -# Copy changed spec JSON files to target location -git diff --name-only $PREV_SPEC_COMMIT HEAD 'spec/*.json' | while read LINE; do - - #ignore registry files - if [[ $LINE =~ "registry.json" ]]; then - continue - fi - - # extract target file name from $id field in spec files - URL=$(cat $LINE | jq -r '.["$id"]') - - # extract target location in website repo - LOC="${WEBSITE_DIR}/static/${URL#*//*/}" - LOC_DIR="${LOC%/*}" - - # create dir if necessary, and copy files - mkdir -p $LOC_DIR - cp $LINE $LOC -done - -# commit new spec and push -git-website add -A .last_spec_commit_id -git-website add -A static/spec/ -git --git-dir "$WEBSITE_DIR/.git" --work-tree "$WEBSITE_DIR" commit -m "openlineage specification update" -git-website push