CI and deploy steps for docs

Signed-off-by: Pawel Leszczynski <[email protected]>
pawel-big-lebowski · Aug 1, 2024 · 18ce7a6 · 18ce7a6
1 parent c0f6c01
commit 18ce7a6
Show file tree

Hide file tree

Showing 11 changed files with 210 additions and 172 deletions.
diff --git a/.circleci/continue_config.yml b/.circleci/continue_config.yml
@@ -993,29 +993,6 @@ jobs:
       - run: mkdir -p target/wheels && cp target/wheels/* dist/
       - run: python -m twine upload --non-interactive --verbose --repository pypi dist/*
 
-  publish-javadoc:
-    working_directory: ~/openlineage/client/java
-    docker:
-      - image: cimg/openjdk:11.0
-    steps:
-      - *checkout_project_root
-      - add_ssh_keys:
-          fingerprints:
-            - "7d:bc:78:35:09:c9:25:04:63:f9:eb:4b:f1:f4:d1:91"
-      - run: ./gradlew --console=plain javadoc
-      - run: ./release-javadoc.sh
-
-  publish-spec:
-    working_directory: ~/openlineage
-    docker:
-      - image: cimg/base:2021.07
-    steps:
-      - *checkout_project_root
-      - add_ssh_keys:
-          fingerprints:
-            - "7d:bc:78:35:09:c9:25:04:63:f9:eb:4b:f1:f4:d1:91"
-      - run: spec/release.sh
-
   build-proxy-fluentd:
     working_directory: ~/openlineage/proxy/fluentd
     docker:

diff --git a/.circleci/workflows/openlineage-integration-publish.yml b/.circleci/workflows/openlineage-integration-publish.yml
diff --git a/.circleci/workflows/openlineage-java.yml b/.circleci/workflows/openlineage-java.yml
@@ -23,12 +23,4 @@ workflows:
           requires:
             - compile-integration-sql-java-linux-arm
             - compile-integration-sql-java-linux-x86
-            - compile-integration-sql-java-macos
-      - publish-javadoc:
-          filters:
-            branches:
-              only: main
-          context: release
-      - workflow_complete:
-          requires:
-            - publish-javadoc
+            - compile-integration-sql-java-macos
diff --git a/.github/workflows/javadoc-deploy.yml b/.github/workflows/javadoc-deploy.yml
@@ -0,0 +1,51 @@
+name: Build & Deploy javadoc
+
+on:
+  push:
+    tags:
+      - '[0-9]+.[0-9]+.[0-9]+'
+    branches:
+      # testing only, not allowed to run on OpenLineage/OpenLineage repo
+      - test-docs-deploy
+
+jobs:
+  generate_javadoc:
+    if: |
+      (github.ref == 'refs/heads/test-docs-deploy' && github.repository != 'OpenLineage/OpenLineage' ) || 
+      (github.ref != 'refs/heads/test-docs-deploy' && github.repository == 'OpenLineage/OpenLineage')
+    name: "Generate Java docs"
+    env:
+      branch: ${{ (github.head_ref || github.ref_name) == 'test-docs-deploy' && 'test-docs-deploy' || 'main'  }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/setup-java@v4
+        with:
+          distribution: 'corretto'
+          java-version: '11'
+      - uses: actions/checkout@v4
+      - name: Generate new java doc
+        run: |
+          cd client/java
+          ./gradlew --console=plain javadoc
+      - name: Get git branch
+        run: |
+          git config user.name github-actions
+          git config user.email [email protected]
+          git fetch
+          git checkout ${{ env.branch }}
+      - name: Clear existing javadoc
+        run: rm -rf website/static/apidocs/javadoc
+      - name: Copy docs to website directory
+        run: cp -r client/java/build/docs/javadoc website/static/apidocs
+      - name: Commit javadoc
+        run: |
+          git pull
+          git add website/static/apidocs/javadoc/
+          git commit -m "[generated] reloading javadoc"
+          git push
+
+  site-deploy-changes:
+    uses: ./.github/workflows/site-deploy.yml
+    needs: generate_javadoc
+    with:
+      branch: ${{ (github.head_ref || github.ref_name) == 'test-docs-deploy' && 'test-docs-deploy' || 'main'  }}
diff --git a/.github/workflows/site-deploy.yml b/.github/workflows/site-deploy.yml
@@ -0,0 +1,37 @@
+name: 'Reusable netlify deploy workflow'
+
+on:
+  workflow_call:
+    inputs:
+      branch:
+        required: true
+        type: string
+
+jobs:
+  site-deploy:
+    name: Deploy to GitHub Pages
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.branch }}
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 22
+          cache: "npm"
+          cache-dependency-path: website/package-lock.json
+      - name: Install dependencies
+        run: yarn install --frozen-lockfile
+      - name: Build openapi docs
+        run: yarn run build:docs
+      - name: Build website
+        run: yarn build
+      - name: Setup Pages
+        uses: actions/configure-pages@v5
+      - name: Upload artifact
+        uses: actions/upload-pages-artifact@v3
+        with:
+          path: './website/build'
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v4
diff --git a/.github/workflows/spec-deploy.yml b/.github/workflows/spec-deploy.yml
@@ -0,0 +1,118 @@
+name: Build & Deploy spec to Netlify
+
+on:
+  pull_request:
+    branches:
+      - main
+  push:
+    branches:
+      # testing only, not allowed to run on OpenLineage/OpenLineage repo
+      - test-docs-deploy
+
+jobs:
+  generate_spec:
+    if: |
+      (github.ref == 'refs/heads/test-docs-deploy' && github.repository != 'OpenLineage/OpenLineage' ) || 
+      (github.event.pull_request.merged == true && github.repository == 'OpenLineage/OpenLineage')
+    name: 'Commit Spec definition'
+    runs-on: ubuntu-latest
+    env:
+      branch: ${{ github.head_ref || github.ref_name }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 2
+      - uses: shogo82148/actions-setup-perl@v1
+        with:
+          perl-version: "5.38"
+      - name: Setup NodeJS
+        uses: actions/setup-node@v4
+        with:
+          node-version: 22
+          cache: "npm"
+          cache-dependency-path: website/package-lock.json
+      - name: Check changes
+        id: check-changes
+        run: |
+          if git diff --name-only --exit-code HEAD^ HEAD -- 'spec/*.json' 'spec/OpenLineage.yml' >> /dev/null; then
+            echo "no changes in spec detected, skipping publishing spec"
+            exit 1
+          fi
+          echo "spec changes detected"
+          exit 0
+        continue-on-error: true
+      - name: Check spec changes
+        if: steps.check-changes.outcome == 'success'
+        run: |
+          # Copy changed spec JSON files to target location
+          git diff --name-only HEAD^ HEAD -- 'spec/*.json' | while read LINE; do
+            # ignore registry files
+            if [[ $LINE =~ "registry.json" ]]; then
+              continue
+            fi
+          
+            # extract target file name from $id field in spec files
+            URL=$(cat $LINE | jq -r '.["$id"]')
+          
+            # extract target location in website repo
+            LOC="website/static/${URL#*//*/}"
+            LOC_DIR="${LOC%/*}"
+          
+            # create dir if necessary, and copy files
+            echo "change detected in $LINE"
+            mkdir -p $LOC_DIR
+            cp $LINE $LOC
+          done
+      - name: Generate openapi
+        if: steps.check-changes.outcome == 'success'
+        run: |
+          SPEC_DIR="$(pwd)/website/static/spec"
+          APIDOC_DIR="$(pwd)/website/static/apidocs"
+
+          pushd $SPEC_DIR
+          LATEST_VERSION=$(find . -maxdepth 1 | grep -v 'facets' | grep '[0-9]*-[0-9]-[0-9]' | sort -Vr | head -1)
+          echo latest version is $LATEST_VERSION
+          rm -f ./OpenLineage.json 2>/dev/null
+          perl -i -pe"s/version: [[:alnum:]\.-]*/version: ${LATEST_VERSION:2}/g" ./OpenLineage.yml
+          echo Versions updated
+
+          mkdir "${LATEST_VERSION}/facets"
+          for i in $(ls -d ./facets/* | sort); do cp $i/*.json ${LATEST_VERSION}/facets; done;
+          
+          echo "LATEST_VERSION=${LATEST_VERSION}" >> "$GITHUB_ENV"
+          
+          pushd $LATEST_VERSION
+          ln -sf ../OpenLineage.yml .
+          popd
+      - name: Generate redoc
+        if: steps.check-changes.outcome == 'success'
+        run : |
+          npm install redoc-cli
+          yarn run redoc-cli build -o website/static/apidocs/openapi/index.html website/static/spec/$LATEST_VERSION/OpenLineage.yml --title "OpenLineage API Docs"
+      - name: Clean temp files
+        run: |
+          pushd website/static/spec/$LATEST_VERSION
+          rm -rf facets
+          rm OpenLineage.yml
+          popd
+      - name: Commit changes
+        if: steps.check-changes.outcome == 'success'
+        run: |    
+          # verify if there are any changes
+          if [[ $(git status --porcelain | wc -l) -gt 0 ]]; then
+              git config user.name github-actions
+              git config user.email [email protected]
+              git fetch
+              git checkout ${{ env.branch }}
+              git add website/static/*
+              git commit -m "[generated] adding spec changes"
+              git push
+            else
+              echo "nothing to commit"
+          fi
+
+  site-deploy-changes:
+    uses: ./.github/workflows/site-deploy.yml
+    needs: generate_spec
+    with:
+      branch: ${{ github.head_ref || github.ref_name }} # shall be `main` or `test-docs-deploy` for testing
diff --git a/client/java/release-javadoc.sh b/client/java/release-javadoc.sh
diff --git a/client/java/src/main/java/io/openlineage/client/OpenLineageClient.java b/client/java/src/main/java/io/openlineage/client/OpenLineageClient.java
@@ -20,7 +20,7 @@
 import lombok.NonNull;
 import lombok.extern.slf4j.Slf4j;
 
-/** HTTP client used to emit {@link OpenLineage.RunEvent}s to HTTP backend. */
+/** HTTP client used to emit {@link OpenLineage.RunEvent}s to HTTP backend. DUMMY TEST CHANGE */
 @Slf4j
 public final class OpenLineageClient {
   final Transport transport;

diff --git a/spec/OpenLineage.json b/spec/OpenLineage.json
@@ -1,6 +1,6 @@
 {
   "$schema": "https://json-schema.org/draft/2020-12/schema",
-  "$id": "https://openlineage.io/spec/2-0-2/OpenLineage.json",
+  "$id": "https://openlineage.io/spec/2-0-10/OpenLineage.json",
   "$defs": {
     "BaseEvent": {
       "type": "object",