CI and deploy steps for docs

Signed-off-by: Pawel Leszczynski <[email protected]>
pawel-big-lebowski · Jul 25, 2024 · 2a9be71 · 2a9be71
1 parent c0f6c01
commit 2a9be71
Show file tree

Hide file tree

Showing 9 changed files with 196 additions and 170 deletions.
diff --git a/.circleci/continue_config.yml b/.circleci/continue_config.yml
@@ -993,29 +993,6 @@ jobs:
       - run: mkdir -p target/wheels && cp target/wheels/* dist/
       - run: python -m twine upload --non-interactive --verbose --repository pypi dist/*
 
-  publish-javadoc:
-    working_directory: ~/openlineage/client/java
-    docker:
-      - image: cimg/openjdk:11.0
-    steps:
-      - *checkout_project_root
-      - add_ssh_keys:
-          fingerprints:
-            - "7d:bc:78:35:09:c9:25:04:63:f9:eb:4b:f1:f4:d1:91"
-      - run: ./gradlew --console=plain javadoc
-      - run: ./release-javadoc.sh
-
-  publish-spec:
-    working_directory: ~/openlineage
-    docker:
-      - image: cimg/base:2021.07
-    steps:
-      - *checkout_project_root
-      - add_ssh_keys:
-          fingerprints:
-            - "7d:bc:78:35:09:c9:25:04:63:f9:eb:4b:f1:f4:d1:91"
-      - run: spec/release.sh
-
   build-proxy-fluentd:
     working_directory: ~/openlineage/proxy/fluentd
     docker:

diff --git a/.circleci/workflows/openlineage-integration-publish.yml b/.circleci/workflows/openlineage-integration-publish.yml
diff --git a/.circleci/workflows/openlineage-java.yml b/.circleci/workflows/openlineage-java.yml
@@ -23,12 +23,4 @@ workflows:
           requires:
             - compile-integration-sql-java-linux-arm
             - compile-integration-sql-java-linux-x86
-            - compile-integration-sql-java-macos
-      - publish-javadoc:
-          filters:
-            branches:
-              only: main
-          context: release
-      - workflow_complete:
-          requires:
-            - publish-javadoc
+            - compile-integration-sql-java-macos
diff --git a/.github/workflows/javadoc-deploy.yml b/.github/workflows/javadoc-deploy.yml
@@ -0,0 +1,59 @@
+name: Build & Deploy javadoc
+
+on:
+  push:
+    tags:
+      - '[0-9]+.[0-9]+.[0-9]+'
+    branches:
+      # testing only, not allowed to run on OpenLineage/OpenLineage repo
+      - test-docs-deploy
+
+jobs:
+  env:
+    # hack from https://github.com/orgs/community/discussions/25725#discussioncomment-3248924
+    # to allow setting variable based on conditions
+    branch: fromJSON('["main", "test-docs-deploy"]')[(github.head_ref || github.ref_name) == 'test-docs-deploy']
+  generate_javadoc:
+    if: |
+      (github.ref == 'refs/heads/test-docs-deploy' && github.repository != 'OpenLineage/OpenLineage' ) || 
+      (github.ref != 'refs/heads/test-docs-deploy' && github.repository == 'OpenLineage/OpenLineage')
+    name: "Generate Java docs"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/setup-java@v4
+        with:
+          distribution: 'corretto'
+          java-version: '11'
+      - uses: actions/checkout@v4
+      - name: Generate new java doc
+        run: |
+          cd client/java
+          ./gradlew --console=plain javadoc
+      - name: Get git branch
+        run: |
+          git config user.name github-actions
+          git config user.email [email protected]
+          git fetch
+          git checkout ${{ env.branch }}
+      - name: Clear existing javadoc
+        run: rm -rf website/static/apidocs/javadoc
+      - name: Copy docs to website directory
+        run: cp -r client/java/build/docs/javadoc website/static/apidocs
+      - name: Commit javadoc
+        run: |
+          git add website/static/apidocs/javadoc/
+          git commit -m "[generated] reloading javadoc"
+          git push
+
+  netlify-deploy-changes:
+    env:
+      # hack from https://github.com/orgs/community/discussions/25725#discussioncomment-3248924
+      # to allow setting variable based on conditions
+      branch: fromJSON('["main", "test-docs-deploy"]')[(github.head_ref || github.ref_name) == 'test-docs-deploy']
+    uses: ./.github/workflows/netlify-deploy.yml
+    needs: generate_javadoc
+    inputs:
+      branch: ${{ env.branch }} # shall be `main` or `test-docs-deploy` for testing
+    secrets:
+      NETLIFY_SITE_ID: ${{ secrets.NETLIFY_SITE_ID }}
+      NETLIFY_TOKEN_SECRET: ${{ secrets.NETLIFY_TOKEN_SECRET }}
diff --git a/.github/workflows/netlify-deploy.yml b/.github/workflows/netlify-deploy.yml
@@ -0,0 +1,51 @@
+name: 'Reusable netlify deploy workflow'
+
+on:
+  workflow_call:
+    inputs:
+      branch:
+        required: true
+        type: string
+    secrets:
+      NETLIFY_SITE_ID:
+        required: true
+      NETLIFY_TOKEN_SECRET:
+        required: true
+
+jobs:
+  netlify-deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Repository Checkout
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.branch }}
+      - name: Setup NodeJS
+        uses: actions/setup-node@v4
+        with:
+          node-version: 22
+          cache: "npm"
+          cache-dependency-path: website/package-lock.json
+      - name: Install Netlify
+        run: |
+          cd website
+          npm install [email protected] -g
+      - name: Install Dependencies
+        run: |
+          cd website
+          yarn
+      - name: Build project
+        run: |
+          cd website
+          yarn build
+      - name: Deploy to Netlify
+        id: netlify_deploy
+        run: |
+          cd website/build
+          netlify deploy \
+            --cwd . \
+            --dir . \
+            --site ${{ secrets.NETLIFY_SITE_ID }} \
+            --auth ${{ secrets.NETLIFY_TOKEN_SECRET }} \
+            --prod \
+            --open
diff --git a/.github/workflows/spec-deploy.yml b/.github/workflows/spec-deploy.yml
@@ -0,0 +1,84 @@
+name: Build & Deploy spec to Netlify
+
+on:
+  pull_request:
+    branches:
+      - main
+  push:
+    branches:
+      # testing only, not allowed to run on OpenLineage/OpenLineage repo
+      - test-docs-deploy
+
+jobs:
+  generate_spec:
+    env:
+      branch: ${{ github.head_ref || github.ref_name }}
+    if: |
+      (github.ref == 'refs/heads/test-docs-deploy' && github.repository != 'OpenLineage/OpenLineage' ) || 
+      (github.event.pull_request.merged == true && github.repository == 'OpenLineage/OpenLineage')
+    name: 'Commit Spec definition'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 2
+      - name: Check changes
+        id: check-changes
+        run: |
+          if git diff --name-only --exit-code HEAD^ HEAD -- 'spec/*.json' 'spec/OpenLineage.yml' >> /dev/null; then
+            echo "no changes in spec detected, skipping publishing spec"
+            exit 0
+          fi
+          exit -1
+        continue-on-error: true
+      - name: Check & Commit spec changes
+        if: steps.check-changes.outcome == 'success'
+        run: |
+          # Copy changed spec JSON files to target location
+          git diff --name-only HEAD^ HEAD -- 'spec/*.json' | while read LINE; do
+            # ignore registry files
+            if [[ $LINE =~ "registry.json" ]]; then
+              continue
+            fi
+          
+            # extract target file name from $id field in spec files
+            URL=$(cat $LINE | jq -r '.["$id"]')
+          
+            # extract target location in website repo
+            LOC="website/static/${URL#*//*/}"
+            LOC_DIR="${LOC%/*}"
+          
+            # create dir if necessary, and copy files
+            echo "change detected in $LINE"
+            mkdir -p $LOC_DIR
+            cp $LINE $LOC
+          done
+      - name: Commit changes
+        if: steps.check-changes.outcome == 'success'
+        run: |    
+          # verify if there are any changes
+          if [[ $(git status --porcelain | wc -l) -gt 0 ]]; then
+              git config user.name github-actions
+              git config user.email [email protected]
+              git fetch
+              git checkout ${{ env.branch }}
+              git add website/static/spec/*
+              git commit -m "[generated] adding spec changes"
+              git push
+            else
+              echo "nothing to commit"
+          fi
+
+  # TODO: openapi build, question should be committed ?
+  # step similar to this: https://github.com/OpenLineage/docs/blob/main/scripts/build-docs.sh
+
+  netlify-deploy-changes:
+    env:
+      branch: ${{ github.head_ref || github.ref_name }}
+    uses: ./.github/workflows/netlify-deploy.yml
+    needs: generate_spec
+    inputs:
+      branch: ${{ env.branch }} # shall be `main` or `test-docs-deploy` for testing
+    secrets:
+      NETLIFY_SITE_ID: ${{ secrets.NETLIFY_SITE_ID }}
+      NETLIFY_TOKEN_SECRET: ${{ secrets.NETLIFY_TOKEN_SECRET }}
diff --git a/client/java/release-javadoc.sh b/client/java/release-javadoc.sh
diff --git a/spec/OpenLineage.json b/spec/OpenLineage.json
@@ -1,6 +1,6 @@
 {
   "$schema": "https://json-schema.org/draft/2020-12/schema",
-  "$id": "https://openlineage.io/spec/2-0-2/OpenLineage.json",
+  "$id": "https://openlineage.io/spec/2-0-/OpenLineage.json",
   "$defs": {
     "BaseEvent": {
       "type": "object",

diff --git a/spec/release.sh b/spec/release.sh