CI and deploy steps for docs

Signed-off-by: Pawel Leszczynski <[email protected]>

.circleci/continue_config.yml

@@ -993,29 +993,6 @@ jobs:
       - run: mkdir -p target/wheels && cp target/wheels/* dist/
       - run: python -m twine upload --non-interactive --verbose --repository pypi dist/*
 
-  publish-javadoc:
-    working_directory: ~/openlineage/client/java
-    docker:
-      - image: cimg/openjdk:11.0
-    steps:
-      - *checkout_project_root
-      - add_ssh_keys:
-          fingerprints:
-            - "7d:bc:78:35:09:c9:25:04:63:f9:eb:4b:f1:f4:d1:91"
-      - run: ./gradlew --console=plain javadoc
-      - run: ./release-javadoc.sh
-
-  publish-spec:
-    working_directory: ~/openlineage
-    docker:
-      - image: cimg/base:2021.07
-    steps:
-      - *checkout_project_root
-      - add_ssh_keys:
-          fingerprints:
-            - "7d:bc:78:35:09:c9:25:04:63:f9:eb:4b:f1:f4:d1:91"
-      - run: spec/release.sh
-
   build-proxy-fluentd:
     working_directory: ~/openlineage/proxy/fluentd
     docker:

.circleci/workflows/openlineage-java.yml

@@ -23,12 +23,4 @@ workflows:
           requires:
             - compile-integration-sql-java-linux-arm
             - compile-integration-sql-java-linux-x86
-            - compile-integration-sql-java-macos
-      - publish-javadoc:
-          filters:
-            branches:
-              only: main
-          context: release
-      - workflow_complete:
-          requires:
-            - publish-javadoc
+            - compile-integration-sql-java-macos

.github/workflows/docs-deploy.yml

@@ -0,0 +1,45 @@
+name: Build & Deploy javadoc
+
+on:
+  push:
+    tags:
+      - '[0-9]+.[0-9]+.[0-9]+'
+    branches:
+      # testing only, not allowed to run on OpenLineage/OpenLineage repo
+      - test-docs-deploy
+
+jobs:
+  generate_javadoc:
+    if: ${{ (github.event.pull_request.base.ref == 'test-docs-deploy' && github.repository != 'OpenLineage/OpenLineage' ) ||  }}
+    name: "Generate Java docs"
+    steps:
+      - uses: actions/setup-java@v4
+        with:
+          distribution: 'corretto'
+          java-version: '11'
+      - name: Clear existing javadoc
+        run: rm -rf website/static/apidocs/javadoc
+      - name: Generate new java doc
+        run: |
+          cd client/java
+          ./gradlew --console=plain javadoc
+      - name: Copy docs to website directory
+        run: cp client/java/build/docs/javadoc website/static/apidocs
+      - name: Commit javadoc to main
+        run: |
+          git config user.name github-actions
+          git config user.email [email protected]
+          git fetch
+          git checkout main
+          git add website/static/apidocs/*
+          git commit -m "[generated] reloading javadoc"
+          git push      
+
+  netlify_deploy:
+    if: ${{ (github.event.pull_request.base.ref == 'test-docs-deploy' && github.repository != 'OpenLineage/OpenLineage' ) }}
+    name: 'Deploy to Netlify'
+    runs-on: ubuntu-latest
+    needs:
+      generate_javadoc
+    steps:
+      - uses: actions/nifi-deploy-action@v1

.github/workflows/netlify-deploy-action.yml

@@ -0,0 +1,34 @@
+name: 'Netlify deploy action'
+description: 'Deploys website to netlify site'
+runs:
+  using: "composite"
+  steps:
+    - name: Repository Checkout
+      uses: actions/checkout@v4
+    - name: Setup NodeJS
+      uses: actions/setup-node@v4
+      with:
+        node-version: 20
+        cache: "npm"
+        cache-dependency-path: website/package-lock.json
+    - name: Install Netlify
+      run: |
+        cd website
+        npm install [email protected] -g
+    - name: Install Dependencies
+      run: |
+        cd website
+        yarn
+    - name: Build project
+      run: |
+        cd website
+        yarn build
+    - name: Deploy to Netlify
+      id: netlify_deploy
+      run: |
+        cd website/build
+        netlify deploy \
+          --cwd . \
+          --site ${{ secrets.NETLIFY_SITE_ID }} \
+          --auth ${{ secrets.NETLIFY_TOKEN_SECRET }} \
+          --prod

.github/workflows/spec-deploy.yml

@@ -0,0 +1,81 @@
+name: Build & Deploy spec to Netlify
+
+on:
+  pull_request:
+    branches:
+      - main
+      - test-docs-deploy # testing only, not allowed to run on OpenLineage/OpenLineage repo
+
+jobs:
+  generate_spec:
+    if: |
+       (
+          github.event.pull_request.base.ref == 'test-docs-deploy' && 
+          github.repository != 'OpenLineage/OpenLineage' 
+       ) || (
+          github.event.pull_request.merged == true && 
+          github.event.pull_request.base.ref == 'main' &&
+          github.repository == 'OpenLineage/OpenLineage
+      )
+    name: 'Commit Spec definition'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 2
+      - name: Check changes
+        id: check-changes
+        run: |
+          if git diff --name-only --exit-code HEAD^ HEAD -- 'spec/*.json' 'spec/OpenLineage.yml' >> /dev/null; then
+            echo "no changes in spec detected, skipping publishing spec"
+            exit 0
+          fi
+          exit -1
+        continue-on-error: true
+      - name: Check & Commit spec changes
+        if: steps.check-changes.outcome == 'success'
+        run: |
+          # Copy changed spec JSON files to target location
+          git diff --name-only HEAD^ HEAD -- 'spec/*.json' | while read LINE; do
+            # ignore registry files
+            if [[ $LINE =~ "registry.json" ]]; then
+              continue
+            fi
+          
+            # extract target file name from $id field in spec files
+            URL=$(cat $LINE | jq -r '.["$id"]')
+          
+            # extract target location in website repo
+            LOC="website/static/${URL#*//*/}"
+            LOC_DIR="${LOC%/*}"
+          
+            # create dir if necessary, and copy files
+            echo "change detected in $LINE"
+            mkdir -p $LOC_DIR
+            cp $LINE $LOC
+          done
+      - name: Commit changes
+        if: steps.check-changes.outcome == 'success'
+        run: |    
+          # verify if there are any changes
+          if [[ $(git status --porcelain | wc -l) -gt 0 ]]; then
+              git config user.name github-actions
+              git config user.email [email protected]
+              git fetch
+              git checkout main
+              git add website/static/spec/*
+              git commit -m "[generated] adding spec changes"
+              git push
+            else
+              echo "nothing to commit"
+          fi
+
+  # TODO: openapi build, question should be committed ?
+
+  netlify_deploy:
+    name: 'Deploy to Netlify'
+    runs-on: ubuntu-latest
+    needs:
+      generate_spec
+    steps:
+      - uses: actions/nifi-deploy-action@v1

spec/OpenLineage.json

@@ -1,6 +1,6 @@
 {
   "$schema": "https://json-schema.org/draft/2020-12/schema",
-  "$id": "https://openlineage.io/spec/2-0-2/OpenLineage.json",
+  "$id": "https://openlineage.io/spec/2-0-/OpenLineage.json",
   "$defs": {
     "BaseEvent": {
       "type": "object",