From 18ce7a678dc95e3749770a837b533a15f2dc10f6 Mon Sep 17 00:00:00 2001
From: Pawel Leszczynski <leszczynski.pawel@gmail.com>
Date: Mon, 22 Jul 2024 14:48:49 +0200
Subject: [PATCH] CI and deploy steps for docs

Signed-off-by: Pawel Leszczynski <leszczynski.pawel@gmail.com>
---
 .circleci/continue_config.yml                 |  23 ----
 .../openlineage-integration-publish.yml       |  11 --
 .circleci/workflows/openlineage-java.yml      |  10 +-
 .github/workflows/javadoc-deploy.yml          |  51 ++++++++
 .github/workflows/site-deploy.yml             |  37 ++++++
 .github/workflows/spec-deploy.yml             | 118 ++++++++++++++++++
 client/java/release-javadoc.sh                |  46 -------
 .../openlineage/client/OpenLineageClient.java |   2 +-
 spec/OpenLineage.json                         |   2 +-
 spec/release.sh                               |  80 ------------
 website/CNAME                                 |   2 +-
 11 files changed, 210 insertions(+), 172 deletions(-)
 delete mode 100644 .circleci/workflows/openlineage-integration-publish.yml
 create mode 100644 .github/workflows/javadoc-deploy.yml
 create mode 100644 .github/workflows/site-deploy.yml
 create mode 100644 .github/workflows/spec-deploy.yml
 delete mode 100755 client/java/release-javadoc.sh
 delete mode 100755 spec/release.sh

diff --git a/.circleci/continue_config.yml b/.circleci/continue_config.yml
index 5c0685c270..cf5fa44060 100644
--- a/.circleci/continue_config.yml
+++ b/.circleci/continue_config.yml
@@ -993,29 +993,6 @@ jobs:
       - run: mkdir -p target/wheels && cp target/wheels/* dist/
       - run: python -m twine upload --non-interactive --verbose --repository pypi dist/*
 
-  publish-javadoc:
-    working_directory: ~/openlineage/client/java
-    docker:
-      - image: cimg/openjdk:11.0
-    steps:
-      - *checkout_project_root
-      - add_ssh_keys:
-          fingerprints:
-            - "7d:bc:78:35:09:c9:25:04:63:f9:eb:4b:f1:f4:d1:91"
-      - run: ./gradlew --console=plain javadoc
-      - run: ./release-javadoc.sh
-
-  publish-spec:
-    working_directory: ~/openlineage
-    docker:
-      - image: cimg/base:2021.07
-    steps:
-      - *checkout_project_root
-      - add_ssh_keys:
-          fingerprints:
-            - "7d:bc:78:35:09:c9:25:04:63:f9:eb:4b:f1:f4:d1:91"
-      - run: spec/release.sh
-
   build-proxy-fluentd:
     working_directory: ~/openlineage/proxy/fluentd
     docker:
diff --git a/.circleci/workflows/openlineage-integration-publish.yml b/.circleci/workflows/openlineage-integration-publish.yml
deleted file mode 100644
index 10ac69c238..0000000000
--- a/.circleci/workflows/openlineage-integration-publish.yml
+++ /dev/null
@@ -1,11 +0,0 @@
-workflows:
-  openlineage-integration-publish:
-    jobs:
-      - publish-spec:
-          filters:
-            branches:
-              only: main
-          context: release
-      - workflow_complete:
-          requires:
-            - publish-spec
diff --git a/.circleci/workflows/openlineage-java.yml b/.circleci/workflows/openlineage-java.yml
index 4ff1484262..750d346c8d 100644
--- a/.circleci/workflows/openlineage-java.yml
+++ b/.circleci/workflows/openlineage-java.yml
@@ -23,12 +23,4 @@ workflows:
           requires:
             - compile-integration-sql-java-linux-arm
             - compile-integration-sql-java-linux-x86
-            - compile-integration-sql-java-macos
-      - publish-javadoc:
-          filters:
-            branches:
-              only: main
-          context: release
-      - workflow_complete:
-          requires:
-            - publish-javadoc
\ No newline at end of file
+            - compile-integration-sql-java-macos
\ No newline at end of file
diff --git a/.github/workflows/javadoc-deploy.yml b/.github/workflows/javadoc-deploy.yml
new file mode 100644
index 0000000000..973ed41601
--- /dev/null
+++ b/.github/workflows/javadoc-deploy.yml
@@ -0,0 +1,51 @@
+name: Build & Deploy javadoc
+
+on:
+  push:
+    tags:
+      - '[0-9]+.[0-9]+.[0-9]+'
+    branches:
+      # testing only, not allowed to run on OpenLineage/OpenLineage repo
+      - test-docs-deploy
+
+jobs:
+  generate_javadoc:
+    if: |
+      (github.ref == 'refs/heads/test-docs-deploy' && github.repository != 'OpenLineage/OpenLineage' ) || 
+      (github.ref != 'refs/heads/test-docs-deploy' && github.repository == 'OpenLineage/OpenLineage')
+    name: "Generate Java docs"
+    env:
+      branch: ${{ (github.head_ref || github.ref_name) == 'test-docs-deploy' && 'test-docs-deploy' || 'main'  }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/setup-java@v4
+        with:
+          distribution: 'corretto'
+          java-version: '11'
+      - uses: actions/checkout@v4
+      - name: Generate new java doc
+        run: |
+          cd client/java
+          ./gradlew --console=plain javadoc
+      - name: Get git branch
+        run: |
+          git config user.name github-actions
+          git config user.email github-actions@github.com
+          git fetch
+          git checkout ${{ env.branch }}
+      - name: Clear existing javadoc
+        run: rm -rf website/static/apidocs/javadoc
+      - name: Copy docs to website directory
+        run: cp -r client/java/build/docs/javadoc website/static/apidocs
+      - name: Commit javadoc
+        run: |
+          git pull
+          git add website/static/apidocs/javadoc/
+          git commit -m "[generated] reloading javadoc"
+          git push
+
+  site-deploy-changes:
+    uses: ./.github/workflows/site-deploy.yml
+    needs: generate_javadoc
+    with:
+      branch: ${{ (github.head_ref || github.ref_name) == 'test-docs-deploy' && 'test-docs-deploy' || 'main'  }}
\ No newline at end of file
diff --git a/.github/workflows/site-deploy.yml b/.github/workflows/site-deploy.yml
new file mode 100644
index 0000000000..1ef7e71449
--- /dev/null
+++ b/.github/workflows/site-deploy.yml
@@ -0,0 +1,37 @@
+name: 'Reusable netlify deploy workflow'
+
+on:
+  workflow_call:
+    inputs:
+      branch:
+        required: true
+        type: string
+
+jobs:
+  site-deploy:
+    name: Deploy to GitHub Pages
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.branch }}
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 22
+          cache: "npm"
+          cache-dependency-path: website/package-lock.json
+      - name: Install dependencies
+        run: yarn install --frozen-lockfile
+      - name: Build openapi docs
+        run: yarn run build:docs
+      - name: Build website
+        run: yarn build
+      - name: Setup Pages
+        uses: actions/configure-pages@v5
+      - name: Upload artifact
+        uses: actions/upload-pages-artifact@v3
+        with:
+          path: './website/build'
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v4
\ No newline at end of file
diff --git a/.github/workflows/spec-deploy.yml b/.github/workflows/spec-deploy.yml
new file mode 100644
index 0000000000..1d7a143798
--- /dev/null
+++ b/.github/workflows/spec-deploy.yml
@@ -0,0 +1,118 @@
+name: Build & Deploy spec to Netlify
+
+on:
+  pull_request:
+    branches:
+      - main
+  push:
+    branches:
+      # testing only, not allowed to run on OpenLineage/OpenLineage repo
+      - test-docs-deploy
+
+jobs:
+  generate_spec:
+    if: |
+      (github.ref == 'refs/heads/test-docs-deploy' && github.repository != 'OpenLineage/OpenLineage' ) || 
+      (github.event.pull_request.merged == true && github.repository == 'OpenLineage/OpenLineage')
+    name: 'Commit Spec definition'
+    runs-on: ubuntu-latest
+    env:
+      branch: ${{ github.head_ref || github.ref_name }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 2
+      - uses: shogo82148/actions-setup-perl@v1
+        with:
+          perl-version: "5.38"
+      - name: Setup NodeJS
+        uses: actions/setup-node@v4
+        with:
+          node-version: 22
+          cache: "npm"
+          cache-dependency-path: website/package-lock.json
+      - name: Check changes
+        id: check-changes
+        run: |
+          if git diff --name-only --exit-code HEAD^ HEAD -- 'spec/*.json' 'spec/OpenLineage.yml' >> /dev/null; then
+            echo "no changes in spec detected, skipping publishing spec"
+            exit 1
+          fi
+          echo "spec changes detected"
+          exit 0
+        continue-on-error: true
+      - name: Check spec changes
+        if: steps.check-changes.outcome == 'success'
+        run: |
+          # Copy changed spec JSON files to target location
+          git diff --name-only HEAD^ HEAD -- 'spec/*.json' | while read LINE; do
+            # ignore registry files
+            if [[ $LINE =~ "registry.json" ]]; then
+              continue
+            fi
+          
+            # extract target file name from $id field in spec files
+            URL=$(cat $LINE | jq -r '.["$id"]')
+          
+            # extract target location in website repo
+            LOC="website/static/${URL#*//*/}"
+            LOC_DIR="${LOC%/*}"
+          
+            # create dir if necessary, and copy files
+            echo "change detected in $LINE"
+            mkdir -p $LOC_DIR
+            cp $LINE $LOC
+          done
+      - name: Generate openapi
+        if: steps.check-changes.outcome == 'success'
+        run: |
+          SPEC_DIR="$(pwd)/website/static/spec"
+          APIDOC_DIR="$(pwd)/website/static/apidocs"
+
+          pushd $SPEC_DIR
+          LATEST_VERSION=$(find . -maxdepth 1 | grep -v 'facets' | grep '[0-9]*-[0-9]-[0-9]' | sort -Vr | head -1)
+          echo latest version is $LATEST_VERSION
+          rm -f ./OpenLineage.json 2>/dev/null
+          perl -i -pe"s/version: [[:alnum:]\.-]*/version: ${LATEST_VERSION:2}/g" ./OpenLineage.yml
+          echo Versions updated
+
+          mkdir "${LATEST_VERSION}/facets"
+          for i in $(ls -d ./facets/* | sort); do cp $i/*.json ${LATEST_VERSION}/facets; done;
+          
+          echo "LATEST_VERSION=${LATEST_VERSION}" >> "$GITHUB_ENV"
+          
+          pushd $LATEST_VERSION
+          ln -sf ../OpenLineage.yml .
+          popd
+      - name: Generate redoc
+        if: steps.check-changes.outcome == 'success'
+        run : |
+          npm install redoc-cli
+          yarn run redoc-cli build -o website/static/apidocs/openapi/index.html website/static/spec/$LATEST_VERSION/OpenLineage.yml --title "OpenLineage API Docs"
+      - name: Clean temp files
+        run: |
+          pushd website/static/spec/$LATEST_VERSION
+          rm -rf facets
+          rm OpenLineage.yml
+          popd
+      - name: Commit changes
+        if: steps.check-changes.outcome == 'success'
+        run: |    
+          # verify if there are any changes
+          if [[ $(git status --porcelain | wc -l) -gt 0 ]]; then
+              git config user.name github-actions
+              git config user.email github-actions@github.com
+              git fetch
+              git checkout ${{ env.branch }}
+              git add website/static/*
+              git commit -m "[generated] adding spec changes"
+              git push
+            else
+              echo "nothing to commit"
+          fi
+
+  site-deploy-changes:
+    uses: ./.github/workflows/site-deploy.yml
+    needs: generate_spec
+    with:
+      branch: ${{ github.head_ref || github.ref_name }} # shall be `main` or `test-docs-deploy` for testing
\ No newline at end of file
diff --git a/client/java/release-javadoc.sh b/client/java/release-javadoc.sh
deleted file mode 100755
index 9bd9f70aca..0000000000
--- a/client/java/release-javadoc.sh
+++ /dev/null
@@ -1,46 +0,0 @@
-#!/usr/bin/env bash
-#
-# Copyright 2018-2024 contributors to the OpenLineage project
-# SPDX-License-Identifier: Apache-2.0
-#
-
-set -e
-
-function git-website() {
-  command git --git-dir "$WEBSITE_DIR/.git" --work-tree "$WEBSITE_DIR" $@
-}
-
-git config --global user.email "openlineage-bot-key@openlineage.io"
-git config --global user.name "OpenLineage deploy bot"
-
-WEBSITE_DIR=${WEBSITE_DIR:-$HOME/build/website}
-REPO="git@github.com:OpenLineage/docs"
-
-if [[ -d $WEBSITE_DIR ]]; then
-  # Check if we're in git repository and the repository points at website
-  if [[ $(git-website rev-parse --is-inside-work-tree) == "true" && $(git-website config --get remote.origin.url) == "$REPO" ]]; then
-    # Make sure we're at the head of the main branch
-    git checkout main
-    git reset --hard origin/master
-  else
-    echo "$WEBSITE_DIR is not empty - failing"
-    exit 1
-  fi
-else
-  git clone --depth 1 $REPO $WEBSITE_DIR
-fi
-
-# check if there are any changes in javadoc in the latest commit
-if [[ $(diff -qr $WEBSITE_DIR/static/apidocs/javadoc './build/docs/javadoc' | wc -l) -eq 0 ]]; then
-  echo "no changes in javadoc detected, skipping publishing javadoc"
-  exit 0
-fi
-
-echo "Changes detected, updating javadoc..."
-rm -rf $WEBSITE_DIR/static/apidocs/javadoc
-mv ./build/docs/javadoc $WEBSITE_DIR/static/apidocs
-
-# commit new spec and push
-git-website add -A static/apidocs/javadoc
-git --git-dir "$WEBSITE_DIR/.git" --work-tree "$WEBSITE_DIR" commit -m "openlineage javadoc update"
-git-website push
diff --git a/client/java/src/main/java/io/openlineage/client/OpenLineageClient.java b/client/java/src/main/java/io/openlineage/client/OpenLineageClient.java
index 6610ebb012..0d61398391 100644
--- a/client/java/src/main/java/io/openlineage/client/OpenLineageClient.java
+++ b/client/java/src/main/java/io/openlineage/client/OpenLineageClient.java
@@ -20,7 +20,7 @@
 import lombok.NonNull;
 import lombok.extern.slf4j.Slf4j;
 
-/** HTTP client used to emit {@link OpenLineage.RunEvent}s to HTTP backend. */
+/** HTTP client used to emit {@link OpenLineage.RunEvent}s to HTTP backend. DUMMY TEST CHANGE */
 @Slf4j
 public final class OpenLineageClient {
   final Transport transport;
diff --git a/spec/OpenLineage.json b/spec/OpenLineage.json
index 58167f7140..dbbeaf1bfe 100644
--- a/spec/OpenLineage.json
+++ b/spec/OpenLineage.json
@@ -1,6 +1,6 @@
 {
   "$schema": "https://json-schema.org/draft/2020-12/schema",
-  "$id": "https://openlineage.io/spec/2-0-2/OpenLineage.json",
+  "$id": "https://openlineage.io/spec/2-0-10/OpenLineage.json",
   "$defs": {
     "BaseEvent": {
       "type": "object",
diff --git a/spec/release.sh b/spec/release.sh
deleted file mode 100755
index 138f0a6ba6..0000000000
--- a/spec/release.sh
+++ /dev/null
@@ -1,80 +0,0 @@
-#!/usr/bin/env bash
-#
-# Copyright 2018-2024 contributors to the OpenLineage project
-# SPDX-License-Identifier: Apache-2.0
-
-set -e
-
-function git-website() {
-  command git --git-dir "$WEBSITE_DIR/.git" --work-tree "$WEBSITE_DIR" $@
-}
-
-git config --global user.email "openlineage-bot-key@openlineage.io"
-git config --global user.name "OpenLineage deploy bot"
-
-WEBSITE_DIR=${WEBSITE_DIR:-$HOME/build/website}
-REPO="git@github.com:OpenLineage/docs"
-
-if [[ -d $WEBSITE_DIR ]]; then
-  # Check if we're in git repository and the repository points at website
-  if [[ $(git-website rev-parse --is-inside-work-tree) == "true" && $(git-website config --get remote.origin.url) == "$REPO" ]]; then
-    # Make sure we're at the head of the main branch
-    git checkout main
-    git reset --hard origin/master
-  else
-    echo "$WEBSITE_DIR is not empty - failing"
-    exit 1
-  fi
-else
-  git clone --depth 1 $REPO $WEBSITE_DIR
-fi
-
-WEBSITE_COMMIT_FILE="$WEBSITE_DIR/.last_spec_commit_id"
-
-# Check on which commit we deployed spec last
-if [[ -f $WEBSITE_COMMIT_FILE ]]; then
-  PREV_SPEC_COMMIT=$(cat "$WEBSITE_COMMIT_FILE")
-else
-  # Before lifecycle state facet
-  PREV_SPEC_COMMIT="d66c41872f3cc7f7cd5c99664d401e070e60ff48"
-fi
-
-# check if there are any changes in spec in the latest commit
-if git diff --name-only --exit-code $PREV_SPEC_COMMIT HEAD 'spec/*.json' 'spec/OpenLineage.yml' >> /dev/null; then
-  echo "no changes in spec detected, skipping publishing spec"
-  exit 0
-fi
-
-echo "Copying spec files from commit $PREV_SPEC_COMMIT"
-
-# Mark last commit on which we finished copying spec
-echo "$CIRCLE_SHA1" > "$WEBSITE_COMMIT_FILE"
-
-# Copy changed spec YML file to target location
-cp spec/OpenLineage.yml ${WEBSITE_DIR}/static/spec/OpenLineage.yml
-
-# Copy changed spec JSON files to target location
-git diff --name-only $PREV_SPEC_COMMIT HEAD 'spec/*.json' | while read LINE; do
-
-  #ignore registry files
-  if [[ $LINE =~ "registry.json" ]]; then
-      continue
-  fi
-
-  # extract target file name from $id field in spec files
-  URL=$(cat $LINE | jq -r '.["$id"]')
-
-  # extract target location in website repo
-  LOC="${WEBSITE_DIR}/static/${URL#*//*/}"
-  LOC_DIR="${LOC%/*}"
-
-  # create dir if necessary, and copy files
-  mkdir -p $LOC_DIR
-  cp $LINE $LOC
-done
-
-# commit new spec and push
-git-website add -A .last_spec_commit_id
-git-website add -A static/spec/
-git --git-dir "$WEBSITE_DIR/.git" --work-tree "$WEBSITE_DIR" commit -m "openlineage specification update"
-git-website push
diff --git a/website/CNAME b/website/CNAME
index 622635b5d7..347a3963ff 100644
--- a/website/CNAME
+++ b/website/CNAME
@@ -1 +1 @@
-openlineage.io
+pawel-big-lebowski.github.io
\ No newline at end of file