CI and deploy steps for docs

Signed-off-by: Pawel Leszczynski <[email protected]>

.circleci/continue_config.yml

@@ -1063,23 +1063,6 @@ jobs:
           paths:
             - ~/.gradle
 
-  # REST
-  publish-javadoc:
-    working_directory: ~/openlineage/client/java
-    docker:
-      - image: cimg/openjdk:17.0
-    steps:
-      - *checkout_project_root
-      - when:
-          condition:
-            equal: [ inactive, << pipeline.parameters.nightly-run >> ]
-          steps:
-            - add_ssh_keys:
-                fingerprints:
-                  - "7d:bc:78:35:09:c9:25:04:63:f9:eb:4b:f1:f4:d1:91"
-            - run: ./gradlew --console=plain javadoc
-            - run: ./release-javadoc.sh
-
   publish-spec:
     working_directory: ~/openlineage
     docker:

.circleci/workflows/openlineage-java.yml

@@ -41,11 +41,6 @@ workflows:
             - compile-integration-sql-java-linux-arm
             - compile-integration-sql-java-linux-x86
             - compile-integration-sql-java-macos
-      - publish-javadoc:
-          filters:
-            branches:
-              only: main
-          context: release
       - release-client-java:
           filters:
             tags:
@@ -70,6 +65,5 @@ workflows:
             tags:
               only: /^[0-9]+(\.[0-9]+){2}(-rc\.[0-9]+)?$/
           requires:
-            - publish-javadoc
             - release-client-java
             - release-integration-sql-java

.circleci/workflows/openlineage-spec.yml

@@ -0,0 +1,15 @@
+workflows:
+  openlineage-spec:
+    jobs:
+      - verify-spec-snapshot:
+          filters:
+            tags:
+              only: /^[0-9]+(\.[0-9]+){2}(-rc\.[0-9]+)?$/
+          context: << pipeline.parameters.build-context >>
+      - workflow_complete:
+          filters:
+            tags:
+              only: /^[0-9]+(\.[0-9]+){2}(-rc\.[0-9]+)?$/
+          requires:
+            - release-client-java
+            - release-integration-sql-java

.github/workflows/website-snapshot.yml

@@ -0,0 +1,89 @@
+name: Build & Deploy spec to Github Pages
+
+on:
+  pull_request:
+    branches:
+      - main
+  push:
+    branches:
+      # testing only, not allowed to run on OpenLineage/OpenLineage repo
+      - main
+      - test-docs-deploy
+      - docs-deploy
+
+permissions:
+  id-token: write
+
+jobs:
+  site-deploy:
+    name: Build and Publish To openlineage-site repo
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          path: 'source'
+      - uses: actions/checkout@v4
+        with:
+          repository: 'pawel-big-lebowski/openlineage-site' # TODO: change this
+          path: 'target'
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 22
+          cache: "npm"
+          cache-dependency-path: source/website/package-lock.json
+      - uses: actions/setup-java@v4
+        with:
+          distribution: 'corretto'
+          java-version: '17'
+      - uses: shogo82148/actions-setup-perl@v1
+        with:
+          perl-version: "5.38"
+      - name: Regenerate Javadoc to website
+        run: |
+          cd source/client/java
+          ./gradlew javadoc
+          rm -rf ../../../target/static/apidocs/javadoc
+          mv ./build/docs/javadoc ../../../target/static/apidocs/
+      - name: Refresh spec
+        run: |
+          rm -rf target/static/spec
+          cp -r source/website/static/spec target/static/spec
+      - name: Install dependencies
+        run:  |
+          cd website
+          yarn install --frozen-lockfile
+      - name: Generate openapi
+        run: |
+          SPEC_DIR="$(pwd)/source/website/static/spec"
+          APIDOC_DIR="$(pwd)/source/website/static/apidocs"
+          pushd $SPEC_DIR
+          LATEST_VERSION=$(find . -maxdepth 1 | grep -v 'facets' | grep '[0-9]*-[0-9]-[0-9]' | sort -Vr | head -1)
+          echo latest version is $LATEST_VERSION
+          rm -f ./OpenLineage.json 2>/dev/null
+          perl -i -pe"s/version: [[:alnum:]\.-]*/version: ${LATEST_VERSION:2}/g" ./OpenLineage.yml
+          echo Versions updated
+          mkdir "${LATEST_VERSION}/facets"
+          for i in $(ls -d ./facets/* | sort); do cp $i/*.json ${LATEST_VERSION}/facets; done;
+
+          echo "LATEST_VERSION=${LATEST_VERSION}" >> "$GITHUB_ENV"
+
+          pushd $LATEST_VERSION
+          ln -sf ../OpenLineage.yml .
+          popd
+      - name: Generate redoc
+        run: |
+          npm install redoc-cli
+          yarn run redoc-cli build -o target/static/apidocs/openapi/index.html source/website/static/spec/$LATEST_VERSION/OpenLineage.yml --title "OpenLineage API Docs"
+      - name: Clean temp files
+        run: |
+          pushd website/static/spec/$LATEST_VERSION
+          rm -rf facets
+          rm OpenLineage.yml
+          popd
+      - name: Push target repo
+        run: |
+          cp -r source/website target
+          cd target
+          git add static
+          git commit -m "refreshing static content from main repo"
+          git push

.pre-commit-config.yaml

@@ -107,3 +107,8 @@ repos:
         files: ^client/java/src/.*\.java$
         exclude: ".*test.*"
         require_serial: true
+      - id: spec-changes
+        name: Verify spec changes committed to website
+        language: script
+        files: ^spec/
+        entry: .pre_commit/run-spec-snapshot.sh

.pre_commit/run-spec-snapshot.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+# Copy changed spec JSON files to target location
+git diff --name-only HEAD -- 'spec/*.json' 'spec/OpenLineage.yml' | while read LINE; do
+  # ignore registry files
+  if [[ $LINE =~ "registry.json" ]]; then
+    continue
+  fi
+
+  # extract target file name from $id field in spec files
+  URL=$(cat $LINE | jq -r '.["$id"]')
+
+  # extract target location in website repo
+  LOC="website/static/${URL#*//*/}"
+  LOC_DIR="${LOC%/*}"
+
+  # create dir if necessary, and copy files
+  echo "change detected in $LINE"
+  mkdir -p $LOC_DIR
+  cp $LINE $LOC
+done

client/java/regenerate-docs-to-website.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+#
+# Copyright 2018-2024 contributors to the OpenLineage project
+# SPDX-License-Identifier: Apache-2.0
+#
+
+cd "$(dirname "$0")"
+./gradlew javadoc
+rm -rf ../../website/static/apidocs/javadoc
+mv ./build/docs/javadoc ../../website/static/apidocs/

website/CNAME

@@ -1 +1 @@
-openlineage.io
+openlineage.github.io

website/docusaurus.config.js

@@ -26,8 +26,8 @@ const linksSocial = [
 const config = {
   title: 'OpenLineage',
   tagline: 'OpenLineage',
-  url: 'https://openlineage.io',
-  baseUrl: '/',
+  url: 'https://openlineage.github.io', // TODO: replace with openlineage.io in the next step
+  baseUrl: '/OpenLineage',
   onBrokenLinks: 'throw',
   onBrokenMarkdownLinks: 'throw',
   favicon: 'img/favicon.ico',