migrate docs to Openlineage/Openlineage repo

Signed-off-by: Pawel Leszczynski <[email protected]>

.circleci/continue_config.yml

@@ -993,29 +993,6 @@ jobs:
       - run: mkdir -p target/wheels && cp target/wheels/* dist/
       - run: python -m twine upload --non-interactive --verbose --repository pypi dist/*
 
-  publish-javadoc:
-    working_directory: ~/openlineage/client/java
-    docker:
-      - image: cimg/openjdk:11.0
-    steps:
-      - *checkout_project_root
-      - add_ssh_keys:
-          fingerprints:
-            - "7d:bc:78:35:09:c9:25:04:63:f9:eb:4b:f1:f4:d1:91"
-      - run: ./gradlew --console=plain javadoc
-      - run: ./release-javadoc.sh
-
-  publish-spec:
-    working_directory: ~/openlineage
-    docker:
-      - image: cimg/base:2021.07
-    steps:
-      - *checkout_project_root
-      - add_ssh_keys:
-          fingerprints:
-            - "7d:bc:78:35:09:c9:25:04:63:f9:eb:4b:f1:f4:d1:91"
-      - run: spec/release.sh
-
   build-proxy-fluentd:
     working_directory: ~/openlineage/proxy/fluentd
     docker:

.circleci/workflows/openlineage-java.yml

@@ -23,12 +23,4 @@ workflows:
           requires:
             - compile-integration-sql-java-linux-arm
             - compile-integration-sql-java-linux-x86
-            - compile-integration-sql-java-macos
-      - publish-javadoc:
-          filters:
-            branches:
-              only: main
-          context: release
-      - workflow_complete:
-          requires:
-            - publish-javadoc
+            - compile-integration-sql-java-macos

.github/workflows/docs-deploy.yml

@@ -0,0 +1,23 @@
+name: Build & Deploy docs to Netlify GitHub Pages
+
+on:
+  push:
+    tags:
+      - '[0-9]+.[0-9]+.[0-9]+'
+
+jobs:
+  generate_java_doc:
+    release-javadoc:
+    # Generates java doc for Java client, add it to the repo under website/static/javadoc
+    # TODO: generate javadoc for java client to be generated
+    #./gradlew --console=plain javadoc
+    # https://github.com/marketplace/actions/update-files-on-github
+  netlify_deploy:
+    name: 'Deploy to Netlify'
+    steps:
+      - uses: actions/checkout@v3
+      - uses: jsmrcaga/[email protected]
+        with:
+          NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_TOKEN_SECRET }}
+          NETLIFY_SITE_ID: ${{ secrets.NETLIFY_SITE_ID }}
+          NETLIFY_DEPLOY_TO_PROD: true # can be false for now

.github/workflows/release_spec.sh

@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+#
+# Copyright 2018-2024 contributors to the OpenLineage project
+# SPDX-License-Identifier: Apache-2.0
+
+set -e
+
+# check if there are any changes in spec in the latest commit
+if git diff --name-only --exit-code HEAD^ HEAD -- 'spec/*.json' 'spec/OpenLineage.yml' >> /dev/null; then
+  echo "no changes in spec detected, skipping publishing spec"
+  exit 0
+fi
+
+# Copy changed spec JSON files to target location
+git diff --name-only HEAD^ HEAD -- 'spec/*.json' | while read LINE; do
+
+  #ignore registry files
+  if [[ $LINE =~ "registry.json" ]]; then
+      continue
+  fi
+
+  # extract target file name from $id field in spec files
+  URL=$(cat $LINE | jq -r '.["$id"]')
+
+  # extract target location in website repo
+  LOC="website/static/${URL#*//*/}"
+  LOC_DIR="${LOC%/*}"
+
+  # create dir if necessary, and copy files
+  echo "change detected in $LINE"
+  mkdir -p $LOC_DIR
+  cp $LINE $LOC
+done
+
+# verify if there are any changes
+if [[ $(git status --porcelain | wc -l) -gt 0 ]]; then
+  git config user.name github-actions
+  git config user.email [email protected]
+  git fetch
+  git checkout main
+  git add website/static/spec/*
+  git commit -m "[generated] adding spec changes"
+  git push
+else
+  echo "nothing to commit"
+fi
+
+# testing weird thing
+sleep 10s

.github/workflows/spec-deploy.yml

@@ -0,0 +1,92 @@
+name: Build & Deploy spec to Netlify
+
+on:
+  push:
+    branches: # should be pull request
+      - migrate-docs
+      #- main # spec is released whenever pushed to main
+
+jobs:
+  generate_spec:
+    # if: github.event.pull_request.merged == true
+    name: 'Commit Spec definition'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Check & Commit spec changes
+        run: |
+          echo ls
+          echo git
+          # check if there are any changes in spec in the latest commit
+          if git diff --name-only --exit-code HEAD^ HEAD -- 'spec/*.json' 'spec/OpenLineage.yml' >> /dev/null; then
+            echo "no changes in spec detected, skipping publishing spec"
+            exit 0
+          fi
+          
+          # Copy changed spec JSON files to target location
+          git diff --name-only HEAD^ HEAD -- 'spec/*.json' | while read LINE; do
+            # ignore registry files
+            if [[ $LINE =~ "registry.json" ]]; then
+              continue
+            fi
+          
+            # extract target file name from $id field in spec files
+            URL=$(cat $LINE | jq -r '.["$id"]')
+          
+            # extract target location in website repo
+            LOC="website/static/${URL#*//*/}"
+            LOC_DIR="${LOC%/*}"
+          
+            # create dir if necessary, and copy files
+            echo "change detected in $LINE"
+            mkdir -p $LOC_DIR
+            cp $LINE $LOC
+          done
+          
+          # verify if there are any changes
+          if [[ $(git status --porcelain | wc -l) -gt 0 ]]; then
+            git config user.name github-actions
+            git config user.email [email protected]
+            git fetch
+            git checkout main
+            git add website/static/spec/*
+            git commit -m "[generated] adding spec changes"
+            git push
+            else
+            echo "nothing to commit"
+          fi
+
+  netlify_deploy:
+    # if: github.event.pull_request.merged == true
+    name: 'Deploy to Netlify'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Repository Checkout
+        uses: actions/checkout@v4
+      - name: Setup NodeJS
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20
+          cache: "npm"
+          cache-dependency-path: website/package-lock.json
+      - name: Install Netlify
+        run: |
+          cd website
+          npm install [email protected] -g
+      - name: Install Dependencies
+        run: |
+          cd website
+          yarn
+      - name: Build project
+        run:  |
+          cd website
+          yarn build
+      - name: Deploy to Netlify
+        id: netlify_deploy
+        run: |
+          cd website/build
+          netlify deploy \
+            --cwd .
+            --site ${{ secrets.NETLIFY_SITE_ID }} \
+            --auth ${{ secrets.NETLIFY_TOKEN_SECRET }}
+            --prod

spec/OpenLineage.json

@@ -1,6 +1,6 @@
 {
   "$schema": "https://json-schema.org/draft/2020-12/schema",
-  "$id": "https://openlineage.io/spec/2-0-2/OpenLineage.json",
+  "$id": "https://openlineage.io/spec/2-0-7/OpenLineage.json",
   "$defs": {
     "BaseEvent": {
       "type": "object",

website/.gitignore

@@ -0,0 +1,26 @@
+# Dependencies
+node_modules
+
+# Production
+/build
+
+# Generated files
+.docusaurus
+.cache-loader
+
+# Misc
+.DS_Store
+.env.local
+.env.development.local
+.env.test.local
+.env.production.local
+
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+
+# intellij
+.idea
+
+argos/screenshots
+argos/test-results

website/CNAME

@@ -0,0 +1 @@
+openlineage.io