The OSS Review Toolkit (ORT) #352
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: The OSS Review Toolkit (ORT) | |
| on: | |
| pull_request: | |
| workflow_dispatch: | |
| inputs: | |
| branch_name: | |
| description: "The branch to run against the ORT tool" | |
| required: true | |
| default: "main" | |
| jobs: | |
| run-ort: | |
| name: Create attribution files | |
| runs-on: ubuntu-latest | |
| # 1. For workflow_dispatch, always allow | |
| # 2. For pull_request, run if branch is not autogenerated ort-diff-for- branches | |
| if: > | |
| github.event_name == 'workflow_dispatch' || | |
| (github.event_name == 'pull_request' && !startsWith(github.head_ref, 'ort-diff-for-')) | |
| strategy: | |
| fail-fast: false | |
| env: | |
| PYTHON_ATTRIBUTIONS: "python/THIRD_PARTY_LICENSES_PYTHON" | |
| NODE_ATTRIBUTIONS: "node/THIRD_PARTY_LICENSES_NODE" | |
| RUST_ATTRIBUTIONS: "glide-core/THIRD_PARTY_LICENSES_RUST" | |
| JAVA_ATTRIBUTIONS: "java/THIRD_PARTY_LICENSES_JAVA" | |
| INPUT_TARGET_BRANCH: ${{ github.event.inputs.branch_name }} | |
| EVENT_NAME: ${{ github.event_name }} | |
| HEAD_REF: ${{ github.head_ref }} | |
| steps: | |
| - name: Setup target branch and commit | |
| run: | | |
| if [ "$EVENT_NAME" == 'workflow_dispatch' ]; then | |
| echo "TARGET_BRANCH=$INPUT_TARGET_BRANCH" >> $GITHUB_ENV | |
| elif [ "$EVENT_NAME" == 'pull_request' ]; then | |
| echo "TARGET_BRANCH=$HEAD_REF" >> $GITHUB_ENV | |
| fi | |
| - name: Checkout target branch | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ env.TARGET_BRANCH }} | |
| repository: ${{ github.event.pull_request.head.repo.full_name }} | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| fetch-depth: 0 # Fetch all history for all branches and tags | |
| - name: Setup target commit | |
| run: | | |
| echo "TARGET_COMMIT=`git rev-parse HEAD`" >> $GITHUB_ENV | |
| - name: Set up JDK 11 for the ORT package | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: "temurin" | |
| java-version: 11 | |
| - name: Cache ORT and Gradle packages | |
| uses: actions/cache@v4 | |
| id: cache-ort | |
| with: | |
| path: | | |
| ./ort | |
| ~/.gradle/caches | |
| ~/.gradle/wrapper | |
| key: ${{ runner.os }}-ort | |
| - name: Checkout ORT Repository | |
| if: steps.cache-ort.outputs.cache-hit != 'true' | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: "oss-review-toolkit/ort" | |
| path: "./ort" | |
| ref: "26.0.0" | |
| submodules: recursive | |
| - name: Install Rust toolchain | |
| uses: dtolnay/[email protected] | |
| - name: Build and install ORT | |
| if: steps.cache-ort.outputs.cache-hit != 'true' | |
| working-directory: ./ort/ | |
| run: | | |
| export JAVA_OPTS="$JAVA_OPTS -Xmx8g" | |
| ./gradlew installDist | |
| - name: Create ORT config file | |
| run: | | |
| mkdir -p ~/.ort/config | |
| cat << EOF > ~/.ort/config/config.yml | |
| ort: | |
| analyzer: | |
| allowDynamicVersions: true | |
| enabledPackageManagers: [Cargo, NPM, PIP, GradleInspector] | |
| EOF | |
| cat ~/.ort/config/config.yml | |
| ### NodeJS ### | |
| - name: Set up Node.js 16.x | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: 16.x | |
| - name: Create package.json file for the Node wrapper | |
| uses: ./.github/workflows/node-create-package-file | |
| with: | |
| release_version: ${{ env.TARGET_COMMIT }} | |
| os: "ubuntu-latest" | |
| - name: Fix Node base NPM package.json file for ORT | |
| working-directory: ./node/npm/glide | |
| run: | | |
| # Remove the glide-rs dependency to avoid duplication | |
| sed -i '/ "glide-rs":/d' ../../package.json | |
| export pkg_name=valkey-glide-base | |
| export package_version="${{ env.TARGET_COMMIT }}" | |
| export scope=`if [ "$NPM_SCOPE" != '' ]; then echo "$NPM_SCOPE/"; fi` | |
| mv package.json package.json.tmpl | |
| envsubst < package.json.tmpl > "package.json" | |
| cat package.json | |
| - name: Run ORT tools for Node | |
| uses: ./.github/workflows/run-ort-tools | |
| with: | |
| folder_path: "${{ github.workspace }}/node" | |
| ### Python ### | |
| - name: Set up Python 3.10 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.10" | |
| - name: Install python-inspector | |
| working-directory: ./python | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install git+https://github.com/nexB/python-inspector | |
| - name: Run ORT tools for Python | |
| uses: ./.github/workflows/run-ort-tools | |
| with: | |
| folder_path: "${{ github.workspace }}/python" | |
| ### Rust glide-core ### | |
| - name: Run ORT tools for glide-core | |
| uses: ./.github/workflows/run-ort-tools | |
| with: | |
| folder_path: "${{ github.workspace }}/glide-core" | |
| ### Java ### | |
| - name: Set up JDK 11 | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: "temurin" | |
| java-version: 11 | |
| - name: Run ORT tools for Java | |
| uses: ./.github/workflows/run-ort-tools | |
| with: | |
| folder_path: "${{ github.workspace }}/java" | |
| ### Get licenses ### | |
| - name: Retrieve licenses list | |
| working-directory: ./utils | |
| run: | | |
| { | |
| echo 'LICENSES_LIST<<EOF' | |
| python3 get_licenses_from_ort.py | |
| echo EOF | |
| } >> "$GITHUB_ENV" | |
| ### Upload licenses ### | |
| - name: Get current date | |
| id: date | |
| run: | | |
| CURR_DATE=$(date +'%Y-%m-%d-%H') | |
| echo "date=${CURR_DATE}" >> $GITHUB_OUTPUT | |
| - name: Upload the final package list | |
| continue-on-error: true | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: final-package-list-${{ steps.date.outputs.date }} | |
| path: | | |
| utils/final_package_list.txt | |
| retention-days: 30 | |
| - name: Upload the skipped package list | |
| continue-on-error: true | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: skipped-package-list-${{ steps.date.outputs.date }} | |
| path: | | |
| utils/skipped_package_list.txt | |
| retention-days: 30 | |
| - name: Upload the unknown/unapproved package list | |
| continue-on-error: true | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: unapproved-package-list-${{ steps.date.outputs.date }} | |
| path: | | |
| utils/unapproved_package_list.txt | |
| retention-days: 30 | |
| ### TODO: Fail if there are unapproved packages ### | |
| ### Check for attributions diff ### | |
| - name: Check for diff | |
| run: | | |
| cp python/ort_results/NOTICE_DEFAULT $PYTHON_ATTRIBUTIONS | |
| cp node/ort_results/NOTICE_DEFAULT $NODE_ATTRIBUTIONS | |
| cp glide-core/ort_results/NOTICE_DEFAULT $RUST_ATTRIBUTIONS | |
| cp java/ort_results/NOTICE_DEFAULT $JAVA_ATTRIBUTIONS | |
| GIT_DIFF=`git diff $PYTHON_ATTRIBUTIONS $NODE_ATTRIBUTIONS $RUST_ATTRIBUTIONS $JAVA_ATTRIBUTIONS` | |
| if [ -n "$GIT_DIFF" ]; then | |
| echo "FOUND_DIFF=true" >> $GITHUB_ENV | |
| else | |
| echo "FOUND_DIFF=false" >> $GITHUB_ENV | |
| fi | |
| ### Create PR, Note a potential race on the source branch ### | |
| - name: Create pull request | |
| if: ${{ env.FOUND_DIFF == 'true' && github.event_name != 'pull_request' }} | |
| run: | | |
| export ORT_DIFF_BRANCH_NAME="ort-diff-for-$TARGET_BRANCH" | |
| echo "Creating pull request from branch $ORT_DIFF_BRANCH_NAME to branch $TARGET_BRANCH" | |
| git config --global user.email "[email protected]" | |
| git config --global user.name "ort-bot" | |
| git checkout -b ${ORT_DIFF_BRANCH_NAME} | |
| git add $PYTHON_ATTRIBUTIONS $NODE_ATTRIBUTIONS $RUST_ATTRIBUTIONS $JAVA_ATTRIBUTIONS | |
| git commit -m "Updated attribution files" -s | |
| git push --set-upstream origin ${ORT_DIFF_BRANCH_NAME} -f | |
| # Check if PR already exists | |
| existing_pr=$(gh pr list --base ${TARGET_BRANCH} --head ${ORT_DIFF_BRANCH_NAME} --json number --jq '.[0].number') | |
| if [ -z "$existing_pr" ]; then | |
| # Create a new PR if none exists | |
| title="Updated attribution files for commit ${TARGET_COMMIT}" | |
| gh pr create -B ${TARGET_BRANCH} -H ${ORT_DIFF_BRANCH_NAME} --title "${title}" --body "Created by Github action. ${{ env.LICENSES_LIST }}" | |
| echo "Pull request created successfully." | |
| else | |
| # Update the existing PR | |
| echo "Pull request #$existing_pr already exists. Updating branch." | |
| gh pr edit $existing_pr --title "Updated attribution files for commit ${TARGET_COMMIT}" --body "Created by Github action. ${{ env.LICENSES_LIST }}" | |
| echo "Pull request updated successfully." | |
| fi | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| INPUT_VERSION: ${{ github.event.inputs.version }} | |
| ### Warn of outdated attributions for PR ### | |
| - name: Warn of outdated attributions due to the PR | |
| if: ${{ env.FOUND_DIFF == 'true' && github.event_name == 'pull_request' }} | |
| run: | | |
| ATTRIBUTION_FILES=( | |
| "${{ env.PYTHON_ATTRIBUTIONS }}" | |
| "${{ env.NODE_ATTRIBUTIONS }}" | |
| "${{ env.RUST_ATTRIBUTIONS }}" | |
| "${{ env.JAVA_ATTRIBUTIONS }}" | |
| ) | |
| MESSAGE="WARNING! The attribution files differ in this PR. Please ensure an updating PR is issued using a scheduled or manual run of this workflow!" | |
| # Echo the message to the console | |
| echo "$MESSAGE" | |
| # Emit a general warning in the action log | |
| echo "::warning::$MESSAGE" | |
| # Loop through the attribution files | |
| for FILE in "${ATTRIBUTION_FILES[@]}"; do | |
| if git diff --quiet "$FILE"; then | |
| continue | |
| else | |
| # Emit a warning associated with the changed file | |
| echo "::warning file=$FILE::WARNING! The attribution file '$FILE' differs in this PR." | |
| fi | |
| done |