diff --git a/.github/actions/delete.sh b/.github/actions/delete.sh deleted file mode 100755 index 48cdba842d..0000000000 --- a/.github/actions/delete.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash - -org="$1" -repo="$2" - -# TODO: Package house keeping -# works: curl -L -H "Accept: application/vnd.github+json" -H "Authorization: Bearer $GITHUB_TOKEN" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/orgs/xtclang/packages?package_type=maven" -# works: gh api -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "/orgs/xtclang/packages?package_type=maven" -# also works: gh api "/orgs/xtclang/packages?package_type=maven" -# Package names. -# gh api "/orgs/xtclang/packages?package_type=maven" | jq -r '.[] | .name' -# README: -# How to use the snapshots. -# Get tag, compare with latest tag, if different then add a new tag. -# Check if packages last version is different from the current version. -# -# SEMVER -# https://github.com/swiftzer/semver/blob/main/.github/workflows/check.yml -# - -echo "Deleting workflow runs for $org/$repo" - -workflows_temp=$(mktemp) # Creates a temporary file to store workflow data. -gh api repos/$org/$repo/actions/workflows | jq -r '.workflows[] | [.id, .path] | @tsv' > $workflows_temp # Lookup workflow -cat "$workflows_temp" -workflows_names=$(awk '{print $2}' $workflows_temp | grep -v "main" | grep -v "master") -echo $workflows_names - -if [ -z "$workflows_names" ]; then - echo "All workflows are either successful or failed. Nothing to remove" -else - echo "Removing all the workflows that are not successful or failed" - for workflow_name in $workflows_names; do - workflow_filename=$(basename "$workflow_name") - echo "Deleting |$workflow_filename|, please wait..." - gh run list --limit 500 --workflow $workflow_filename --json databaseId | - jq -r '.[] | .databaseId' | - xargs -I{} gh run delete {} # Delete all workflow runs for workflow name - done -fi - -rm -rf $workflows_temp - -echo "Done." diff --git a/.github/actions/env.sh b/.github/actions/env.sh deleted file mode 100644 index ad2823b48f..0000000000 --- a/.github/actions/env.sh +++ /dev/null @@ -1 +0,0 @@ -' \ No newline at end of file diff --git a/.github/actions/semver.sh b/.github/actions/semver.sh deleted file mode 100755 index 19ee8e697d..0000000000 --- a/.github/actions/semver.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -function semverParseInto() { - local _re='[^0-9]*\([0-9]*\)[.]\([0-9]*\)[.]\([0-9]*\)\([0-9A-Za-z-]*\)' - eval $2=`echo $1 | sed -e "s#$_re#\1#"` - eval $3=`echo $1 | sed -e "s#$_re#\2#"` - eval $4=`echo $1 | sed -e "s#$_re#\3#"` - eval $5=`echo $1 | sed -e "s#$_re#\4#"` -} - -#if [ "___semver.sh" == "___`basename $0`" ]; then - -MAJOR=0 -MINOR=0 -PATCH=0 -SPECIAL="" - -semverParseInto $1 MAJOR MINOR PATCH SPECIAL -echo "$1 -> M: $MAJOR m:$MINOR p:$PATCH s:$SPECIAL" diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000000..cebc277ab1 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,13 @@ +version: 2 +updates: + - package-ecosystem: "gradle" + directory: "/" + schedule: + interval: "weekly" + rebase-strategy: "disabled" + + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + rebase-strategy: "disabled" diff --git a/.github/workflows/_info.yml b/.github/workflows/_info.yml new file mode 100644 index 0000000000..cb4d630320 --- /dev/null +++ b/.github/workflows/_info.yml @@ -0,0 +1,45 @@ +name: XDK Inspect Workflow + +on: + workflow_call: + workflow_dispatch: + +jobs: + inspect: + name: XDK Inspect Workflow + strategy: + matrix: + os: [ ubuntu-latest, windows-latest, macos-latest ] + runs-on: ${{ matrix.os }} + steps: + - name: Dump GitHub context + env: + GITHUB_CONTEXT: ${{ toJson(github) }} + run: echo "$GITHUB_CONTEXT" + - name: Dump job context + env: + JOB_CONTEXT: ${{ toJson(job) }} + run: echo "$JOB_CONTEXT" + - name: Dump steps context + env: + STEPS_CONTEXT: ${{ toJson(steps) }} + run: echo "$STEPS_CONTEXT" + - name: Dump runner context + env: + RUNNER_CONTEXT: ${{ toJson(runner) }} + run: echo "$RUNNER_CONTEXT" + - name: Dump strategy context + env: + STRATEGY_CONTEXT: ${{ toJson(strategy) }} + run: echo "$STRATEGY_CONTEXT" + - name: Dump matrix context + env: + MATRIX_CONTEXT: ${{ toJson(matrix) }} + run: echo "$MATRIX_CONTEXT" + - name: Show default environment variables + run: | + echo "The job_id is: $GITHUB_JOB" # reference the default environment variables + echo "The id of this action is: $GITHUB_ACTION" # reference the default environment variables + echo "The run id is: $GITHUB_RUN_ID" + echo "The GitHub Actor's username is: $GITHUB_ACTOR" + echo "GitHub SHA: $GITHUB_SHA" diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml new file mode 100644 index 0000000000..32a73a7b1d --- /dev/null +++ b/.github/workflows/check.yml @@ -0,0 +1,390 @@ +# +# GitHub runner workflow for building, verifying and testing the XVM repo. +# +# TODO: Add periodic workflows (that can be manually triggered /w workflow_disspatch and also run through a cron file) +# 1) Cleans out old workflows that have finished but not succeeded. +# 2) Cleans Gradle and other build caches, maybe one every 24 h +# 3) Cleans ... +# + +# Check if "pull-request" makes it possible to add a branch protection status check requirement on master. +on: + push: + workflow_dispatch: + inputs: + extra_gradle_options: + description: 'Extra Gradle options to pass to the build' + required: false + skip_manual_tests: + description: 'Skip manual tests' + required: false + skip_manual_tests_parallel: + description: 'Skip parallel manual tests' + required: false + +env: + ORG_GRADLE_PROJECT_includeBuildManualTests: true + ORG_GRADLE_PROJECT_includeBuildAttachManualTests: true + ORG_XTCLANG_JAVATOOLS_SANITY_CHECK_JAR: true + ORG_XTCLANG_REPO_GITHUB_USER: xtclang-bot + ORG_XTCLANG_REPO_GITHUB_TOKEN: ${{ secrets.ORG_XTCLANG_GITHUB_MAVEN_PACKAGE_REPOSITORY_READ_WRITE }} + #GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + gradle_options: -Dorg.gradle.jvmargs=-Xmx4G -Dorg.gradle.caching.debug=true -Dorg.gradle.vfs.verbose=true --warning-mode=all --console=plain --profile ${{ inputs.extra_gradle_options }} + skip_manual_tests: ${{ github.event.inputs.skip_manual_tests || 'true' }} + skip_manual_tests_parallel: ${{ github.event.inputs.skip_manual_tests_parallel || 'true' }} + # Debug flags + DEBUG_GITHUB_CONTEXT: ${{ toJson(github) }} + ORG_GRADLE_PROJECT_xtcPluginOverrideVerboseLogging: true + GRADLE_BUILD_ACTION_CACHE_DEBUG_ENABLED: true + ACTIONS_RUNNER_DEBUG: true + ACTIONS_STEP_DEBUG: true + +#concurrency: +# group: ${{ github.workflow }}-${{ github.ref }} +# cancel-in-progress: true # Cancel any running workflows if the repo is updated. + +jobs: + # Run Gradle installDist and manual tests for sanity checking. + install-dist: + # The tag action should run independently, + if: ${{ false }} + strategy: + matrix: + os: [ ubuntu-latest ] + #os: [ ubuntu-latest, windows-latest ] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + with: + show-progress: true + + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: 21 + + - uses: gradle/actions/setup-gradle@v3 + with: + cache-read-only: false # TODO: Let this be read only in master. + + - uses: gradle/wrapper-validation-action@v2.1.2 + + - name: List purgeable xtclang.org caches + uses: easimon/wipe-cache@main + with: + dry-run: true + + - name: Gradle installDist + shell: bash + run: | + echo "Executing gradlew installDist in ${pwd}" + ./gradlew ${{ env.gradle_options }} installDist + gradle --status + + - name: Upload build profile (and things like JFR runs, gradle-profiler etc) + uses: actions/upload-artifact@v4 + with: + name: xdk-build-profile + path: build/reports/profile + + - name: Rerun manual tests to verify caching strategy + run: ./gradlew ${{ env.gradle_options }} installDist + + - name: Default manualTest tasks + if: ${{ env.skip_manual_tests != 'true' }} + timeout-minutes: 5 + shell: bash + run: | + ./gradlew ${{ env.gradle_options }} manualTests:runXtc + ./gradlew ${{ env.gradle_options }} manualTests:runOne -PtestName=TestMisc + ./gradlew ${{ env.gradle_options }} manualTests:runTwoTestsInSequence + echo "*** manualTests:runParallel: ${{ env.skip_manual_tests_parallel }}" + + - name: Parallel manualTest task + timeout-minutes: 5 + if: ${{ env.skip_manual_tests_parallel != 'true' }} + run: ./gradlew ${{ env.gradle_options }} manualTests:runParallel + + # + # JOB: 'tag' + # + # Compute the named tag for this snapshot or release publication. + # + # 1) If this is a snapshot release, if the tag already exists, delete it from its current commit + # 2) If this is a non-snapshot release, if the tag already exists, fail the build pipeline. + # 3) Tag the last commit and update + # + # TODO: This only needs to be done one one platform, and actually doesn't work if it's rerun on two + # platforms, but if we don't include windows in its run matrix it will run if it builds clean on + # ubuntu but not on windows. I'm pretty sure this can be done with concurrency groups or something,ยจ + # but right now we have those runner.os checks everywhere on the step level, as they can't be on the + # run level. + # + new-tag: + runs-on: ubuntu-latest + permissions: + contents: write + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - name: Checkout tagged code + uses: actions/checkout@v4 + with: + fetch-tags: true + show-progress: true + + - name: Get XDK version from VERSION file. + run: | + ls -l + full_version=$(cat ./VERSION) + version=${full_version%"-SNAPSHOT"} + if [[ "$full_version" == *SNAPSHOT ]]; then + set +e + git tag -d "${tag_local}" + git push origin --delete "refs/tags/${tag_remote}" + prefix="snapshot/" + fi + + remote_tag="refs/tags/${local_tag}v${version}" + echo "tag_prefix: $tag_prefix" + echo "tag: $tag" + set +e + remote=$(git ls-remote --exit-code --tags origin "ref/tags/${tag}") + tag_exists=$? + echo "remote: $remote" + echo "tag_exists: $tag_exists" + if [ $tag_exists -eq 0 ]; then + commit=$(echo $remote | awk '{print $1}') + echo "Tag '${tag}' exists at remote commit: $commit" + git ls-remote --tags origin "ref/tags/${tag}" + fi + + tag: + if: ${{ false }} + #needs: install-dist + runs-on: ubuntu-latest + permissions: + contents: write + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + outputs: + XDK_TAG_LATEST: ${{ steps.create_tag.outputs.XDK_TAG_LATEST }} + steps: + - name: Checkout tagged code + if: ${{ runner.os == 'Linux' }} + uses: actions/checkout@v4 + with: + fetch-depth: 0 # TODO: Do we need to fetch the entire world. + fetch-tags: true + show-progress: true + + - name: Detect Release or Snapshot + if: ${{ runner.os == 'Linux' }} + id: detect + shell: bash + run: | + echo "TAG: 'git fetch --tags' with remote URL ${{ env.git_remote_url }} ..." + git tag --list && echo "Tags after fetch." + git fetch --tags + git tag --list && echo "Tags after fetch." + XDK_FULL_VERSION=$(cat ./VERSION) + XDK_VERSION=${XDK_FULL_VERSION%"-SNAPSHOT"} + if [[ "$XDK_FULL_VERSION" == *SNAPSHOT ]]; then + XDK_TAG_PREFIX="snapshot/" + fi + xdk_tag=${XDK_TAG_PREFIX}v${XDK_VERSION} + XDK_TAG_EXISTS=false + tags=$(git tag -l | grep ^$xdk_tag) && echo "Matching tag:\n '$tags'" + if [ ! -z "$tags" ]; then + XDK_TAG_EXISTS=true + echo "Tag '${xdk_tag}' exists." + else + echo "Tag '${xdk_tag}' does not exist:\n${tags}" + fi + echo "XDK_FULL_VERSION=$XDK_FULL_VERSION" >> $GITHUB_OUTPUT + echo "XDK_VERSION=$XDK_VERSION" >> $GITHUB_OUTPUT + echo "XDK_TAG_EXISTS=$XDK_TAG_EXISTS" >> $GITHUB_OUTPUT + echo "XDK_TAG_PREFIX=$XDK_TAG_PREFIX" >> $GITHUB_OUTPUT + echo "xdk_tag=$xdk_tag" >> $GITHUB_OUTPUT + echo "XDK_LATEST_COMMIT=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT + + - name: Delete existing snapshot tag + if: ${{ runner.os == 'Linux' && steps.detect.outputs.XDK_TAG_PREFIX == 'snapshot/' && steps.detect.outputs.XDK_TAG_EXISTS == 'true' }} + shell: bash + run: | + xdk_tag=${{ steps.detect.outputs.xdk_tag }} + git tag -d "${xdk_tag}" + #|| echo "Delete tag (if exists): '${xdk_tag}' (local)" + git push origin --delete "${xdk_tag}" || echo "Deleted tag: '${xdk_tag}' (remote)" + echo "*** Tags after delete:"; echo "Local:"; git tag --list; echo "Remote:"; git ls-remote --tags origin + + - name: Verify tag does not exist + if: ${{ runner.os == 'Linux' }} + shell: bash + run: | + xdk_tag=${{ steps.detect.outputs.xdk_tag }} + tags=$(git tag -l | grep ^$xdk_tag) && echo "Matching tag:\n '$tags'" + if [ ! -z "$tags" ]; then + echo "ERROR: Tag '${xdk_tag}' already exists. Illegal build state." + exit 1 + fi + + - name: Create new tag + if: ${{ runner.os == 'Linux' }} + id: create_tag + shell: bash + run: | + xdk_tag=${{ steps.detect.outputs.xdk_tag }} + XDK_LATEST_COMMIT=${{ steps.detect.outputs.XDK_LATEST_COMMIT }} + echo "Create (lightweight) tag: (tag=$xdk_tag, exists=$XDK_TAG_EXISTS, branch=${{ github.ref }}, commit=$XDK_LATEST_COMMIT)" + echo "*** Tags before create:"; echo "Local:"; git tag --list; echo "Remote:"; git ls-remote --tags origin + git tag "${xdk_tag}" || echo "Created tag: '${xdk_tag}' (local)" + git push --tags origin || echo "Created tag: '${xdk_tag}' (remote)" + echo "*** Tags after create:"; echo "Local:"; git tag --list; echo "Remote:"; git ls-remote --tags origin + echo "Verifying tag, and adding it to environment." + git show-ref -d --tags && echo "Tag verified: '${xdk_tag}'" + echo "XDK_TAG_LATEST=$xdk_tag" >> $GITHUB_OUTPUT + + # The publish task expects to always find a tag at the last commit. + # This is a requirement for the publish task. It doesn't matter if it's a snapshot or release tag, they use the + # same gradle action. + publish: + if: ${{ false }} + needs: tag + runs-on: ubuntu-latest + permissions: + contents: write + packages: write + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + ENV_TAG: ${{ needs.tag.outputs.XDK_TAG_LATEST }} + steps: + - uses: actions/checkout@v4 + with: + fetch-tags: true + show-progress: true + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: 21 +# - uses: gradle/actions/setup-gradle@v3 + - name: Verify latest tag + id: verify_latest_tag + shell: bash + run: | + echo "ENV TAG: ${{ env.ENV_TAG }}" + echo "NEEDS TAG: ${{ needs.tag.outputs.XDK_TAG_LATEST }}" + echo "Workspace dir: ${{ github.workspace }}" + find ${{ github.workspace }} + XDK_TAG_LATEST=${{ needs.tag.outputs.XDK_TAG_LATEST }} + if [ -z "$XDK_TAG_LATEST" ]; then + echo "ERROR: Missing tag for publish." + exit 1 + fi + git show-ref -d --tags && echo "Tag verified: '${XDK_TAG_LATEST}'" + echo "Latest tag: '${XDK_TAG_LATEST}" + echo "XDK_TAG_LATEST=$XDK_TAG_LATEST" >> $GITHUB_OUTPUT + - name: Checkout tagged code + uses: actions/checkout@v4 + with: +# fetch-depth: 0 + fetch-tags: true + show-progress: true + - name: Publish packages + run: | + XDK_TAG_LATEST=${{ steps.verify_latest_tag.outputs.XDK_TAG_LATEST }} + echo "Running Gradle publish (tag: $XDK_TAG_LATEST)" + ./gradlew ${{ env.gradle_options }} publishRemote --info --stacktrace + + # Useful git commands: + # + # git tag --list (list local tags) + # git ls-remote --tags origin (list remote tags) + # git tag -d name (delete local tag) + # git push origin --delete name (delete remote tag) + # + # We can also delete locally and propagate like: + # git tag -d name + # git push origin :refs/tags/name + # + # Delete all local tags: + # git tag | xargs git tag -d + # Delete all local tags: + # git tag -l | xargs -n 1 git push --delete origin + # Confirm that all tags have been deleted: + # git tag + # git ls-remote --tags origin + # + # Tags that reappear after deletion: + # * pulling remote, and getting tags back - locally deleted tags will be restored on pull + # + # So this is what we should do. + # + # 1) tag exists + # Inspect it: + # git show name (inspect tag) + # git tags --list (show local tags) + # git ls-remote --tags origin (show remote tags) + # git for-each-ref | grep tag + # git show-ref -d --tags (lightweight tags show up only once, annotated twice and wiht ^{} suffix + # + # Delete it: + # git tag -d name (delete local tag, should exist) + # git push origin --delete name (delete remote tag; may have to handle does not exist) + # + # Create new tag: + # git tag name + # + # 2) tag does not exist. + # git tag + # + # 3) Verify tag: git show name + # For 1), + + + # This is an example of a simple followup action that reuses the Gradle cache from setup gradle. + # TODO: I want to figure out why the xtc modules aren't cached and treated as "without history" in the rerunning scan. + + # test-gradle-reuse: + # runs-on: ubuntu-latest + # needs: build-and-verify + # steps: + # - uses: actions/checkout@v4 + # with: + # show-progress: true + # - uses: actions/setup-java@v4 + # with: + # distribution: temurin + # java-version: 21 + # - uses: gradle/actions/setup-gradle@v3 + # with: + # cache-read-only: false + # - run: find . >/tmp/before_reuse.log + # - run: cat /tmp/before_reuse.log + # - uses: actions/download-artifact@v4 + # with: + # name: installDistOutput + # path: /tmp/artifacts + # - run: | + # echo "**** Artifacts from previous run." + # ls -Rlart /tmp/artifacts + # - uses: actions/upload-artifact@v4 + # with: + # path: /tmp/before_reuse.log + # retention-days: 1 + # overwrite: true + # + # - name: Reuse gradle cache from another job. + # shell: bash + # run: | + # ./gradlew ${{ env.gradle_options }} build --info + + +# - run: find . >/tmp/after_install.log +# - run: cat /tmp/after_install.log +# - uses: actions/upload-artifact@v4 +# with: +# name: installDistOutput +# path: /tmp/after_install.log +# retention-days: 1 +# overwrite: true diff --git a/.github/workflows/install-and-check.yml b/.github/workflows/install-and-check.yml deleted file mode 100644 index 500d7c9592..0000000000 --- a/.github/workflows/install-and-check.yml +++ /dev/null @@ -1,144 +0,0 @@ -# -# GitHub runner workflow for building, verifying and testing the XVM repo. -# - -name: XVM Install and Check - -# Check if "pull-request" makes it possible to add a branch protection status check requirement on master. -on: push - -env: - ORG_GRADLE_PROJECT_xtcPluginOverrideVerboseLogging: true - ORG_GRADLE_PROJECT_includeBuildManualTests: true - ORG_GRADLE_PROJECT_includeBuildAttachManualTests: true - ORG_XTCLANG_JAVATOOLS_SANITY_CHECK_JAR: true - GRADLE_BUILD_ACTION_CACHE_DEBUG_ENABLED: false - gradle_options: --stacktrace --warning-mode all --console verbose - -#concurrency: - #group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - #cancel-in-progress: true - -jobs: - setup: - strategy: - matrix: - os: [ ubuntu-latest, windows-latest ] - runs-on: ${{ matrix.os }} - # Declare outputs for one of the steps in the job. - outputs: - version: ${{ steps.properties.outputs.version }} - major: ${{ steps.properties.outputs.major }} - minor: ${{ steps.properties.outputs.minor }} - patch: ${{ steps.properties.outputs.patch }} - snapshot: ${{ steps.properties.outputs.snapshot }} - tag_new: $${ steps.properties.outputs.tag_new }} - tag_last: $${ steps.properties.outputs.tag_last }} - changelog: ${{ steps.properties.outputs.changelog }} - - steps: - - name: Fetch Sources - uses: actions/checkout@v4 - - - name: Export Properties - id: properties - shell: bash - run: | - function parse_version() { - local _re='[^0-9]*\([0-9]*\)[.]\([0-9]*\)[.]\([0-9]*\)\([0-9A-Za-z-]*\)' - local _match="s#$_re#\\$1#" - echo $xdk_version | sed -e $_match - } - - xdk_version="$(cat ./VERSION)" - if [[ "$xdk_version" == *SNAPSHOT ]]; then - echo "snapshot=true" >> $GITHUB_OUTPUT - fi - xdk_version="${xdk_version%-SNAPSHOT}" - - major="$(parse_version 1)" - minor="$(parse_version 2)" - patch="$(parse_version 3)" - tag_last="$(git describe --tags --abbrev=0)" - - echo "version=$xdk_version" >> $GITHUB_OUTPUT - echo "major=$major" >> $GITHUB_OUTPUT - echo "minor=$minor" >> $GITHUB_OUTPUT - echo "patch=$patch" >> $GITHUB_OUTPUT - echo "tag_new=v$xdk_version" >> $GITHUB_OUTPUT - echo "tag_last=$tag_last" >> $GITHUB_OUTPUT - # TODO: changelog - - build: - name: Build installDist - needs: [ setup ] - strategy: - matrix: - os: [ ubuntu-latest, windows-latest ] - runs-on: ${{ matrix.os }} - steps: - - name: Fetch Sources - uses: actions/checkout@v4 - with: - fetch-tags: true # fetch tags even if fetch-depth == 0 - show-progress: true - - - name: Setup Java and system-wide caching - uses: actions/setup-java@v4 - with: - distribution: zulu - java-version: '21' - cache: 'gradle' - - - name: Setup Gradle and Gradle Cache - uses: gradle/actions/setup-gradle@v3 - with: - gradle-home-cache-cleanup: true - cache-read-only: false - - - name: Gradle Wrapper Validation - uses: gradle/wrapper-validation-action@v2.1.2 - - - name: Check XDK version and tags - run: | - echo "*** XDK_VERSION: ${{ needs.setup.outputs.version }}" - echo "*** XDK_SNAPSHOT: ${{ needs.setup.outputs.snapshot }}" - echo "*** XDK_VERSION_MAJOR: ${{ needs.setup.outputs.major }}" - echo "*** XDK_VERSION_MINOR: ${{ needs.setup.outputs.minor }}" - echo "*** XDK_VERSION_PATCH: ${{ needs.setup.outputs.patch }}" - echo "*** XDK_VERSION_TAG: $$${{ needs.setup.outputs.tag_new }}" - echo "*** XDK_VERSION_TAG_LAST: ${{ needs.setup.outputs.tag_last }}" - - - name: Execute Gradle installDist - run: ./gradlew ${{ env.gradle_options }} installDist - - - name: Execute Gradle manualTest sanity checks - run: | - ./gradlew ${{ env.gradle_options }} manualTests:runXtc - ./gradlew ${{ env.gradle_options }} manualTests:runOne -PtestName=TestMisc - ./gradlew ${{ env.gradle_options }} manualTests:runTwoTestsInSequence - ./gradlew ${{ env.gradle_options }} manualTests:runParallel - - # Always publish a snapshot on push (TODO: change to push to master, not just push) - - name: XDK Publish Snapshot to xtclang.org GitHub Maven Packages. - if: ${{ needs.setup.output.snapshot == 'true' }} - run: echo "*** XDK_VERSION Publishing SNAPSHOT" && ./gradlew publishRemoteSnapshot - env: - ORG_XTCLANG_REPO_GITHUB_USER: xtclang-bot - ORG_XTCLANG_REPO_GITHUB_TOKEN: ${{ secrets.ORG_XTCLANG_GITHUB_MAVEN_PACKAGE_REPOSITORY_READ_WRITE }} - - - name: Check Existing Tag - uses: mukunku/tag-exists-action@v1.4.0 - id: checkTag - with: - tag: ${{ env.needs.setup.outputs.tag_new }} - - # Publish a release if non snapshot, and we don't already have the tag. If we have the tag, we fail. - - name: XDK Publish Release Publication for non-snapshot and tag source - if: ${{ needs.setup.output.snapshot != 'true' }} - run: | - echo "TAG:" ${{ steps.checkTag.outputs.exists }} - if [ ${{ steps.checkTag.outputs.exists }} ]; then - echo "FAILURE. Tag exists." - exit 1 - fi diff --git a/.github/workflows/lint-checkstyle.yml b/.github/workflows/lint-checkstyle.yml index 9086a27b49..9299423885 100644 --- a/.github/workflows/lint-checkstyle.yml +++ b/.github/workflows/lint-checkstyle.yml @@ -2,28 +2,32 @@ # GitHub runner workflow for building, verifying and testing the XVM repo. # -name: XVM Lint +name: XDK Lint # Check if "pull-request" makes it possible to add a branch protection status check requirement on master. -on: push +on: + #push: + #pull_request: [ master ] + workflow_dispatch: + inputs: + lint-enabled: + description: "Enable lint action." + type: boolean + default: false + checkstyle-enabled: + description: "Enable checkstyle action." + type: boolean + default: false env: ORG_GRADLE_PROJECT_xtcPluginOverrideVerboseLogging: true ORG_GRADLE_PROJECT_includeBuildManualTests: true ORG_GRADLE_PROJECT_includeBuildAttachManualTests: true - - ORG_XTCLANG_JAVA_LINT: true - ORG_XTCLANG_JAVA_MAX_WARNINGS: 10000 - ORG_XTCLANG_JAVA_MAX_ERRORS: 10000 - ORG_XTCLANG_JAVA_WARNINGS_AS_ERRORS: true - ORG_XTCLANG_BUILD_CHECKSTYLE: false #TODO - ORG_XTCLANG_BUILD_CHECKSTYLE_FAIL_ON_ERROR: false - gradle_options: --stacktrace --warning-mode all --console verbose jobs: lint: - if: ${{ false }} + if: ${{ false }} # TODO Why does this run even though there is just workflow_dispatch? runs-on: [ ubuntu-latest ] steps: - uses: actions/checkout@v4 @@ -33,7 +37,14 @@ jobs: java-version: '21' - uses: gradle/actions/setup-gradle@v3 with: - cache-disabled: true - name: Ensure code standard and Java linting is correct. + env: + ORG_XTCLANG_JAVA_LINT: false + ORG_XTCLANG_JAVA_MAX_WARNINGS: 10000 + ORG_XTCLANG_JAVA_MAX_ERRORS: 10000 + ORG_XTCLANG_JAVA_WARNINGS_AS_ERRORS: true + ORG_XTCLANG_BUILD_CHECKSTYLE: false #TODO + ORG_XTCLANG_BUILD_CHECKSTYLE_FAIL_ON_ERROR: false run: ./gradlew ${{ env.gradle_options }} build | tee ${{ github.workspace }}/lint-checkstyle-build.log + # TODO Upload artifact of warnings. diff --git a/.github/workflows/manual.yml b/.github/workflows/manual.yml deleted file mode 100644 index 77cab73df6..0000000000 --- a/.github/workflows/manual.yml +++ /dev/null @@ -1,25 +0,0 @@ -# This is a basic workflow that is manually triggered - -name: XVM Manual workflow -on: - workflow_dispatch: - inputs: - name: - # Friendly description to be shown in the UI instead of 'name' - description: 'Person to greet' - # Default value if no value is explicitly provided - default: 'World' - # Input has to be provided for the workflow to run - required: true - # The data type of the input - type: string - -# A workflow run is made up of one or more jobs that can run sequentially or in parallel -jobs: - # This workflow contains a single job called "greet" - greet: - # The type of runner that the job will run on - runs-on: ubuntu-latest - steps: - - name: Send greeting - run: echo "Hello ${{ inputs.name }}" diff --git a/.github/workflows/prune-snapshots.yml b/.github/workflows/prune-snapshots.yml deleted file mode 100644 index 62ec1c0ae2..0000000000 --- a/.github/workflows/prune-snapshots.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: XVM Prune Old Snapshots -on: - push: - branches: - - publish-and-release - schedule: - - cron: '0 0 * * *' # Run every day at midnight -env: - ORG_GRADLE_PROJECT_xtcPluginOverrideVerboseLogging: true - ORG_XTCLANG_REPO_GITHUB_USER: xtclang-bot -jobs: - xdk-prune-snapshots: - if: ${{ false }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Delete old snapshots of XTC Plugin - uses: joshdk/github-packages-delete@v5 - with: - token: ${{ secrets.ORG_XTCLANG_GITHUB_MAVEN_PACKAGE_REPOSITORY_READ_WRITE }} - min-versions-to-keep: 10 - package-type: maven - package-name: org.xtclang.xtc-plugin - package-version: SNAPSHOT - - name: Delete old snapshots of XTC Plugin Metainfo - uses: joshdk/github-packages-delete@v5 - with: - token: ${{ secrets.ORG_XTCLANG_GITHUB_MAVEN_PACKAGE_REPOSITORY_READ_WRITE }} - min-versions-to-keep: 10 - package-type: maven - package-name: org.xtclang.xtc-plugin.org.xtclang.xtc-plugin.gradle.plugin - package-version: SNAPSHOT - - name: Delete old snapshots of XDK - uses: joshdk/github-packages-delete@v5 - with: - token: ${{ secrets.ORG_XTCLANG_GITHUB_MAVEN_PACKAGE_REPOSITORY_READ_WRITE }} - min-versions-to-keep: 10 - package-type: maven - package-name: org.xtclang.xdk - package-version: SNAPSHOT diff --git a/.github/workflows/prune-workflows.yml b/.github/workflows/prune-workflows.yml index 36123b73f3..1a2c6fa773 100644 --- a/.github/workflows/prune-workflows.yml +++ b/.github/workflows/prune-workflows.yml @@ -1,17 +1,18 @@ name: XVM Prune Old Workflows + on: - schedule: [{ cron: "30 1 * * *" }] # Schedule to run the workflow every day at 1:30 AM + #schedule: [{ cron: "30 1 * * *" }] # Schedule to run the workflow every day at 1:30 AM workflow_dispatch: # Allow manual workflow trigger + jobs: clean: - if: ${{ false }} + #if: ${{ false }} name: Delete Expired Workflow Runs runs-on: ubuntu-latest steps: - name: Checkout Action Code - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Run Delete Script - # run: sh ./delete.sh (your org or github name) (your repo name) run: sh ./delete.sh ${ env.github_org }} ${{ env.repo_name }} working-directory: .github/actions env: diff --git a/.github/workflows/publish-release.yml b/.github/workflows/publish-release.yml deleted file mode 100644 index 40f55e11b6..0000000000 --- a/.github/workflows/publish-release.yml +++ /dev/null @@ -1,35 +0,0 @@ -# Always publish a release if it does not exist already. - -# TODO: Github variables, GitHub secrets and Github deployment keys can all be used from here. -# - -name: XVM Publish Release -on: - push: - branches: - - master - - publish-and-release -env: - ORG_GRADLE_PROJECT_xtcPluginOverrideVerboseLogging: true -jobs: - xdk-publish-release: - if: ${{ false }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - uses: actions/setup-java@v4 - with: - distribution: 'temurin' - java-version: '21' - - - uses: gradle/actions/setup-gradle@v3 - - - name: XVM installDist as Publication Input - run: ./gradlew installDist - - - name: XDK Publish Snapshot to xtclang.org GitHub Maven Packages. - run: ./gradlew publishRemoteSnapshot - env: - ORG_XTCLANG_REPO_GITHUB_USER: xtclang-bot - ORG_XTCLANG_REPO_GITHUB_TOKEN: ${{ secrets.ORG_XTCLANG_GITHUB_MAVEN_PACKAGE_REPOSITORY_READ_WRITE }} diff --git a/.github/workflows/purge-cache.yml b/.github/workflows/purge-cache.yml new file mode 100644 index 0000000000..fe9ce14402 --- /dev/null +++ b/.github/workflows/purge-cache.yml @@ -0,0 +1,20 @@ +name: Clear all Github actions caches on sundays +on: + schedule: + - cron: "0 0 * * 0" + workflow_dispatch: + inputs: + dry-run: + description: 'List caches only, do not clear them.' + required: false + default: 'false' + +jobs: + my-job: + name: Delete all caches, org-wide for xtclang.org/GitHub + runs-on: ubuntu-latest + steps: + - name: Purge all xtclang.org caches + uses: easimon/wipe-cache@main + with: + dry-run: ${{ inputs.dry_run }} diff --git a/.gitignore b/.gitignore index b14d33ad50..fd080d0b5a 100644 --- a/.gitignore +++ b/.gitignore @@ -67,6 +67,9 @@ prj/ *.ipr *.iws +# Act .secrets file for local testing of GitHub Actions +.secrets + # Gradle profiler output gradle-user-home/ profile-out/ diff --git a/bin/github-delete-publications.sh b/bin/github-delete-publications.sh new file mode 100755 index 0000000000..d1237a65a9 --- /dev/null +++ b/bin/github-delete-publications.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +gh api --method DELETE "/orgs/xtclang/packages/maven/org.xtclang.xtc-plugin.org.xtclang.xtc-plugin.gradle.plugin" +gh api --method DELETE "/orgs/xtclang/packages/maven/org.xtclang.xtc-plugin" +gh api --method DELETE "/orgs/xtclang/packages/maven/org.xtclang.xdk" + + + diff --git a/bin/github.sh b/bin/github.sh new file mode 100755 index 0000000000..c407e1cb24 --- /dev/null +++ b/bin/github.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +function list_pubs() { + curl -L -H "Accept: application/vnd.github+json" -H "Authorization: Bearer $GITHUB_TOKEN" -H "X-GitHub-Api-Version: 2022-11-28" https://api.github.com/orgs/xtclang/packages?package_type=maven +} + +list_pubs + diff --git a/build-logic/aggregator/src/main/kotlin/org.xtclang.build.aggregator.gradle.kts b/build-logic/aggregator/src/main/kotlin/org.xtclang.build.aggregator.gradle.kts index 090504cd78..107fa35814 100644 --- a/build-logic/aggregator/src/main/kotlin/org.xtclang.build.aggregator.gradle.kts +++ b/build-logic/aggregator/src/main/kotlin/org.xtclang.build.aggregator.gradle.kts @@ -17,6 +17,9 @@ private class XdkBuildAggregator(project: Project) : Runnable { private val prefix = "[${project.name}]" override fun run() { + // We call this for the root project. We should actually assign the root project versions here, accordingly. + logger.info("$prefix Running build aggregate for project ${project.name}.") + gradle.includedBuilds.forEachIndexed { i, includedBuild -> logger.info("$prefix Included build #$i: ${includedBuild.name} [project dir: ${includedBuild.projectDir}]") } @@ -25,7 +28,7 @@ private class XdkBuildAggregator(project: Project) : Runnable { val attachKey = "includeBuildAttach${it.replaceFirstChar(Char::titlecase)}" val attach = (properties[attachKey]?.toString() ?: "true").toBoolean() if (!attach) { - logger.info("$prefix Included build '$it' is explicitly configured to be outside the root lifecycle ($attachKey: $attach).") + logger.info("$prefix Included build '$it' is explicitly configured to be outside the root lifecycle ($attachKey: false).") } !attach }.toSet() diff --git a/build-logic/common-plugins/src/main/kotlin/GitHubPackages.kt b/build-logic/common-plugins/src/main/kotlin/GitHubPackages.kt deleted file mode 100644 index 00d7aae86c..0000000000 --- a/build-logic/common-plugins/src/main/kotlin/GitHubPackages.kt +++ /dev/null @@ -1,149 +0,0 @@ -import XdkPropertiesImpl.Companion.REDACTED -import com.fasterxml.jackson.databind.JsonNode -import io.github.rybalkinsd.kohttp.dsl.context.Method -import io.github.rybalkinsd.kohttp.dsl.context.Method.DELETE -import io.github.rybalkinsd.kohttp.dsl.context.Method.GET -import io.github.rybalkinsd.kohttp.dsl.http -import io.github.rybalkinsd.kohttp.jackson.ext.toJson -import okhttp3.Response -import org.gradle.api.Project - -/** - * Helper class to access GitHub packages for the "xtclang" org, and other build logic - * for publishing XDK build artifacts. - */ -class GitHubPackages(project: Project) : XdkProjectBuildLogic(project) { - companion object Protocol { - private const val GITHUB_HOST = "api.github.com" - private const val GITHUB_SCHEME = "https" - private const val GITHUB_JSON_PACKAGE_NAME = "name" - - private const val GITHUB_PREFIX = "org.xtclang.repo.github" - private const val GITHUB_ORG = "$GITHUB_PREFIX.org" - private const val GITHUB_USER = "$GITHUB_PREFIX.user" - private const val GITHUB_TOKEN = "$GITHUB_PREFIX.token" - private const val GITHUB_URL = "$GITHUB_PREFIX.url" - - private const val GITHUB_URL_DEFAULT_VALUE = "https://maven.pkg.github.com/xtclang" - private const val GITHUB_ORG_DEFAULT_VALUE = "xtclang" - private const val GITHUB_USER_RO_DEFAULT_VALUE = "xtclang-bot" - - val publishTaskPrefixes = listOf("list", "delete") - val publishTaskSuffixesRemote = listOf("AllRemotePublications") - val publishTaskSuffixesLocal = listOf("AllLocalPublications") - - fun restHeaders(token: String): List> = listOf( - "Accept" to "application/vnd.github+json", - "X-GitHub-Api-Version" to "2022-11-28", - "Authorization" to "Bearer $token") - } - - private val org: String - private val packagesUrl: String - private val credentials: Pair - - init { - with(project) { - val user = getXdkProperty(GITHUB_USER, GITHUB_USER_RO_DEFAULT_VALUE) - credentials = user to getXdkProperty(GITHUB_TOKEN, "") - packagesUrl = getXdkProperty(GITHUB_URL, GITHUB_URL_DEFAULT_VALUE) - org = getXdkProperty(GITHUB_ORG, GITHUB_ORG_DEFAULT_VALUE) - } - } - - val uri: String get() = packagesUrl - - val user: String get() = credentials.first - - val token: String get() = credentials.second - - val organization: String get() = this.org - - fun queryXtcLangPackageNames(): List { - return buildList { - val (_, json) = restCall( - GET, - "/orgs/$org/packages", - "package_type" to "maven" - ) - json?.forEach { node -> node[GITHUB_JSON_PACKAGE_NAME]?.asText()?.also { add(it) } } - }.filter { - it.contains(project.group.toString()) && it.contains(project.name) - } - } - - fun queryXtcLangPackageVersions(packageName: String): List { - val (_, json) = restCall(GET, "/orgs/$org/packages/maven/$packageName/versions") - return buildList { - json?.forEach { node -> node[GITHUB_JSON_PACKAGE_NAME]?.asText()?.also { add(it) } } - } - } - - fun deleteXtcLangPackages(): Int { - val packageNames = queryXtcLangPackageNames() - if (packageNames.isEmpty()) { - logger.warn("$prefix No Maven packages found to delete.") - return 0 - } - packageNames.forEach { - deleteXtcLangPackage(it) - } - return packageNames.size - } - - private fun deleteXtcLangPackage(packageName: String) { - logger.lifecycle("$prefix Deleting package: '$packageName'") - restCall(DELETE, "/orgs/$org/packages/maven/$packageName") - } - - fun verifyGitHubConfig(): Boolean { - val (user, token) = credentials - val hasGitHubUser = user.isNotEmpty() - val hasGitHubToken = token.isNotEmpty() - val hasGitHubUrl = uri.isNotEmpty() - val hasGitHubConfig = hasGitHubUser && hasGitHubToken && hasGitHubUrl - if (!hasGitHubConfig) { - logger.warn( - """ - $prefix GitHub credentials are not completely set; publication to GitHub will be disabled. - $prefix '$GITHUB_PREFIX.url' [configured: $hasGitHubUrl ($uri)] - $prefix '$GITHUB_PREFIX.user' [configured: $hasGitHubUser ($user)] - $prefix '$GITHUB_PREFIX.token' [configured: $hasGitHubToken ($REDACTED)] - """.trimIndent() - ) - return false - } - - logger.info("$prefix Checking GitHub repo URL: '$uri'") - if (uri != uri.lowercase()) { - throw project.buildException("The repository URL '$uri' needs to contain all-lowercase owner and repository names.") - } - - logger.info("$prefix GitHub credentials appear to be well-formed. (user: '$user')") - return true - } - - private fun restCall(mtd: Method, httpPath: String, vararg params: Pair): Pair { - return http(method = mtd) { - scheme = GITHUB_SCHEME - host = GITHUB_HOST - path = httpPath - header { - val token = credentials.second - if (token.isEmpty()) { - throw project.buildException("Could not resolve an access token for GitHub from the properties and/or environment.") - } - restHeaders(token).forEach { (k, v) -> k to v } - } - param { - params.forEach { (k, v) -> k to v } - } - }.use { - logger.info("$prefix REST $mtd response status code: ${it.code()}") - if (!it.isSuccessful) { - throw project.buildException("REST $mtd response not successful: $it (code: ${it.code()})") - } - it to runCatching { it.toJson() }.getOrNull() - } - } -} diff --git a/build-logic/common-plugins/src/main/kotlin/XdkBuildLogic.kt b/build-logic/common-plugins/src/main/kotlin/XdkBuildLogic.kt index 35cd277ce6..e4c92696b9 100644 --- a/build-logic/common-plugins/src/main/kotlin/XdkBuildLogic.kt +++ b/build-logic/common-plugins/src/main/kotlin/XdkBuildLogic.kt @@ -6,13 +6,13 @@ import org.gradle.api.file.DirectoryProperty import org.gradle.api.file.ProjectLayout import org.gradle.api.invocation.Gradle import org.gradle.api.logging.LogLevel +import org.gradle.api.logging.LogLevel.INFO import org.gradle.api.logging.LogLevel.LIFECYCLE import java.io.ByteArrayOutputStream import java.io.File import java.nio.file.Path import java.text.SimpleDateFormat -import java.util.Date -import java.util.Locale +import java.util.* abstract class XdkProjectBuildLogic(protected val project: Project) { protected val logger = project.logger @@ -24,10 +24,6 @@ abstract class XdkProjectBuildLogic(protected val project: Project) { } class XdkBuildLogic private constructor(project: Project) : XdkProjectBuildLogic(project) { - private val xdkGitHub: GitHubPackages by lazy { - GitHubPackages(project) - } - private val xdkVersions: XdkVersionHandler by lazy { XdkVersionHandler(project) } @@ -53,10 +49,6 @@ class XdkBuildLogic private constructor(project: Project) : XdkProjectBuildLogic return xdkDistributions } - fun github(): GitHubPackages { - return xdkGitHub - } - fun resolveLocalXdkInstallation(): File { return findLocalXdkInstallation() ?: throw project.buildException("Could not find local installation of XVM.") } @@ -82,11 +74,12 @@ class XdkBuildLogic private constructor(project: Project) : XdkProjectBuildLogic val instance = XdkBuildLogic(project) singletonCache[project] = instance project.logger.info( - """ + """ ${project.prefix} Creating new XdkBuildLogic for project '${project.name}' ${project.prefix} (singletonCache) ${System.identityHashCode(singletonCache)} ${project.prefix} (project -> instance) ${System.identityHashCode(project)} -> ${System.identityHashCode(instance)} - """.trimIndent()) + """.trimIndent() + ) return instance } @@ -142,8 +135,6 @@ val Project.compositeRootBuildDirectory: DirectoryProperty get() = gradle.rootLa val Project.userInitScriptDirectory: File get() = File(gradle.gradleUserHomeDir, "init.d") -val Project.buildRepoDirectory get() = compositeRootBuildDirectory.dir("repo") - val Project.xdkBuildLogic: XdkBuildLogic get() = XdkBuildLogic.instanceFor(this) val Project.prefix: String get() = "[$name]" @@ -159,18 +150,46 @@ val Project.xdkImplicitsPath: String get() = "$compositeRootProjectDirectory/lib val Project.xdkImplicitsFile: File get() = File(xdkImplicitsPath) -fun Project.executeCommand(vararg args: String): String = project.run { - val output = ByteArrayOutputStream() - val result = project.exec { - commandLine(*args) - standardOutput = output - isIgnoreExitValue = false +fun Project.executeCommand(throwOnError: Boolean = false, vararg args: String): Pair = project.run { + logger.lifecycle("$prefix executeCommand: '${args.joinToString(" ")}'") + return executeCommand(LIFECYCLE, emptyMap(), throwOnError, args.toList()) +} + +private fun Project.executeCommand( + logLevel: LogLevel = INFO, + env: Map = emptyMap(), + throwOnError: Boolean = false, + args: List +): Pair = project.run { + + val cmd = args.joinToString(" ") + val executable = args.first() + val cmdPrefix = "$prefix [$executable]" + val cmdOutputPrefix = "$cmdPrefix [output]" + + logger.log(logLevel, "$cmdPrefix executeCommand (throwOnError=$throwOnError, env=$env): '$cmd'") + + val os = ByteArrayOutputStream() + val execResult = exec { + standardOutput = os + isIgnoreExitValue = !throwOnError + environment(env) + commandLine(args) } - if (result.exitValue != 0) { - logger.error("$prefix ERROR: Command '${args.joinToString(" ")}' failed with exit code ${result.exitValue}") - return "" + + val result = execResult.exitValue to os.toString().trim() + val (exitValue, output) = result + if (exitValue != 0) { + logger.error("$prefix ERROR: Command '$cmd' failed with exit code $exitValue") + return result + } + + logger.log(logLevel, "$cmdPrefix Command: '$cmd' executed successfully.") + if (output.isEmpty()) { + logger.log(logLevel, "$cmdOutputPrefix [no output]") } - return output.toString().trim() + output.lines().forEach { logger.log(logLevel, "$cmdOutputPrefix $it") } + return result } // TODO these should probably be lazy for input purposes @@ -197,18 +216,25 @@ fun Project.buildException(msg: String, level: LogLevel = LIFECYCLE): Throwable return GradleException(prefixed) } +fun Project.isRelease(): Boolean { + return !isSnapshot() +} + fun Project.isSnapshot(): Boolean { + val versionOverride = findProperty("XDK_OVERRIDE_VERSION") ?: version + if (versionOverride != version) { + logger.warn("$prefix WARNING: Version for project was overridden: $versionOverride (project version resolved to $version)") + } if (version == Project.DEFAULT_VERSION) { throw buildException("Project was not versioned. Cannot tell whether it's a snapshot or not.") } - return version.toString().endsWith("SNAPSHOT"); + return version.toString().endsWith("SNAPSHOT") } -fun Project.checkSnapshot(snapshot: Boolean): Boolean { - if (snapshot != isSnapshot()) { - throw buildException("Project '${project.name}' is not a snapshot. This operation is only allowed for snapshot builds.") - } - return true +fun Project.checkSnapshot(): Boolean { + val snapshot = project.getXdkPropertyBoolean("org.xtclang.publish.snapshot", true) + val versionMismatch = snapshot != isSnapshot() + return !versionMismatch } /** @@ -247,3 +273,56 @@ fun Task.getXdkPropertyInt(key: String, defaultValue: Int? = null): Int { fun Task.getXdkProperty(key: String, defaultValue: String? = null): String { return registerXdkPropertyInput(this, key, project.getXdkProperty(key, defaultValue)) } + +/* +fun spawn(vararg args: String): String { + System.err.println("SPAWN: " + args.joinToString(" ")) + val process = ProcessBuilder(*args).redirectErrorStream(true).start() + val exitValue = process.waitFor() + val output = process.inputStream.bufferedReader().use { reader -> + reader.readText().trim() + } + //val output = process.inputReader().useLines { lines -> + // buildString { + // lines.forEach { appendLine(it) } + // } + //}.trim() + assert(process.exitValue() == exitValue) { + "Illegal state: spawn exit value: ${process.exitValue()} != waitFor: $exitValue" + } + System.err.println("exitValue: $exitValue") + System.err.println("OUTPUT: $output") + if (exitValue != 0) { + throw IllegalStateException("Failed to execute command: '${args.joinToString(" ")}' (exit value: $exitValue)") + } + return output +}*/ + +/** + * + * // snapshot and failOnError=true should just skip the task + * // All other version mismatches throw an exception + * /* + * val failOnError = getXdkPropertyBoolean("org.xtclang.repo.github.failOnError", true) + * + * logger.warn(""" + * $prefix Version mismatch for project '${project.name}': snapshot=$snapshot, isSnapshot=${isSnapshot()} + * $prefix The project VERSION is ${project.version} + * $prefix Fail on error: $failOnError + * """.trimIndent()) + * + * if (project.isRelease()) { + * // publishSnapshot for a release build should either throw error or skip the task. + * val msg = "Project '${project.name}' is not a snapshot. This operation is only allowed for snapshot builds." + * if (failOnError) { + * logger.warn("$prefix Failing on error, throwing build exception.") + * throw buildException(msg) + * } + * logger.warn("$prefix WARNING: Skipping publication task (reason: '$msg')") + * return false + * } + * + * // project is a snapshot, but we have been told to build a release + * throw buildException("Project '${project.name}' is a snapshot, but we have been told to build a release.") + * } + */ \ No newline at end of file diff --git a/build-logic/common-plugins/src/main/kotlin/XdkDistribution.kt b/build-logic/common-plugins/src/main/kotlin/XdkDistribution.kt index 213f9a87c0..2590286d8d 100644 --- a/build-logic/common-plugins/src/main/kotlin/XdkDistribution.kt +++ b/build-logic/common-plugins/src/main/kotlin/XdkDistribution.kt @@ -3,7 +3,10 @@ import org.gradle.api.Task import org.gradle.api.file.Directory import org.gradle.api.file.RegularFile import org.gradle.api.provider.Provider +import org.gradle.api.publish.maven.MavenPublication import org.gradle.internal.os.OperatingSystem +import org.gradle.kotlin.dsl.assign +import org.gradle.kotlin.dsl.extra class XdkDistribution(project: Project): XdkProjectBuildLogic(project) { companion object { @@ -54,7 +57,10 @@ class XdkDistribution(project: Project): XdkProjectBuildLogic(project) { } val distributionCommit: String get() = buildString { - return project.executeCommand("git", "rev-parse", "HEAD") + val last = project.executeCommand(throwOnError = true, "git", "rev-parse", "HEAD") + logger.lifecycle("$prefix distributionCommit info: $last") + assert(last.first == 0) + return last.second } fun configScriptFilename(installDir: Provider): RegularFile { @@ -98,6 +104,7 @@ class XdkDistribution(project: Project): XdkProjectBuildLogic(project) { } fun shouldCreateWindowsDistribution(): Boolean { + System.err.println("Remove the windows distribution thing.") val runDistExe = project.getXdkPropertyBoolean("org.xtclang.install.distExe", false) if (runDistExe) { logger.info("$prefix 'distExe' task is enabled; will attempt to build Windows installer.") @@ -110,6 +117,33 @@ class XdkDistribution(project: Project): XdkProjectBuildLogic(project) { return false } + fun configureMavenPublications(publication: MavenPublication) = publication.run { + pom { + licenses { + license { + name = "The XDK License" + url = "https://github.com/xtclang/xvm/tree/master/license" + } + } + developers { + developer { + name = "The XTC Language Organization" + email = "info@xtclang.org" + } + } + scm { + connection = "scm:git:git://github.com/xtclang/xvm.git" + developerConnection = "scm:git:ssh://github.com/xtclang/xvm.git" + url = "https://github.com/xtclang/xvm/tree/master" + } + withXml { + val propNode = asNode().appendNode("properties") + propNode.appendNode("gitCommit", project.executeCommand(throwOnError = true, "git", "rev-parse", "HEAD")) + propNode.appendNode("xdkVersion", project.version) + } + } + } + override fun toString(): String { return "$distributionName-$distributionVersion" } diff --git a/build-logic/common-plugins/src/main/kotlin/org.xtclang.build.publish.gradle.kts b/build-logic/common-plugins/src/main/kotlin/org.xtclang.build.publish.gradle.kts index 25ff894c61..a777c8b250 100644 --- a/build-logic/common-plugins/src/main/kotlin/org.xtclang.build.publish.gradle.kts +++ b/build-logic/common-plugins/src/main/kotlin/org.xtclang.build.publish.gradle.kts @@ -1,4 +1,3 @@ -import gradle.kotlin.dsl.accessors._dc4d05473e8224611f7ddf3b32129942.implementation import org.gradle.api.publish.plugins.PublishingPlugin.PUBLISH_TASK_GROUP plugins { @@ -7,119 +6,132 @@ plugins { java } -//dependencies { -// implementation("org.kohsuke:github-api:1.320") -//} - -// Implement logic that checks the latest publication tag for master. -// SNapshots don't need a tag. -// If we push a non snapshot version, we should tag the commit - -/* - * Should we publish the plugin to a common build repository and copy it to any localDist? - */ -private fun shouldPublishPluginToLocalDist(): Boolean { - return project.getXdkPropertyBoolean("org.xtclang.publish.localDist", false).also { - if (it) { - logger.warn("$prefix The plugin publication logic for local distribution is currently not maintained.") - } - } -} +val semanticVersion: SemanticVersion by extra -/** - * Configure repositories for XDK artifact publication. Currently, we publish the XDK zip "xdkArchive", and - * the XTC plugin, "xtcPlugin". - */ publishing { repositories { - logger.info("$prefix Configuring publications for repository mavenLocal().") mavenLocal() - - if (shouldPublishPluginToLocalDist()) { - logger.info("$prefix Configuring publications for repository local flat dir: '$buildRepoDirectory'") - maven { - name = "build" - description = "Publish all publications to the local build directory repository." - url = uri(buildRepoDirectory) - } - } - - logger.info("$prefix Configuring publications for xtclang.org GitHub repository.") - with(xdkBuildLogic.github()) { - if (verifyGitHubConfig()) { - logger.info("$prefix Found GitHub package credentials for XTC (url: $uri, user: $user, org: $organization") - maven { - name = "GitHub" - description = "Publish all publications to the xtclang.org GitHub repository." - url = uri(uri) - credentials { - username = user - password = token - } - } + maven { + name = "GitHub" + url = uri(project.getXdkProperty("org.xtclang.repo.github.url")) + credentials { + username = project.getXdkProperty("org.xtclang.repo.github.user", "xtclang-bot") + password = project.getXdkProperty("org.xtclang.repo.github.token", System.getenv("GITHUB_TOKEN")) } } } publications.withType().configureEach { - pom { - licenses { - license { - name = "The XDK License" - url = "https://github.com/xtclang/xvm/tree/master/license" - } - } - developers { - developer { - name = "The XTC Language Organization" - email = "info@xtclang.org" - } - } - scm { - connection = "scm:git:git://github.com/xtclang/xvm.git" - developerConnection = "scm:git:ssh://github.com/xtclang/xvm.git" - url = "https://github.com/xtclang/xvm/tree/master" - } - withXml { - val propNode = asNode().appendNode("properties") - propNode.appendNode("gitCommit", project.executeCommand("git", "rev-parse", "HEAD")) - propNode.appendNode("xdkVersion", project.version) - } - } + xdkBuildLogic.distro().configureMavenPublications(this) } } -val publishLocal by tasks.registering { - group = PUBLISH_TASK_GROUP - description = "Task that publishes project publications to local repositories (e.g. build and mavenLocal)." - dependsOn(publishAllPublicationsToMavenLocalRepository) +val publishAllPublicationsToMavenLocalRepository by tasks.existing { + doLast { + logger.lifecycle("$prefix Publishing project ${if (project.isSnapshot()) "snapshot" else "release"} artifacts to the mavenLocal repository.") + } } -// TODO: Remove this until we have figured out if and how we want the publication of the XTC Plugin as part of the distribution. -// It may be a good ide to have a local directory that is a maven repository for the plugin, at least until we have the -// full artifact work done. -val pruneBuildRepo by tasks.registering { - group = PUBLISH_TASK_GROUP - description = - "Helper task called internally to make sure the build repo is wiped out before republishing. Used by installPlatformDist and remote publishing only." - if (shouldPublishPluginToLocalDist()) { - logger.lifecycle("$prefix Installing copy of the plugin to local distribution when it exists.") - delete(buildRepoDirectory) +val ensureTag by tasks.registering { + doLast { + fun output(vararg args: String) = project.executeCommand(throwOnError = true, *args).second + + fun execute(throwOnError: Boolean = false, vararg args: String): Pair { + val result = project.executeCommand(throwOnError, *args) + if (result.first != 0) { + logger.error("$prefix Git returned non zero value: $result") + } + return result + } + + fun remoteTag(localTag: String): String { + return "refs/tag/$localTag" + } + + fun deleteTag(throwOnError: Boolean = false, localTag: String): Boolean { + // only if tag exists + val remoteTag = remoteTag(localTag) + //execute(throwOnError = throwOnError, "git", "tag", "-d", localTag) + // 1) Delete the tag on any remote before pushing + execute(throwOnError = throwOnError, "git", "push", "origin", ":${remoteTag(localTag)}") + logger.lifecycle("$prefix Deleted tags: (local: $localTag, remote: $remoteTag)") + return true + } + + fun existingCommits(localTag: String): Pair { + val (localTagValue, localTagCommit) = execute(throwOnError = false, "git", "rev-list", "-n", "1", localTag) + val (remoteTagValue, remoteTagCommitAndName) = execute(throwOnError = false, "git", "ls-remote", "--tags", "origin", remoteTag(localTag)) + val local = if (localTagValue == 0) localTagCommit else "" + val remote = if (remoteTagValue == 0) remoteTagCommitAndName.removeSuffix(remoteTag(localTag)).trim() else "" + return local to remote + } + + val parsedVersion = File(compositeRootProjectDirectory.asFile, "VERSION").readText().trim() + if (parsedVersion != version.toString()) { + throw buildException("$prefix Version mismatch: parsed version '$parsedVersion' does not match project version '$version'") + } + val baseVersion = parsedVersion.removeSuffix("-SNAPSHOT") + val isSnapshot = project.isSnapshot() + + val localTag = buildString { + append(if (isSnapshot) "snapshot/v" else "v") + append(baseVersion) + } + + val remoteTag = remoteTag(localTag) + execute(throwOnError = true, "git", "fetch", "--force", "--tags") // TODO P options? + val localBranchName = output("git", "branch", "--show-current") + val remoteBranchName = "remotes/origin/$localBranchName" + val localLastCommit = output("git", "rev-parse", "HEAD") + val remoteLastCommit = output("git", "ls-remote", "origin", "HEAD").removeSuffix("HEAD").trim() + val (localTagCommit, remoteTagCommit) = existingCommits(localTag) + val hasLocalTag = localTagCommit.isNotEmpty() + val hasRemoteTag = remoteTagCommit.isNotEmpty() + val hasTag = hasLocalTag || hasRemoteTag + + logger.lifecycle(""" + $prefix createTag for version $version (base version: $baseVersion) + $prefix isSnapshot: $isSnapshot + $prefix hasLocalTag: $hasLocalTag, hasRemoteTag: $hasRemoteTag + $prefix Local branch: '$localBranchName' + $prefix last commit: '$localLastCommit' + $prefix tag : '$localTag' + $prefix tag place at local commit: '$localTagCommit' + $prefix Remote branch: '$remoteBranchName' + $prefix last: '$remoteLastCommit' + $prefix tag : '$remoteTag' + $prefix tag placed at remote commit: '$remoteTagCommit' + """.trimIndent()) + + if (hasTag) { + logger.warn("$prefix Tag $localTag already exists ($hasLocalTag, $hasRemoteTag).") + if (!isSnapshot) { + throw buildException("FATAL ERROR: Cannot tag non-snapshot release with an existing tag: $localTag") + } + // delete on any remote before push. + //execute(throwOnError = true, "git", "push", "origin", ":${remoteTag(localTag)}") + } + + if (isSnapshot) { + execute(throwOnError = false, "git", "tag", "-d", localTag) + execute(throwOnError = false, "git", "push", "origin", ":refs/tags/$localTag") + } + execute(throwOnError = true, "git", "tag", localTag, localLastCommit) + execute(throwOnError = true, "git", "push", "origin", "--tags") + // Alternatively git push origin --tags to push all local tag changes or git push origin } } -if (shouldPublishPluginToLocalDist()) { - logger.warn("$prefix Configuring local distribution plugin publication.") - val publishAllPublicationsToBuildRepository by tasks.existing { - dependsOn(pruneBuildRepo) - mustRunAfter(pruneBuildRepo) - } - publishLocal { - dependsOn(publishAllPublicationsToBuildRepository) +val publishAllPublicationsToGitHubRepository by tasks.existing { + dependsOn(ensureTag) + doLast { + logger.lifecycle("$prefix Publishing project ${if (project.isSnapshot()) "snapshot" else "release"} artifacts to the GitHub repository.") } } -val publishAllPublicationsToMavenLocalRepository by tasks.existing +val publishRemote by tasks.registering { + dependsOn(publishAllPublicationsToGitHubRepository) +} val deleteAllLocalPublications by tasks.registering { doLast { @@ -156,22 +168,7 @@ val listAllRemotePublications by tasks.registering { group = PUBLISH_TASK_GROUP description = "Task that lists publications for this project on the 'xtclang' org GitHub package repo." doLast { - val github = xdkBuildLogic.github() - logger.lifecycle("$prefix '$name' Listing remote publications for project '${project.group}:${project.name}':") - val packageNames = github.queryXtcLangPackageNames() - if (packageNames.isEmpty()) { - logger.lifecycle("$prefix No Maven packages found.") - return@doLast - } - packageNames.forEach { pkg -> - logger.lifecycle("$prefix Maven package: '$pkg':") - val versions = github.queryXtcLangPackageVersions(pkg) - if (versions.isEmpty()) { - logger.warn("$prefix WARNING: No versions found for this package. Corrupted package repo?") - return@forEach - } - versions.forEach { logger.lifecycle("$prefix version: '$it'") } - } + TODO("Use gh") // gh api "/orgs/xtclang/packages?package_type=maven" | jq -r '.[] | .name' } } @@ -179,8 +176,9 @@ val deleteAllRemotePublications by tasks.registering { group = PUBLISH_TASK_GROUP description = "Delete all versions of all packages on the 'xtclang' org GitHub package repo. WARNING: ALL VERSIONS ARE PURGED." doLast { - val github = xdkBuildLogic.github() - github.deleteXtcLangPackages() // TODO: Add a pattern that can be set thorugh a property to get finer granularity here than "kill everything!". - logger.lifecycle("$prefix Finished '$name' deleting publications for project: '${project.group}:${project.name}'.") + listOf("org.xtclang.xdk", "org.xtclang.xtc-plugin", "org.xtclang.xtc-plugin.org.xtclang.xtc-plugin.gradle.plugin").forEach { + logger.lifecycle("$prefix Deleting all versions of package '$it' from the GitHub package repository.") + TODO("Use gh - implement me") //spawn("gh", "api", "--method", "DELETE", "/orgs/xtclang/packages/maven/$it") + } } } diff --git a/build-logic/settings-plugins/src/main/kotlin/org.xtclang.build.common.settings.gradle.kts b/build-logic/settings-plugins/src/main/kotlin/org.xtclang.build.common.settings.gradle.kts index 67fbed8b5e..f5fa91eacc 100644 --- a/build-logic/settings-plugins/src/main/kotlin/org.xtclang.build.common.settings.gradle.kts +++ b/build-logic/settings-plugins/src/main/kotlin/org.xtclang.build.common.settings.gradle.kts @@ -21,6 +21,7 @@ val libsVersionCatalog = compositeRootRelativeFile("gradle/libs.versions.toml")! // If we can read properties here, we can also patch the catalog files. dependencyResolutionManagement { + // Hook up mavenCentral, so we can load the third party artifacts declared in libs.versions.toml @Suppress("UnstableApiUsage") repositories { mavenCentral() @@ -36,29 +37,26 @@ dependencyResolutionManagement { } // For bootstrapping reasons, we manually load the properties file, instead of falling back to the build logic automatic property handler. - val xdkVersionInfo = compositeRootRelativeFile("GROUP")!! to compositeRootRelativeFile("VERSION")!! - val pluginDir = xdkVersionInfo.first.parentFile.resolve("plugin") - val xdkPluginVersionInfo = pluginDir.resolve("GROUP").let { if (it.isFile) it else xdkVersionInfo.first } to pluginDir.resolve("VERSION").let { if (it.isFile) it else xdkVersionInfo.second } - val (xdkGroup, xdkVersion) = xdkVersionInfo.toList().map { trimmed(it) } - val (xtcPluginGroup, xtcPluginVersion) = xdkPluginVersionInfo.toList().map { trimmed(it) } + val xdkGroup = System.getenv("XTC_OVERRIDE_GROUP") ?: trimmed(compositeRootRelativeFile("GROUP")!!) + val xdkVersion = System.getenv("XTC_OVERRIDE_VERSION") ?: trimmed(compositeRootRelativeFile("VERSION")!!) val prefix = "[${rootProject.name}]" logger.info("$prefix Configuring and versioning artifact: '$xdkGroup:${rootProject.name}:$xdkVersion'") logger.info( """ $prefix XDK VERSION INFO: $prefix Project : '${rootProject.name}' - $prefix Group : '$xdkGroup' (plugin: '$xtcPluginGroup') - $prefix Version : '$xdkVersion' (plugin: '$xtcPluginVersion') + $prefix Group : '$xdkGroup' + $prefix Version : '$xdkVersion' """.trimIndent() ) versionCatalogs { val libs by creating { - from(files(libsVersionCatalog)) // load versions + from(files(libsVersionCatalog)) version("xdk", xdkVersion) - version("xtc-plugin", xtcPluginVersion) + version("xtc-plugin", xdkVersion) version("group-xdk", xdkGroup) - version("group-xtc-plugin", xtcPluginGroup) + version("group-xtc-plugin", xdkGroup) } } } diff --git a/build-logic/settings-plugins/src/main/kotlin/settings.kt b/build-logic/settings-plugins/src/main/kotlin/settings.kt new file mode 100644 index 0000000000..38e11a4a52 --- /dev/null +++ b/build-logic/settings-plugins/src/main/kotlin/settings.kt @@ -0,0 +1,5 @@ +// TODO How can we execute this code? + +fun testSettingsKt() { + println("TEST SETTINGS KT WORKS!") +} diff --git a/build.gradle.kts b/build.gradle.kts index 0775f479a9..5864457790 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -17,7 +17,6 @@ plugins { // quite a bit, so sadly it would still have to be done at the settings level, though. version = file("VERSION").readText().trim() group = file("GROUP").readText().trim() - // we should probably just marshall a build jreleaser releases tasks here into the // publishable sub projects. //xdkBuildLogic.versions().assignSemanticVersionFromCatalog() @@ -50,29 +49,26 @@ val installDist by tasks.registering { * './gradlew publishToMavenLocal'. Snapshot builds should only be allowed to be published * in local repositories. * - * Publishing tasks can be racy, but it seems that Gradle serializes tasks that have a common - * output directory, which should be the case here. If not, we will have to put back the - * parallel check/task failure condition. + * If the snapshot=true property is set, we will build a snapshot release. + * However, if the VERSION file does not contain a snapshot version, we either fail or skip depending on another property + * + * If the snapshot=false property is set, we will build a normal release. Requires a tag. + * We should never attempt to build an existing release, it will fail on the GitHub side. */ val publishRemote by tasks.registering { group = PUBLISH_TASK_GROUP description = "Publish (aggregate) all artifacts in the XDK to the remote repositories." - onlyIf { - checkSnapshot(false) - } - includedBuildsWithPublications.forEach { - dependsOn(it.task(":publishAllPublicationsToGitHubRepository")) + val versionMatches = checkSnapshot() + if (versionMatches) { + includedBuildsWithPublications.forEach { + dependsOn(it.task(":$name")) + } } -} - -val publishRemoteSnapshot by tasks.registering { - group = PUBLISH_TASK_GROUP - description = "Publish (aggregate) all artifacts in the XDK to the remote snapshot repositories." onlyIf { - checkSnapshot(true) - } - includedBuildsWithPublications.forEach { - dependsOn(it.task(":publishAllPublicationsToGitHubRepository")) + if (!versionMatches) { + logger.warn("$prefix Skipping snapshot publication. VERSION is not a snapshot."); + } + versionMatches } } @@ -83,8 +79,14 @@ val publishRemoteSnapshot by tasks.registering { val publishLocal by tasks.registering { group = PUBLISH_TASK_GROUP description = "Publish (aggregated) all artifacts in the XDK to the local Maven repository." - includedBuildsWithPublications.forEach { - dependsOn(it.task(":$name")) + val versionMatches = checkSnapshot() + if (versionMatches) { + includedBuildsWithPublications.forEach { + dependsOn(it.task(":publishAllPublicationsToMavenLocalRepository")) + } + } + onlyIf { + versionMatches } } @@ -92,14 +94,11 @@ val publish by tasks.registering { group = PUBLISH_TASK_GROUP description = "Task that aggregates publish tasks for builds with publications." dependsOn(publishLocal) - dependsOn(if (isSnapshot()) publishRemoteSnapshot else publishRemote) + dependsOn(publishRemote) } -GitHubPackages.publishTaskPrefixes.forEach { prefix -> - buildList { - addAll(GitHubPackages.publishTaskSuffixesLocal) - addAll(GitHubPackages.publishTaskSuffixesRemote) - }.forEach { suffix -> +listOf("list", "delete").forEach { prefix -> + listOf("AllLocalPublications", "AllRemotePublications").forEach { suffix -> val taskName = "$prefix$suffix" tasks.register(taskName) { group = PUBLISH_TASK_GROUP @@ -110,3 +109,4 @@ GitHubPackages.publishTaskPrefixes.forEach { prefix -> } } } + diff --git a/gradle.properties b/gradle.properties index fef2c0b4c0..d8507f33c2 100644 --- a/gradle.properties +++ b/gradle.properties @@ -4,12 +4,13 @@ org.gradle.parallel=true org.gradle.caching=true +#org.gradle.configureondemand=true # TODO: Experiment with enabling the configuration cache later #org.gradle.configuration-cache=true # JVM and Gradle daemon flags. The default memory usage is 700M for a Daemon, which can be slow in extreme environments. -org.gradle.jvmargs=-Dfile.encoding=UTF-8 +#org.gradle.jvmargs=-Dfile.encoding=UTF-8 # Logging and warning levels # diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 00f3d8255a..c2fc2f4836 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -79,15 +79,15 @@ javatools-utils = { group = "org.xtclang", name = "javatools-utils", version.ref javatools-turtle = { group = "org.xtclang", name = "javatools-turtle", version.ref = "xdk" } javatools-bridge = { group = "org.xtclang", name = "javatools-bridge", version.ref = "xdk" } -semver = { module = "net.swiftzer.semver:semver", version.ref = "semver" } -github-api = { module = "org.kohsuke:github-api", version.ref = "github-api" } -rewrite = { module = "org.openrewrite.recipe:rewrite-static-analysis", version.ref = "rewrite-lib" } kohttp = { module = "io.github.rybalkinsd:kohttp", version.ref = "kohttp" } kohttp-jackson = { module = "io.github.rybalkinsd:kohttp-jackson", version.ref = "kohttp" } javax-activation = { module = "com.sun.activation:javax.activation", version = "1.2.0" } jakarta-xml-bind-api = { module = "jakarta.xml.bind:jakarta.xml.bind-api", version.ref = "jakarta" } jaxb-runtime = { module = "org.glassfish.jaxb:jaxb-runtime", version.ref = "jakarta" } jline = { module = "org.jline:jline", version.ref = "jline" } +#semver = { module = "net.swiftzer.semver:semver", version.ref = "semver" } +#github-api = { module = "org.kohsuke:github-api", version.ref = "github-api" } +#rewrite = { module = "org.openrewrite.recipe:rewrite-static-analysis", version.ref = "rewrite-lib" } [bundles] unicode = ["javax-activation", "jakarta-xml-bind-api", "jaxb-runtime"] diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index e7646dead0..b82aa23a4f 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.7-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.7-bin.zip networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME diff --git a/javatools/build.gradle.kts b/javatools/build.gradle.kts index 6d75c455c2..efcd2acec0 100644 --- a/javatools/build.gradle.kts +++ b/javatools/build.gradle.kts @@ -61,9 +61,16 @@ val copyDependencies by tasks.registering(Copy::class) { include("**/*.jar") } into(layout.buildDirectory.dir("javatools-dependencies")) +} + +val extractDependencies by tasks.registering { + dependsOn(copyDependencies) + val dependencyJars = copyDependencies.map { fileTree(it.destinationDir) } + inputs.files(dependencyJars) doLast { - outputs.files.asFileTree.forEach { - logger.info("$prefix Resolved javatools dependency file: $it") + // Add every extracted jar as an output. + dependencyJars.get().forEach { jarFile -> + logger.lifecycle("$prefix Extracting classes from: ${jarFile.name}") } } } @@ -92,7 +99,19 @@ val jar by tasks.existing(Jar::class) { * TODO: an alternative solution would be to leave the run-time dependent libraries "as is" and * use the "Class-Path" attribute of the manifest to point to them. */ - from(copyDependencies.map { fileTree(it.destinationDir).map { jarFile -> zipTree(jarFile) } }) + logger.warn("$prefix TODO: Figure out why the copy task dependencies map is not lazy and why it's called twice") + from(copyDependencies.map { + val deps = fileTree(it.destinationDir) + logger.lifecycle("Dependencies: ${deps.toList()}") + deps.map { jarFile -> + logger.lifecycle("$prefix (executing: ${state.executing}); adding classes from : ${jarFile.name}") + val tree = zipTree(jarFile) + tree.forEach { zippedClass -> + logger.info("$prefix ${jarFile.name} -> ${zippedClass.name}") + } + tree + } + }) archiveBaseName = "javatools" @@ -136,15 +155,19 @@ val sanityCheckJar by tasks.registering { onlyIf { checkJar } + doLast { + if (!checkJar && System.getenv("CI") == "true") { + throw buildException("$prefix CI runs should execute with org.xtclang.javatools.sanityCheckJar=true.") + } logger.info("$prefix Sanity checking integrity of generated jar file.") val size = DebugBuild.verifyJarFileContents( project, listOf( - "implicit.x", // verify the implicits are in the jar - "org/xvm/tool/Compiler", // verify the javatools package inclusion - "org/xvm/util/Severity", // verify the javatools_utils package inclusion - "org/jline/reader/LineReader" // verify the jline library inclusion + "implicit.x", // verify the implicits are in the jar + "org/xvm/tool/Compiler.class", // verify the javatools package inclusion + "org/xvm/util/Severity.class", // verify the javatools_utils package inclusion + "org/jline/reader/LineReader.class" // verify the jline library inclusion ), expectedEntryCount ) diff --git a/manualTests/build.gradle.kts b/manualTests/build.gradle.kts index b4c4c6822b..6c81043c18 100644 --- a/manualTests/build.gradle.kts +++ b/manualTests/build.gradle.kts @@ -318,6 +318,7 @@ val runOne by tasks.registering(XtcRunTask::class) { val runParallel by tasks.registering(XtcRunTask::class) { group = "application" description = "Run all known tests in parallel through the parallel test runner." + verbose = true // TODO: sometimes hangs during run, and should be attached to / thread dumped with jcmd when that reproduces. Also adding verbose output. module { moduleName = "Runner" verbose = false diff --git a/plugin/src/main/java/org/xtclang/plugin/XtcProjectDelegate.java b/plugin/src/main/java/org/xtclang/plugin/XtcProjectDelegate.java index a67e4874c6..c28b0c0bd9 100644 --- a/plugin/src/main/java/org/xtclang/plugin/XtcProjectDelegate.java +++ b/plugin/src/main/java/org/xtclang/plugin/XtcProjectDelegate.java @@ -132,16 +132,14 @@ private void applyJavaPlugin() { // from doing that isn't 100% compatible with our builds. resolveHiddenTaskNames(tasks).forEach(this::hideAndDisableTask); - if (hasVerboseLogging()) { - final var url = getPluginUrl(); - logger.lifecycle("{} XTC plugin executing from location: '{}' (protocol: '{}')", prefix, url, url.getProtocol()); - if ("file".equals(url.getProtocol())) { - final var file = new File(url.getFile()); - assert file.exists(); - final var lastModified = new SimpleDateFormat("YYYY-MM-dd HH:mm").format(new Date(file.lastModified())); - final var length = file.length(); - logger.lifecycle("{} XTC plugin file; lastModified='{}', length='{}' bytes", prefix, lastModified, length); - } + final var url = getPluginUrl(); + logger.info("{} XTC plugin executing from location: '{}' (protocol: '{}')", prefix, url, url.getProtocol()); + if ("file".equalsIgnoreCase(url.getProtocol())) { + final var file = new File(url.getFile()); + assert file.exists(); + final var lastModified = new SimpleDateFormat("YYYY-MM-dd HH:mm").format(new Date(file.lastModified())); + final var length = file.length(); + logger.info("{} XTC plugin file; lastModified: {}, length: {} bytes", prefix, lastModified, length); } } @@ -456,8 +454,6 @@ private void addJavaToolsContentsAttributes(final Configuration config) { private void createXtcDependencyConfigs(final SourceSet sourceSet) { final var compileTask = createCompileTask(sourceSet); - //final var runAllTask = createRunTask(sourceSet, XtcRunAllTask.class); - logger.info("{} Created compile task for sourceSet '{}' -> '{}'.", prefix, sourceSet.getName(), compileTask.getName()); final var xtcModuleConsumerConfig = incomingXtcModuleDependencies(sourceSet); diff --git a/plugin/src/main/java/org/xtclang/plugin/launchers/JavaExecLauncher.java b/plugin/src/main/java/org/xtclang/plugin/launchers/JavaExecLauncher.java index 1d8a422450..0853af3e18 100644 --- a/plugin/src/main/java/org/xtclang/plugin/launchers/JavaExecLauncher.java +++ b/plugin/src/main/java/org/xtclang/plugin/launchers/JavaExecLauncher.java @@ -41,9 +41,9 @@ public ExecResult apply(final CommandLine cmd) { logger.info("{} {} (launcher: {}); Using '{}' in classpath from: {}", prefix, cmd.getIdentifier(), cmd.getClass(), JAVATOOLS_JAR_NAME, javaToolsJar); + logger.info("{} JavaExec command (launcher {}):", prefix, getClass().getSimpleName()); if (task.hasVerboseLogging()) { final var launchLine = cmd.toString(javaToolsJar); - logger.lifecycle("{} JavaExec command (launcher {}):", prefix, getClass().getSimpleName()); logger.lifecycle("{} {}", prefix, launchLine); } diff --git a/plugin/src/main/java/org/xtclang/plugin/tasks/XtcCompileTask.java b/plugin/src/main/java/org/xtclang/plugin/tasks/XtcCompileTask.java index 6bda05e62e..e54df201b2 100644 --- a/plugin/src/main/java/org/xtclang/plugin/tasks/XtcCompileTask.java +++ b/plugin/src/main/java/org/xtclang/plugin/tasks/XtcCompileTask.java @@ -180,7 +180,7 @@ public Property getDisableWarnings() { } @InputFiles - @PathSensitive(PathSensitivity.ABSOLUTE) + @PathSensitive(PathSensitivity.RELATIVE) Provider getResourceDirectory() { // TODO: This is wrong. The compile task should not be the one depending on resources src, but resources build. // But that is java behavior, so make sure at least we get the resource input dependency. @@ -226,7 +226,7 @@ public void executeTask() { if (hasVerboseLogging()) { logger.lifecycle("{} Stamping XTC module with version: '{}'", prefix, moduleVersion); } - args.add("--set-version", stripSnapshot(moduleVersion)); + args.add("--set-version", semanticVersion(moduleVersion)); } args.addRepeated("-L", resolveFullModulePath()); @@ -241,15 +241,8 @@ public void executeTask() { finalizeOutputs(); } - private String stripSnapshot(final String version) { - if (version.endsWith("-SNAPSHOT")) { - // TODO: Fix this - we don't necessarily want to store the SNAPSHOT name as part of the version - // encoding. It's enough if we can set a bit in the version tree, and have a getter that works - // called "isSnapshot", or something like that. - logger.warn("{} WARNING: XTC module version is a SNAPSHOT. 'xcc' does not fully support semantic versioning. stripping suffix.", prefix()); - return version.substring(0, version.indexOf("-SNAPSHOT")); - } - return version; + private static String semanticVersion(final String version) { + return version.endsWith("-SNAPSHOT") ? version.replace("-SNAPSHOT", "+SNAPSHOT") : version; } private String resolveModuleVersion() { diff --git a/plugin/src/main/java/org/xtclang/plugin/tasks/XtcExtractXdkTask.java b/plugin/src/main/java/org/xtclang/plugin/tasks/XtcExtractXdkTask.java index c29a066d94..dce3e4b5e5 100644 --- a/plugin/src/main/java/org/xtclang/plugin/tasks/XtcExtractXdkTask.java +++ b/plugin/src/main/java/org/xtclang/plugin/tasks/XtcExtractXdkTask.java @@ -40,7 +40,7 @@ private static boolean isXdkArchive(final File file) { } @InputFiles - @PathSensitive(PathSensitivity.ABSOLUTE) + @PathSensitive(PathSensitivity.RELATIVE) FileCollection getInputXdkArchive() { return filesFromConfigs(XDK_CONFIG_NAME_INCOMING_ZIP, XDK_CONFIG_NAME_INCOMING); } diff --git a/plugin/src/main/java/org/xtclang/plugin/tasks/XtcLauncherTask.java b/plugin/src/main/java/org/xtclang/plugin/tasks/XtcLauncherTask.java index 124e213cd6..1d90973ccb 100644 --- a/plugin/src/main/java/org/xtclang/plugin/tasks/XtcLauncherTask.java +++ b/plugin/src/main/java/org/xtclang/plugin/tasks/XtcLauncherTask.java @@ -115,7 +115,7 @@ protected E getExtension() { } @InputFiles - @PathSensitive(PathSensitivity.ABSOLUTE) + @PathSensitive(PathSensitivity.RELATIVE) FileCollection getInputXtcJavaToolsConfig() { return project.files(project.getConfigurations().getByName(XDK_CONFIG_NAME_JAVATOOLS_INCOMING)); } @@ -280,14 +280,14 @@ protected List resolveFullModulePath() { } @InputDirectory - @PathSensitive(PathSensitivity.ABSOLUTE) + @PathSensitive(PathSensitivity.RELATIVE) Provider getInputXdkContents() { return XtcProjectDelegate.getXdkContentsDir(project); } @Optional @InputFiles - @PathSensitive(PathSensitivity.ABSOLUTE) + @PathSensitive(PathSensitivity.RELATIVE) FileCollection getXtcModuleDependencies() { final List sourceSets = getDependentSourceSets(); final List xtcDependencyConfigNames = sourceSets.stream().map(XtcProjectDelegate::incomingXtcModuleDependencies).toList(); diff --git a/plugin/src/main/java/org/xtclang/plugin/tasks/XtcRunTask.java b/plugin/src/main/java/org/xtclang/plugin/tasks/XtcRunTask.java index 22ef0fdc3d..e8dc8cc17b 100644 --- a/plugin/src/main/java/org/xtclang/plugin/tasks/XtcRunTask.java +++ b/plugin/src/main/java/org/xtclang/plugin/tasks/XtcRunTask.java @@ -101,7 +101,7 @@ public final String getJavaLauncherClassName() { // XTC modules needed to resolve module path (the contents of the XDK required to build and run this project) @InputDirectory - @PathSensitive(PathSensitivity.ABSOLUTE) + @PathSensitive(PathSensitivity.RELATIVE) Provider getInputXdkModules() { return XtcProjectDelegate.getXdkContentsDir(project); // Modules in the XDK directory, if one exists. } @@ -109,7 +109,7 @@ Provider getInputXdkModules() { // XTC modules needed to resolve module path (the ones in the output of the project source set, that the compileXtc tasks create) @Optional @InputFiles // should really be enough with an "inputdirectories" but that doesn't exist in gradle. - @PathSensitive(PathSensitivity.ABSOLUTE) + @PathSensitive(PathSensitivity.RELATIVE) FileCollection getInputModulesCompiledByProject() { FileCollection fc = objects.fileCollection(); for (final var sourceSet : getDependentSourceSets()) { diff --git a/plugin/src/main/java/org/xtclang/plugin/tasks/XtcSourceTask.java b/plugin/src/main/java/org/xtclang/plugin/tasks/XtcSourceTask.java index 117485efe8..f5aac9760d 100644 --- a/plugin/src/main/java/org/xtclang/plugin/tasks/XtcSourceTask.java +++ b/plugin/src/main/java/org/xtclang/plugin/tasks/XtcSourceTask.java @@ -73,7 +73,7 @@ protected PatternFilterable getPatternSet() { @InputFiles @SkipWhenEmpty @IgnoreEmptyDirectories - @PathSensitive(PathSensitivity.ABSOLUTE) + @PathSensitive(PathSensitivity.RELATIVE) public FileTree getSource() { return sourceFiles.getAsFileTree().matching(patternSet); } diff --git a/plugin/xtc-plugin.properties b/plugin/xtc-plugin.properties index 05dfb49852..0cd2dd1d2d 100644 --- a/plugin/xtc-plugin.properties +++ b/plugin/xtc-plugin.properties @@ -19,12 +19,9 @@ org.xtclang.plugin.vcs.url=https://github.com/xtclang/xvm # The default is to derive javatools.jar from the xdk or xdkDistribution configurations. However, it is useful to be able to # invoke it from the same thread without JavaExec when debugging crossovers between the build system and the XDK # implementation. -org.xtclang.plugin.bundle.javatools=true +org.xtclang.plugin.bundle.javatools=false # Do we want to Gradle artifact to be built too? The contents are redundant compared to the parent Maven publication, # but the metadata is not, which is required for XTC support in an XTC project outside the XDK build. org.xtclang.plugin.isAutomatedPublishing=true -# Should we install the plugin as part of the installDist, so we can use -# something like XDK_HOME/plugin/repo as a plugin source? -org.xtclang.publish.localDist=false diff --git a/version.json b/version.json deleted file mode 100644 index 39dd04f1ae..0000000000 --- a/version.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "project": "xdk", - "version": "0.4.44-SNAPSHOT", - "group": "org.xtclang" -}, -{ - "project": "plugin", - "version": "0.4.44-SNAPSHOT", - "group": "org.xtclang" -} diff --git a/xdk/build.gradle.kts b/xdk/build.gradle.kts index 5931c4dcbc..4d6cf78c9e 100644 --- a/xdk/build.gradle.kts +++ b/xdk/build.gradle.kts @@ -110,22 +110,6 @@ publishing { } } -private fun shouldPublishPluginToLocalDist(): Boolean { - return project.getXdkPropertyBoolean("org.xtclang.publish.localDist", false) -} - -val publishPluginToLocalDist by tasks.registering { - group = BUILD_GROUP - // TODO: includeBuild dependency; Slightly hacky - use a configuration from the plugin project instead. - if (shouldPublishPluginToLocalDist()) { - dependsOn(gradle.includedBuild("plugin").task(":publishAllPublicationsToBuildRepository")) - outputs.dir(buildRepoDirectory) - doLast { - logger.info("$prefix Published plugin to build repository: ${buildRepoDirectory.get()}") - } - } -} - distributions { // Creates a main distribution (with the name 'xdk' that can be unpacked and used on any platform, // but require the launcher config scripts to run. @@ -264,12 +248,6 @@ private fun Distribution.contentSpec(distName: String, distVersion: String, dist rename("javatools-${project.version}.jar", JAVATOOLS_INSTALLATION_NAME) into("javatools") } - if (shouldPublishPluginToLocalDist()) { - val published = publishPluginToLocalDist.map { it.outputs.files } - from(published) { - into("repo") - } - } from(tasks.xtcVersionFile) if (installLaunchers) { // Do we want to install launchers that work on the host system?