diff --git a/.dockerignore b/.dockerignore index 53414716..6a76809f 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,4 +1,4 @@ .* venv poetry.lock - +build diff --git a/.github/workflows/_deb_build.yml b/.github/workflows/_deb_build.yml index a64df594..b13eb503 100644 --- a/.github/workflows/_deb_build.yml +++ b/.github/workflows/_deb_build.yml @@ -30,19 +30,18 @@ concurrency: permissions: packages: read -env: - DEBUG: y - VERBOSITY: debug - jobs: config: runs-on: ubuntu-latest outputs: - BUILDER_TAG: ${{ steps.config.outputs.BUILDER_TAG }} - REF_TAG: ${{ steps.config.outputs.REF_TAG }} + DEB_ARTIFACT: ${{ steps.config.outputs.DEB_ARTIFACT }} + DEB_BUILDER_TAG: ${{ steps.config.outputs.DEB_BUILDER_TAG }} RUNNER: ${{ steps.config.outputs.RUNNER }} - TEST_IMAGE: ${{ steps.config.outputs.TEST_IMAGE }} TEST_ARTIFACT: ${{ steps.config.outputs.TEST_ARTIFACT }} + TEST_DATE: ${{ steps.config.outputs.TEST_DATE }} + TEST_ID: ${{ steps.config.outputs.TEST_ID }} + TEST_IMAGE: ${{ steps.config.outputs.TEST_IMAGE }} + TEST_RUNNER_ARTIFACT: ${{ steps.config.outputs.TEST_RUNNER_ARTIFACT }} steps: - name: Clone uno uses: actions/checkout@v4 @@ -61,16 +60,31 @@ jobs: runner='["self-hosted", "linux", "arm64", "rpi5"]' ;; esac - builder_tag=$(echo ${{inputs.base-tag}} | tr : - | tr / -) - ref_tag=$(echo ${{ github.ref_name }} | tr / -) - test_id=$(date +%Y%m%d-%H%M%S) - test_artifact=uno-test-deb-${builder_tag}-${ref_tag}-${{inputs.platform}}__${test_id} + case "${{github.ref_type}}" in + tag) + image_version=${{github.ref_name}} + ;; + *) + sha_short=$(cd src/uno && git rev-parse --short HEAD) + image_version=${{github.ref_name}}@${sha_short} + ;; + esac + image_version="$(echo ${image_version} | tr / -)" + deb_builder_tag=$(echo ${{inputs.base-tag}} | tr : - | tr / -) + test_date=$(date +%Y%m%d-%H%M%S) + test_id=deb-${deb_builder_tag}-${{ inputs.platform }}__${image_version} + test_artifact=uno-test-${test_id}__${test_date} + test_runner_artifact=uno-runner-${deb_builder_tag}-${{ inputs.platform }}__${image_version}__${test_date} + deb_artifact=uno-deb-${deb_builder_tag}-${{ inputs.platform }}__${image_version}__${test_date} ( echo RUNNER=${runner} - echo BUILDER_TAG=${builder_tag} - echo REF_TAG=${ref_tag} - echo TEST_IMAGE=mentalsmash/uno-test-runner:latest + echo DEB_BUILDER_TAG=${deb_builder_tag} + echo DEB_ARTIFACT=${deb_artifact} echo TEST_ARTIFACT=${test_artifact} + echo TEST_DATE=${test_date} + echo TEST_ID=${test_id} + echo TEST_IMAGE=mentalsmash/uno-test-runner:latest + echo TEST_RUNNER_ARTIFACT=${test_runner_artifact} ) >> ${GITHUB_OUTPUT} build-packages: @@ -83,14 +97,6 @@ jobs: path: src/uno submodules: true - - name: Clone support files - uses: actions/checkout@v4 - with: - repository: mentalsmash/uno-ci - token: ${{ secrets.UNO_CI_PAT }} - ref: master - path: src/uno-ci - - name: Log in to GitHub uses: docker/login-action@v3 with: @@ -103,23 +109,75 @@ jobs: make -C src/uno changelog make -C src/uno debuild env: - DEB_BUILDER: ghcr.io/mentalsmash/uno-ci-debian-builder:${{ needs.config.outputs.BUILDER_TAG }} + DEB_BUILDER: ghcr.io/mentalsmash/uno-ci-debian-builder:${{ needs.config.outputs.DEB_BUILDER_TAG }} - name: Upload debian packages uses: actions/upload-artifact@v4 with: - name: uno-${{inputs.platform}}-${{needs.config.outputs.BUILDER_TAG}} + name: ${{ needs.config.outputs.DEB_ARTIFACT }} path: src/uno/debian-dist/* - if: always() + + - name: Upload test runner + uses: actions/upload-artifact@v4 + with: + name: ${{ needs.config.outputs.TEST_RUNNER_ARTIFACT }} + path: src/uno/dist/bundle/default/runner - name: Fix permissions run: | docker run --rm \ -v $(pwd)/src/uno:/uno \ - ghcr.io/mentalsmash/uno-ci-debian-builder:${{ needs.config.outputs.BUILDER_TAG }} \ + ghcr.io/mentalsmash/uno-ci-debian-builder:${{ needs.config.outputs.DEB_BUILDER_TAG }} \ chown -R $(id -u):$(id -g) /uno if: always() + test-packages: + needs: + - config + - build-packages + runs-on: ${{ fromJson(needs.config.outputs.RUNNER) }} + env: + DEB_TESTER: ${{ needs.config.outputs.TEST_IMAGE }} + FIX_DIR: ${{ github.workspace }} + RTI_LICENSE_FILE: ${{ github.workspace }}/src/uno-ci/resource/rti/rti_license.dat + TEST_DATE: ${{ needs.config.outputs.TEST_DATE }} + TEST_ID: ${{ needs.config.outputs.TEST_ID }} + TEST_IMAGE: ${{ needs.config.outputs.TEST_IMAGE }} + steps: + - name: Clone uno + uses: actions/checkout@v4 + with: + path: src/uno + submodules: true + + - name: Clone support files + uses: actions/checkout@v4 + with: + repository: mentalsmash/uno-ci + token: ${{ secrets.UNO_CI_PAT }} + ref: master + path: src/uno-ci + + - name: Download runner artifact + uses: actions/download-artifact@v4 + with: + pattern: ${{ needs.config.outputs.TEST_RUNNER_ARTIFACT }} + + - name: Download debian packages artifact + uses: actions/download-artifact@v4 + with: + pattern: ${{ needs.config.outputs.DEB_ARTIFACT }} + + - name: Move artifacts in place + run: | + mkdir -p src/uno/dist/bundle/default/runner + mv -v ${{ needs.config.outputs.TEST_RUNNER_ARTIFACT }}/* \ + src/uno/dist/bundle/default/runner/ + + mkdir -p src/uno/debian-dist + mv -v ${{ needs.config.outputs.DEB_ARTIFACT }}/* \ + src/uno/debian-dist/ + - name: Build tester image uses: docker/build-push-action@v5 with: @@ -130,38 +188,22 @@ jobs: build-args: | BASE_IMAGE=${{ inputs.base-tag }} - - name: Setup integration tests - run: | - python3 -m venv venv - . venv/bin/activate - pip3 install -U pip setuptools - pip3 install -U -e src/uno - cp src/uno-ci/resource/rti/rti_license.dat rti_license.dat - - name: Run integration tests run: | - . venv/bin/activate - pytest -s -v --junit-xml=test-results/uno-test-results-integration-${{ needs.config.outputs.REF_TAG }}.xml \ - src/uno/test/integration + make -C src/uno debtest env: - DEV: y - RTI_LICENSE_FILE: ${{ github.workspace }}/rti_license.dat - TEST_IMAGE: ${{ needs.config.outputs.TEST_IMAGE }} - TEST_RUNNER: runner + DEBUG: ${{ runner.debug }} - name: Restore permissions changed by integration tests if: always() run: | - docker run --rm \ - -v $(pwd):/workspace \ - ${{ needs.config.outputs.TEST_IMAGE }} \ - fix-root-permissions $(id -u):$(id -g) /workspace + make -C src/uno fix-file-ownership # Always collect and upload available test results - name: Upload test results uses: actions/upload-artifact@v4 with: name: ${{ needs.config.outputs.TEST_ARTIFACT }} - path: test-results/* + path: src/uno/test-results/* if: always() diff --git a/.github/workflows/_install_test.yml b/.github/workflows/_install_test.yml index 55c73359..bced57e8 100644 --- a/.github/workflows/_install_test.yml +++ b/.github/workflows/_install_test.yml @@ -110,16 +110,16 @@ jobs: src/uno/test/install - name: Restore permissions changed by tests - if: ${{ always() }} + if: always() run: | docker run --rm \ -v $(pwd):/workspace \ ${{inputs.tag}} \ - fix-root-permissions $(id -u):$(id -g) /workspace + fix-file-ownership $(id -u):$(id -g) /workspace - name: Upload test results uses: actions/upload-artifact@v4 - if: ${{ always() }} + if: always() with: name: ${{ needs.test-config.outputs.TEST_ARTIFACT }} path: test-results/* diff --git a/.github/workflows/_release_badges.yml b/.github/workflows/_release_badges.yml index d83adf85..88bd4f8a 100644 --- a/.github/workflows/_release_badges.yml +++ b/.github/workflows/_release_badges.yml @@ -34,19 +34,19 @@ jobs: version=${sha_short} color_version=orange tag=nightly - badge_default_version=29b57b0427def87cc3ef4ab81c956c29 - badge_default_base=2d53344e1ccfae961665e08432f18113 - badge_static_version=d73e338805c7d2c348a2d7149a66f66c - badge_static_base=373e55438055b1222c9937797c949f9b + badge_default_version=e7aab205f782cc0c6f394a2fece90509 + badge_default_base=8f31c46dcfd0543b42f356e5b1c6c2c8 + badge_static_version=b310f08c34f051846877aeb59b0be311 + badge_static_base=b0e38a84eb8679d5212e162fbb616773 ;; tag) version=${{github.ref_name}} color_version=green tag=latest - badge_default_version=e7aab205f782cc0c6f394a2fece90509 - badge_default_base=8f31c46dcfd0543b42f356e5b1c6c2c8 - badge_static_version=b310f08c34f051846877aeb59b0be311 - badge_static_base=b0e38a84eb8679d5212e162fbb616773 + badge_default_version=29b57b0427def87cc3ef4ab81c956c29 + badge_default_base=2d53344e1ccfae961665e08432f18113 + badge_static_version=d73e338805c7d2c348a2d7149a66f66c + badge_static_base=373e55438055b1222c9937797c949f9b ;; esac ( diff --git a/.github/workflows/_release_build.yml b/.github/workflows/_release_build.yml index 0abe39fb..ec22c329 100644 --- a/.github/workflows/_release_build.yml +++ b/.github/workflows/_release_build.yml @@ -31,24 +31,9 @@ jobs: path: src/uno submodules: true - - name: Bootstrap dev dependencies with poetry - run: | - python3 -m venv poetry-venv - . poetry-venv/bin/activate - pip install -U poetry - deactivate - cd src/uno - ${GITHUB_WORKSPACE}/poetry-venv/bin/poetry install --with=dev - - name: Validate code run: | - . src/uno/.venv/bin/activate - ruff check - - - name: Validate code format - run: | - . src/uno/.venv/bin/activate - ruff format --check + make -C src/uno code-check - name: Set up QEMU uses: docker/setup-qemu-action@v3 diff --git a/.github/workflows/_release_test.yml b/.github/workflows/_release_test.yml index d607ed96..cf979910 100644 --- a/.github/workflows/_release_test.yml +++ b/.github/workflows/_release_test.yml @@ -19,15 +19,12 @@ concurrency: group: release-test-image-${{ github.ref }}-${{ inputs.tag}}-${{ inputs.platform }} cancel-in-progress: true -env: - DEBUG: '' - VERBOSITY: '' - jobs: - test_config: + config: runs-on: ubuntu-latest outputs: TEST_ARTIFACT: ${{ steps.config.outputs.TEST_ARTIFACT }} + TEST_DATE: ${{ steps.config.outputs.TEST_DATE }} TEST_ID: ${{ steps.config.outputs.TEST_ID }} TEST_IMAGE: ${{ steps.config.outputs.TEST_IMAGE }} RUNNER: ${{ fromJson(steps.config.outputs.RUNNER) }} @@ -62,8 +59,10 @@ jobs: build_version=${{github.ref_name}} ;; esac - test_id=$(date +%Y%m%d-%H%M%S) - test_artifact=uno-test-${{inputs.flavor}}-${{ inputs.platform }}-${build_label}__${build_version}__${test_id} + + test_date=$(date +%Y%m%d-%H%M%S) + test_id=release-${{inputs.flavor}}-${{inputs.platform}}-${build_label}__${build_version} + test_artifact=uno-test-${test_id}__${test_id} test_image=${{ github.repository }}-test-runner:latest ( case "${{ inputs.platform }}" in @@ -86,13 +85,22 @@ jobs: echo TEST_IMAGE=${test_image} echo TEST_ARTIFACT=${test_artifact} echo TEST_ID=${test_id} + echo TEST_DATE=${test_date} echo UNO_MIDDLEWARE=${uno_middleware} ) >> ${GITHUB_OUTPUT} test: - needs: test_config - runs-on: ${{ fromJson(needs.test_config.outputs.RUNNER) }} + needs: config + runs-on: ${{ fromJson(needs.config.outputs.RUNNER) }} + env: + IN_DOCKER: y + FIX_DIR: ${{ github.workspace }} + RTI_LICENSE_FILE: ${{ github.workspace }}/src/uno-ci/resource/rti/rti_license.dat + TEST_DATE: ${{ needs.config.outputs.TEST_DATE }} + TEST_IMAGE: ${{ needs.config.outputs.TEST_IMAGE }} + TEST_RELEASE: y + UNO_MIDDLEWARE: ${{ needs.config.outputs.UNO_MIDDLEWARE }} steps: - name: Clone uno uses: actions/checkout@v4 @@ -108,14 +116,6 @@ jobs: ref: master path: src/uno-ci - - name: Configure tester - id: config - run: | - cp src/uno-ci/resource/rti/rti_license.dat rti_license.dat - - # Create test results directory - mkdir -p test-results - - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -130,7 +130,7 @@ jobs: uses: docker/build-push-action@v5 with: file: src/uno/docker/test/Dockerfile - tags: ${{ needs.test_config.outputs.TEST_IMAGE }} + tags: ${{ needs.config.outputs.TEST_IMAGE }} load: true context: src/uno platforms: ${{ inputs.platform }} @@ -139,81 +139,44 @@ jobs: - name: Run unit tests run: | - docker run --rm \ - -v $(pwd):/workspace \ - -w /workspace \ - -e RTI_LICENSE_FILE=/workspace/rti_license.dat \ - -e VERBOSITY=${{ env.VERBOSITY }} \ - -e DEBUG=${{ env.DEBUG }} \ - ${{ needs.test_config.outputs.TEST_IMAGE }} \ - pytest -s -v --junit-xml=test-results/uno-test-results-unit-${{ needs.test_config.outputs.TEST_ID }}.xml \ - src/uno/test/unit - - - name: Run unit tests (without a license) - run: | - docker run --rm \ - -v $(pwd):/workspace \ - -w /workspace \ - -e VERBOSITY=${{ env.VERBOSITY }} \ - -e DEBUG=${{ env.DEBUG }} \ - ${{ needs.test_config.outputs.TEST_IMAGE }} \ - pytest -s -v --junit-xml=test-results/uno-test-results-unit-${{ needs.test_config.outputs.TEST_ID }}-static.xml \ - src/uno/test/unit - - - name: Restore permissions changed by unit tests - if: ${{ always() }} - run: | - docker run --rm \ - -v $(pwd):/workspace \ - ${{ needs.test_config.outputs.TEST_IMAGE }} \ - fix-root-permissions $(id -u):$(id -g) /workspace + make -C src/uno test-unit + env: + DEBUG: ${{ runner.debug }} + TEST_ID: ${{ needs.config.outputs.TEST_ID }} + - - name: Setup integration tests + - name: Run unit tests (without a license) run: | - python3 -m venv venv - . venv/bin/activate - pip3 install -U pip setuptools - pip3 install -U -e src/uno - case '${{ needs.test_config.outputs.UNO_MIDDLEWARE }}' in - '') - pip3 install -U rti.connext - ;; - *) - pip3 install -U -e src/uno/plugins/${{ needs.test_config.outputs.UNO_MIDDLEWARE }} - ;; - esac + make -C src/uno test-unit + env: + DEBUG: ${{ runner.debug }} + NO_LICENSE: y + TEST_ID: ${{ needs.config.outputs.TEST_ID }}__static - name: Run integration tests run: | - . venv/bin/activate - pytest -s -v --junit-xml=test-results/uno-test-results-integration-${{ needs.test_config.outputs.TEST_ID }}.xml \ - src/uno/test/integration + make -C src/uno test-integration env: - RTI_LICENSE_FILE: ${{ github.workspace }}/rti_license.dat - UNO_MIDDLEWARE: ${{ needs.test_config.outputs.UNO_MIDDLEWARE }} - + DEBUG: ${{ runner.debug }} + TEST_ID: ${{ needs.config.outputs.TEST_ID }} - name: Run integration tests (without a license) run: | - . venv/bin/activate - rm -v rti_license.dat - pytest -s -v --junit-xml=test-results/uno-test-results-integration-${{ needs.test_config.outputs.TEST_ID }}-static.xml \ - src/uno/test/integration + make -C src/uno test-integration env: - UNO_MIDDLEWARE: ${{ needs.test_config.outputs.UNO_MIDDLEWARE }} + DEBUG: ${{ runner.debug }} + NO_LICENSE: y + TEST_ID: ${{ needs.config.outputs.TEST_ID }}__static - - name: Restore permissions changed by integration tests - if: ${{ always() }} + - name: Restore permissions changed by tests + if: always() run: | - docker run --rm \ - -v $(pwd):/workspace \ - ${{ needs.test_config.outputs.TEST_IMAGE }} \ - fix-root-permissions $(id -u):$(id -g) /workspace + make -C src/uno fix-file-ownership # Always collect and upload available test results - name: Upload test results uses: actions/upload-artifact@v4 with: - name: ${{ needs.test_config.outputs.TEST_ARTIFACT }} - path: test-results/* - if: ${{ always() }} + name: ${{ needs.config.outputs.TEST_ARTIFACT }} + path: src/uno/test-results/* + if: always() diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5b2af862..79e70ac3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,26 +50,23 @@ on: default: true concurrency: - group: ci-build-${{ github.ref }}-${{ inputs.uno-middleware || 'default' }}-${{ inputs.build-platform }}-${{ inputs.base-image }} + group: ci-build-${{ github.ref }}-${{ inputs.uno-middleware || 'default' }}-${{ inputs.build-platform }}-${{ inputs.base-image }}-${{inputs.test-without-license}} cancel-in-progress: true permissions: contents: read packages: read -env: - DEBUG: '' - VERBOSITY: '' - jobs: config: runs-on: ubuntu-latest outputs: RUNNER: ${{ fromJson(steps.config.outputs.RUNNER) }} - BUILD_ID: ${{ steps.config.outputs.BUILD_ID }} TEST_ARTIFACT: ${{ steps.config.outputs.TEST_ARTIFACT }} TEST_IMAGE: ${{ steps.config.outputs.TEST_IMAGE }} TEST_BASE_IMAGE: ${{ steps.config.outputs.TEST_BASE_IMAGE }} + TEST_DATE: ${{ steps.config.outputs.TEST_DATE }} + TEST_ID: ${{ steps.config.outputs.TEST_ID }} steps: - name: Clone uno uses: actions/checkout@v4 @@ -83,37 +80,30 @@ jobs: case "${{github.ref_type}}" in tag) image_version=${{github.ref_name}} - build_type=release ;; *) sha_short=$(cd src/uno && git rev-parse --short HEAD) image_version=${{github.ref_name}}@${sha_short} - case "${{github.ref_name}}" in - master) - build_type=nightly - ;; - *) - build_type=test - ;; - esac ;; esac image_version="$(echo ${image_version} | tr / -)" - build_id=$(date +%Y%m%d-%H%M%S) middleware_id=$(echo ${{inputs.uno-middleware}} | sed -e 's/uno.middleware.//') - test_artifact=uno-ci-${build_type}-${middleware_id:-default}-${{ inputs.build-platform }}__${image_version}__${build_id} - test_image=${{ github.repository }}-test-runner:latest base_image_tag=$(echo "${{ inputs.base-image }}" | tr : -) + license_tag=$([ "${{ inputs.test-without-license }}" = false ] || printf -- __static) test_base_image=ghcr.io/mentalsmash/uno-ci-base-tester:${base_image_tag} + test_image=${{ github.repository }}-test-runner:latest + test_date=$(date +%Y%m%d-%H%M%S) + test_id=ci-${middleware_id:-default}-${{ inputs.build-platform }}__${image_version}${license_tag} + test_artifact=uno-test-${test_id}__${test_date} ( case "${{ inputs.build-platform }}" in arm64) - case "${{inputs.uno-middleware}}" in - '') + case "${{inputs.test-without-license}}" in + false) # Force full test suite to run on "beefier" rpi5 nodes printf -- "RUNNER='%s'\n" '["self-hosted", "linux", "arm64", "rpi5"]' ;; - *) + true) # Other test suites can run on any arm64 node printf -- "RUNNER='%s'\n" '["self-hosted", "linux", "arm64"]' ;; @@ -123,15 +113,25 @@ jobs: printf -- "RUNNER='%s'\n" '"ubuntu-latest"' ;; esac - echo TEST_ARTIFACT=${test_artifact} echo TEST_IMAGE=${test_image} + echo TEST_ARTIFACT=${test_artifact} + echo TEST_ID=${test_id} + echo TEST_DATE=${test_date} echo TEST_BASE_IMAGE=${test_base_image} - echo BUILD_ID=${build_id} + echo REG_TAG=${ref_tag} ) >> ${GITHUB_OUTPUT} build-n-test: needs: config runs-on: ${{ fromJson(needs.config.outputs.RUNNER) }} + env: + IN_DOCKER: y + FIX_DIR: ${{ github.workspace }} + RTI_LICENSE_FILE: ${{ github.workspace }}/src/uno/rti_license.dat + TEST_DATE: ${{ needs.config.outputs.TEST_DATE }} + TEST_ID: ${{ needs.config.outputs.TEST_ID }} + TEST_IMAGE: ${{ needs.config.outputs.TEST_IMAGE }} + UNO_MIDDLEWARE: ${{ inputs.uno-middleware }} steps: - name: Clone uno uses: actions/checkout@v4 @@ -139,33 +139,9 @@ jobs: path: src/uno submodules: true - - name: Bootstrap uno with poetry - run: | - python3 -m venv poetry-venv - . poetry-venv/bin/activate - pip3 install -U poetry - deactivate - cd src/uno - ${GITHUB_WORKSPACE}/poetry-venv/bin/poetry install --with=dev - . .venv/bin/activate - case '${{ inputs.uno-middleware }}' in - '') - pip3 install -U rti.connext - ;; - *) - pip3 install -U -e plugins/${{ inputs.uno-middleware }} - ;; - esac - - name: Validate code run: | - . src/uno/.venv/bin/activate - ruff check - - - name: Validate code format - run: | - . src/uno/.venv/bin/activate - ruff format --check + make -C src/uno code-check - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -192,92 +168,50 @@ jobs: BASE_IMAGE=${{ needs.config.outputs.TEST_BASE_IMAGE }} UNO_MIDDLEWARE=${{ inputs.uno-middleware }} + - name: Set up integration tests + if: ${{ inputs.run-integration }} + run: | + make -C src/uno extract-license + - name: Run unit tests if: inputs.run-unit run: | - mkdir -p test-results - docker run --rm \ - -v $(pwd):/workspace \ - -w /workspace \ - -e VERBOSITY=${{ env.VERBOSITY }} \ - -e DEBUG=${{ env.DEBUG }} \ - -e EXPECT_MIDDLEWARE=${{ inputs.uno-middleware || 'uno.middleware.connext' }} \ - ${{ needs.config.outputs.TEST_IMAGE }} \ - pytest -s -v --junit-xml=test-results/uno-test-results-unit-${{ needs.config.outputs.BUILD_ID }}.xml \ - src/uno/test/unit + make -C src/uno test-unit + env: + DEBUG: ${{ runner.debug }} - name: Run unit tests (without license) if: inputs.run-unit && inputs.test-without-license run: | - mkdir -p test-results - docker run --rm \ - -v $(pwd):/workspace \ - -w /workspace \ - -e VERBOSITY=${{ env.VERBOSITY }} \ - -e DEBUG=${{ env.DEBUG }} \ - -e EXPECT_MIDDLEWARE= \ - ${{ needs.config.outputs.TEST_IMAGE }} \ - sh -c "rm /rti_license.dat && unset RTI_LICENSE_FILE && pytest -s -v --junit-xml=test-results/uno-test-results-unit-${{ needs.config.outputs.BUILD_ID }}-static.xml src/uno/test/unit" - - - name: Restore permissions changed by unit tests - if: ${{ inputs.run-unit && always() }} - run: | - docker run --rm \ - -v $(pwd):/workspace \ - ${{ needs.config.outputs.TEST_IMAGE }} \ - fix-root-permissions $(id -u):$(id -g) /workspace - - - name: Set up integration tests - if: ${{ inputs.run-integration }} - run: | - docker run --rm \ - -v $(pwd):/workspace \ - ${{ needs.config.outputs.TEST_IMAGE }} \ - cp /rti_license.dat /workspace/rti_license.dat + make -C src/uno test-unit + env: + DEBUG: ${{ runner.debug }} + NO_LICENSE: y - name: Run integration tests - if: inputs.run-integration + if: inputs.run-integration && !inputs.test-without-license run: | - mkdir -p test-results - . src/uno/.venv/bin/activate - pytest -s -v --junit-xml=test-results/uno-test-results-integration-${{ needs.config.outputs.BUILD_ID }}.xml \ - src/uno/test/integration + make -C src/uno test-integration env: - UNO_MIDDLEWARE: ${{ inputs.uno-middleware }} - EXPECT_MIDDLEWARE: ${{ inputs.uno-middleware || 'uno.middleware.connext' }} - - - name: Restore permissions changed by integration tests - if: ${{ inputs.run-integration && always() }} - run: | - docker run --rm \ - -v $(pwd):/workspace \ - ${{ needs.config.outputs.TEST_IMAGE }} \ - fix-root-permissions $(id -u):$(id -g) /workspace + DEBUG: ${{ runner.debug }} - name: Run integration tests (without license) if: inputs.run-integration && inputs.test-without-license run: | - rm rti_license.dat - mkdir -p test-results - . src/uno/.venv/bin/activate - pytest -s -v --junit-xml=test-results/uno-test-results-integration-${{ needs.config.outputs.BUILD_ID }}-static.xml \ - src/uno/test/integration + make -C src/uno test-integration env: - UNO_MIDDLEWARE: ${{ inputs.uno-middleware }} - EXPECT_MIDDLEWARE: ${{ inputs.uno-middleware || '' }} + DEBUG: ${{ runner.debug }} + NO_LICENSE: y - name: Restore permissions changed by integration tests - if: ${{ inputs.run-integration && always() }} + if: always() run: | - docker run --rm \ - -v $(pwd):/workspace \ - ${{ needs.config.outputs.TEST_IMAGE }} \ - fix-root-permissions $(id -u):$(id -g) /workspace + make -C src/uno fix-file-ownership - name: Upload test results uses: actions/upload-artifact@v4 + if: always() with: name: ${{ needs.config.outputs.TEST_ARTIFACT }} - path: test-results/* - if: ${{ always() }} + path: src/uno/test-results/* diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 1de2f1f6..3a6416dc 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -89,6 +89,10 @@ jobs: base-image: "ubuntu:22.04" uno-middleware: '' test-without-license: false + - build-platform: arm64 + base-image: "ubuntu:22.04" + uno-middleware: '' + test-without-license: true uses: ./.github/workflows/ci.yml secrets: inherit with: @@ -96,3 +100,16 @@ jobs: base-image: ${{matrix.base-image}} uno-middleware: ${{matrix.uno-middleware}} test-without-license: ${{matrix.test-without-license}} + + deb-validation: + needs: check-review-status + if: ${{ needs.check-review-status.outputs.FULL }} + strategy: + matrix: + base-tag: ["ubuntu:22.04"] + platform: [amd64] + uses: ./.github/workflows/_deb_build.yml + secrets: inherit + with: + base-tag: ${{ matrix.base-tag }} + platform: ${{ matrix.platform }} diff --git a/.github/workflows/pull_request_closed.yml b/.github/workflows/pull_request_closed.yml new file mode 100644 index 00000000..7fb0c925 --- /dev/null +++ b/.github/workflows/pull_request_closed.yml @@ -0,0 +1,37 @@ +name: Pull Request (Closed) +run-name: | + PR #${{github.event.pull_request.number}} [closed, ${{github.event.pull_request.merged && 'merged' || 'rejected' }}] ${{github.event.pull_request.title}} + +on: + pull_request: + types: + - closed + +concurrency: + group: pr-closed-${{ github.ref }} + cancel-in-progress: false + +permissions: + actions: write + packages: read + +jobs: + cleanup_jobs: + runs-on: ubuntu-latest + steps: + - name: Clone uno + uses: actions/checkout@v4 + with: + path: src/uno + submodules: true + + - name: "Clean up workflow runs" + run: | + scripts/cleanup_closed_pull_request.sh \ + ${{ github.repository }} \ + ${{ github.event.pull_request.number }} \ + ${{ github.event.pull_request.merged }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + ADMIN_IMAGE: ghcr.io/mentalsmash/uno-ci-admin:latest + diff --git a/.gitignore b/.gitignore index 38fd4462..ea853b3e 100644 --- a/.gitignore +++ b/.gitignore @@ -19,8 +19,7 @@ uvn.networks uno.db .pytest_cache .ruff_cache -venv -.venv +*venv* poetry.lock build debian/.debhelper/ @@ -34,4 +33,4 @@ debian-dist build/ *.spec debian/uno.debhelper.log - +test-results diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bcf768ef..4eae9603 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -104,46 +104,6 @@ ruff format ### Environment Setup -#### pipx + poetry - -1. Install `pipx`: - - ```sh - sudo apt-get install -y pipx - ``` - -2. Install `poetry`: - - ```sh - pipx install poetry - ``` - -3. Clone `uno`'s repository: - - ```sh - git clone --recurse-submodules https://github.com/mentalsmash/uno - ``` - -4. Create a virtual environment with all dependencies: - - ```sh - cd uno - - poetry install --with=dev - ``` - -5. Enable the virtual environment: - - ```sh - . .venv/bin/activate - ``` - -6. Install `git` commit hooks: - - ```sh - pre-commit install - ``` - #### venv + pip 1. Install the `venv` module: @@ -166,28 +126,22 @@ ruff format python3 -m venv .venv ``` -4. Enable the virtual environment: - - ```sh - . .venv/bin/activate - ``` - -5. (Optional) Make sure `pip` and `setuptools` are up to date: +4. (Optional) Make sure `pip` and `setuptools` are up to date: ```sh - pip install -U pip setuptools + .venv/bin/pip install -U pip setuptools ``` -6. Install `uno` and its dependencies: +5. Install `uno` and its dependencies: ```sh - pip install -e . + .venv/bin/pip install -e . ``` -7. Install `git` commit hooks: +6. Install `git` commit hooks: ```sh - pre-commit install + .venv/bin/pre-commit install ``` #### Codespaces diff --git a/Makefile b/Makefile index 4f08756e..20f1d50b 100644 --- a/Makefile +++ b/Makefile @@ -1,15 +1,51 @@ +############################################################################### +# Global makefile configuration +############################################################################### +# Name of the Debian Source Package DSC_NAME := $(shell dpkg-parsechangelog -S Source) +# Debian package version (-) DEB_VERSION := $(shell dpkg-parsechangelog -S Version) +# Debian package upstream version () UPSTREAM_VERSION := $(shell echo $(DEB_VERSION) | rev | cut -d- -f2- | rev) +# Original upstream archive (generated from git repo) UPSTREAM_TARBALL := $(DSC_NAME)_$(UPSTREAM_VERSION).orig.tar.xz +# Python module version PY_VERSION := $(shell grep __version__ uno/__init__.py | cut -d= -f2- | tr -d \" | cut '-d ' -f2-) +# Name of the Python module PY_NAME := $(shell grep '^name =' pyproject.toml | cut -d= -f2- | tr -d \" | cut '-d ' -f2-) +ifneq ($(PY_NAME),uno) +$(warning unexpected Python module name: '$(PY_NAME)') +endif +# Docker image for the Debian Builder container DEB_BUILDER ?= mentalsmash/debian-builder:latest +# Docker image for the Debian Tester container DEB_TESTER ?= mentalsmash/debian-tester:latest +# Local uno clone UNO_DIR ?= $(shell pwd) -RTI_LICENSE_FILE ?= $(UNO_DIR)/rti_license.dat +# Directory where to generate test logs +# When running inside do It MUST be a subdirectory of UNO_DIR +TEST_RESULTS_DIR ?= $(UNO_DIR)/test-results +# A unique ID for the test run +TEST_ID ?= local +# The date when the tests were run +TEST_DATE ?= $(shell date +%Y%m%d-%H%M%S) +# Common prefix for the JUnit XML report generated by tests +TEST_JUNIT_REPORT ?= $(PY_NAME)-test-$(TEST_ID)__$(TEST_DATE) +# Docker image used to run tests +TEST_IMAGE ?= mentalsmash/uno-test-runner:latest +# Global flag telling the makefile to perform actions (e.g. run unit test) in a container. +IN_DOCKER ?= +# Directory targeted by the fix-file-ownership target +FIX_DIR ?= $(UNO_DIR) +# Directory where to generate file +OUT_DIR ?= $(UNO_DIR) -export RTI_LICENSE_FILE +# Set default verbosity from DEBUG flag +ifneq ($(DEBUG),) +VERBOSITY ?= debug +endif +export VERBOSITY +export DEBUG ifneq ($(UPSTREAM_VERSION),$(PY_VERSION)) $(warning unexpected debian upstream version ('$(UPSTREAM_VERSION)' != '$(PY_VERSION)')) @@ -17,44 +53,119 @@ endif ifneq ($(DSC_NAME),$(PY_NAME)) $(warning unexpected debian source package name ('$(DSC_NAME)' != '$(PY_NAME)')) endif + +INVALID_RTI_LICENSE_FILE := \ + printf -- "ERROR: no RTI_LICENSE_FILE when testing without NO_LICENSE\n" >&2 \ + && exit 1 + +ifneq ($(NO_LICENSE),) +ifneq ($(RTI_LICENSE_FILE),) +$(warning suppressing RTI_LICENSE_FILE := $(RTI_LICENSE_FILE)) +endif +override undefine RTI_LICENSE_FILE +else # ifneq ($(NO_LICENSE),) +ifeq ($(RTI_LICENSE_FILE),) +$(warning no RTI_LICENSE_FILE specified) +CHECK_RTI_LICENSE_FILE = $(INVALID_RTI_LICENSE_FILE) +else ifeq ($(wildcard $(RTI_LICENSE_FILE)),) -$(warning no RTI_LICENSE_FILE detected or invalid ('$(RTI_LICENSE_FILE)')) +$(warning invalid RTI_LICENSE_FILE ('$(RTI_LICENSE_FILE)')) +CHECK_RTI_LICENSE_FILE = $(INVALID_RTI_LICENSE_FILE) +endif +endif +endif # ifneq ($(NO_LICENSE),) +ifeq ($(CHECK_RTI_LICENSE_FILE),) +CHECK_RTI_LICENSE_FILE := true +endif # ifeq ($(CHECK_RTI_LICENSE_FILE),) +# Export to make sure it's available to subprocesses +ifneq ($(RTI_LICENSE_FILE),) +override RTI_LICENSE_FILE := $(realpath $(RTI_LICENSE_FILE)) endif +export RTI_LICENSE_FILE + +ifeq ($(UNO_MIDDLEWARE),) +EXPECT_MIDDLEWARE := uno.middleware.connext +else +EXPECT_MIDDLEWARE := $(UNO_MIDDLEWARE) +endif + +ifneq ($(IN_DOCKER),) +IN_DOCKER_PREFIX := \ + docker run --rm \ + $$([ -n "$(TEST_RELEASE)" ] || printf -- '-v $(UNO_DIR):$(UNO_DIR)') \ + $$([ -n "$(NO_LICENSE)" ] || printf -- '-v $(RTI_LICENSE_FILE):/rti_license.dat') \ + -w $(UNO_DIR) \ + -e VERBOSITY=$(VERBOSITY) \ + -e DEBUG=$(DEBUG) \ + -e EXPECT_MIDDLEWARE=$(EXPECT_MIDDLEWARE) \ + $(TEST_IMAGE) +override undefine EXPECT_MIDDLEWARE +else # ifneq ($(IN_DOCKER),) +IN_DOCKER_PREFIX := +endif # ifneq ($(IN_DOCKER),) +export EXPECT_MIDDLEWARE + +# export INTEGRATION_TEST_ARGS in case it's needed by a recursive call +export INTEGRATION_TEST_ARGS .PHONY: \ build \ - tarball \ - clean \ - debuild \ changelog \ + clean \ + code \ + code-check \ + code-format \ + code-format-check \ + code-style \ + code-style-check \ + deb \ debtest \ + debuild \ + dockerimages \ + extract-license \ + fix-file-permissions \ + tarball \ test \ + test-integration \ test-unit \ - test-integration + venv-install \ + code \ + code-check \ + code-format +# Perform all build tasks build: build/default ; +# Build uno into a static binary using pyinstaller.sh build/%: ../$(UPSTREAM_TARBALL) - rm -rf $@ build/pyinstaller-$* - mkdir -p $@/src - tar -xvaf $< -C $@/src + rm -rf $@ dist/bundle/$* + mkdir -p dist/src + tar -xvaf $< -C dist/src scripts/bundle/pyinstaller.sh $* +# Delete files generated by "build" clean: - rm -rf build dist + rm -rf build +# Generate upstream archive tarball: ../$(UPSTREAM_TARBALL) ; ../$(UPSTREAM_TARBALL): git ls-files --recurse-submodules | tar -cvaf $@ -T- +# Update changelog entry and append build codename to version +# Requires the Debian Builder image. changelog: + # Try to make sure changelog is at a clean version + git checkout debian/changelog || true docker run --rm \ -v $(UNO_DIR)/:/uno \ -w /uno \ $(DEB_BUILDER) \ /uno/scripts/bundle/update_changelog.sh +# Build uno's debian packages. +# Requires the Debian Builder image. debuild: docker run --rm \ -v $(UNO_DIR)/:/uno \ @@ -62,18 +173,151 @@ debuild: $(DEB_BUILDER) \ /uno/scripts/debian_build.sh -debtest: - TEST_IMAGE=$(DEB_TESTER) \ - TEST_RUNNER=runner \ - DEV=y \ - pytest -s -v test/integration $(TEST_ARGS) +# Run integration tests using the debian package. +# Requires the Debian Tester image +debtest: .venv + $(MAKE) -C $(UNO_DIR) test-integration \ + TEST_IMAGE=$(DEB_TESTER) \ + TEST_RUNNER=runner +# Build the uno debian pacakge locally +deb: + $(MAKE) -C $(UNO_DIR) debuild + $(MAKE) -C $(UNO_DIR) dockerimage-debian-tester + $(MAKE) -C $(UNO_DIR) debtest + +# Run both unit and integration tests test: test-unit test-integration ; -test-unit: - pytest -s -v test/unit +# Run unit tests +BASE_UNIT_TEST_COMMAND := \ + pytest -s -v \ + --junit-xml=$(TEST_RESULTS_DIR)/$(TEST_JUNIT_REPORT)__unit.xml \ + test/unit +# When run by a CI test, the test image is expected to contain an +# embedded RTI license at /rti_license.dat. To test without it, we +# must change the test command to delete the license +ifneq ($(TEST_RELEASE),) +ifneq ($(NO_LICENSE),) +UNIT_TEST_COMMAND := \ + sh -exc '\ + rm /rti_license.dat; \ + unset RTI_LICENSE_FILE; \ + $(BASE_UNIT_TEST_COMMAND) $(UNIT_TEST_ARGS)' +endif # ifneq ($(NO_LICENSE),) +endif # ifneq ($(TEST_RELEASE),) +ifeq ($(UNIT_TEST_COMMAND),) +UNIT_TEST_COMMAND := $(BASE_UNIT_TEST_COMMAND) $(UNIT_TEST_ARGS) +endif # ifeq ($(UNIT_TEST_COMMAND),) + +test-unit: .venv + @$(CHECK_RTI_LICENSE_FILE) + mkdir -p $(TEST_RESULTS_DIR) + $(IN_DOCKER_PREFIX) $(UNIT_TEST_COMMAND) + +# Run integration tests +test-integration: .venv + @$(CHECK_RTI_LICENSE_FILE) + mkdir -p $(TEST_RESULTS_DIR) + $ Sun, 14 Apr 2024 21:16:34 -0700 + -- mentalsmash.org Tue, 16 Apr 2024 05:44:15 +0000 diff --git a/debian/source/options b/debian/source/options index bb02ccaf..fd7067c3 100644 --- a/debian/source/options +++ b/debian/source/options @@ -1 +1 @@ -extend-diff-ignore="venv|pycache|ruff|build|pytest|poetry|dist|uno.spec|runner.spec|rti_license.dat" +extend-diff-ignore="test-results|venv|pycache|ruff|build|pytest|poetry.lock|dist" diff --git a/debian/uno.install b/debian/uno.install index 62729859..d9812f12 100644 --- a/debian/uno.install +++ b/debian/uno.install @@ -1,4 +1,4 @@ -build/default/src opt/uno/ -build/pyinstaller-default/uno/_internal opt/uno/bin/ -build/pyinstaller-default/uno/uno opt/uno/bin/ +dist/src opt/uno/ +dist/bundle/default/uno/_internal opt/uno/bin/ +dist/bundle/default/uno/uno opt/uno/bin/ usr/bin/uno diff --git a/docker/Dockerfile b/docker/Dockerfile index dbf4a0dc..87add7de 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -101,34 +101,35 @@ COPY . ${UNO_DIR} RUN set -xe; \ # Delete .git directory rm -rf ${UNO_DIR}/.git; \ - # Generate new virtual environment + # Generate new virtual environment and + # install all Python packages inside it python3 -m venv ${UNO_VENV}; \ - # Install all Python packages in the virtual environment - . ${UNO_VENV}/bin/activate; \ # Make sure pip and setuptools are up to date - pip3 install -U pip setuptools; \ + ${UNO_VENV}/bin/pip3 install -U pip setuptools; \ # Check if the user passed a Connext wheel in the root of the context. # Try to install them iteratively (in sorted order), stop as soon as # one installs cleanly. for rti_whl in $(find ${UNO_DIR} -mindepth 1 -maxdepth 1 -name "rti.connext*$(uname -m).whl" | sort); do \ - if pip3 install -U ${rti_whl}; then \ + if ${UNO_VENV}/bin/pip3 install -U ${rti_whl}; then \ break; \ fi; \ done; \ rm -f ${UNO_DIR}/*.whl; \ # reinstall uno and the middleware in "editable" mode # so that they may be overwritten from the host (for development) - pip3 install -U $([ -z "${DEV}" ] || printf -- -e ) \ + ${UNO_VENV}/bin/pip3 install -U \ + $([ -z "${DEV}" ] || printf -- -e ) \ ${UNO_DIR}; \ # Install uno middleware case "${UNO_MIDDLEWARE}" in \ # default middleware requires Connext '') \ - pip3 install -U rti.connext; \ + ${UNO_VENV}/bin/pip3 install -U rti.connext; \ ;; \ *) \ # Other middleware plugins must be installed - pip3 install -U $([ -z "${DEV}" ] || printf -- -e ) \ + ${UNO_VENV}/bin/pip3 install -U \ + $([ -z "${DEV}" ] || printf -- -e ) \ ${UNO_PLUGINS_DIR}/${UNO_MIDDLEWARE}; \ ;; \ esac; \ diff --git a/docker/debian-tester/Dockerfile b/docker/debian-tester/Dockerfile index 02070e19..60345079 100644 --- a/docker/debian-tester/Dockerfile +++ b/docker/debian-tester/Dockerfile @@ -64,5 +64,6 @@ ENTRYPOINT [ "/entrypoint.sh" ] CMD ["agent"] ARG TEST_FLAVOR=default -COPY ./build/pyinstaller-${TEST_FLAVOR}-runner/runner /opt/runner +COPY ./dist/bundle/${TEST_FLAVOR}/runner /opt/runner +RUN chmod +x /opt/runner/runner ENV PATH="/opt/runner:${PATH}" diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index afff9045..d0ece087 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -51,7 +51,7 @@ sync) exec uno sync -r ${UVN_DIR} $@ ;; # Validate arguments and skip to bottom for chown -fix-root-permissions) +fix-file-ownership) shift OWNER=$1 shift diff --git a/pyproject.toml b/pyproject.toml index 316d6b4e..f83be4e6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "uno" -version = "0.9.0" +version = "0.9.1" description = "Dynamic site-to-site VPNs with WireGuard, frrouting, and RTI Connext DDS" authors = ["Andrea Sorbini "] readme = "README.md" diff --git a/scripts/bundle/pyinstaller.sh b/scripts/bundle/pyinstaller.sh index 89643da1..bb8a8a70 100755 --- a/scripts/bundle/pyinstaller.sh +++ b/scripts/bundle/pyinstaller.sh @@ -1,73 +1,91 @@ #!/bin/sh -set -ex +set -e FLAVOR=${1:-default} -DIST_DIR=$(pwd)/build/pyinstaller-${FLAVOR} -VENV_PYINST=${DIST_DIR}/venv -VENV_UNO=${DIST_DIR}/venv-uno -if [ ! -d ${VENV_PYINST} ]; then - python3 -m venv ${VENV_PYINST} - . ${VENV_PYINST}/bin/activate - pip install pyinstaller - deactivate -fi +: "${CLEAN:=yes}" +[ "${CLEAN}" = yes -o "${CLEAN}" = no ] -rm -rf build/uno* build/runner* ${VENV_UNO} -python3 -m venv ${VENV_UNO} -. ${VENV_UNO}/bin/activate -pip install . -case "${FLAVOR}" in - default) - pip install rti.connext - ;; - *) - ;; -esac -pip uninstall --yes pip setuptools -deactivate +: "${DIST_DIR:=$(pwd)/dist/bundle/${FLAVOR}}" +[ -n "${DIST_DIR}" ] -VENV_LIB=$(find ${VENV_UNO}/lib/*/ -mindepth 1 -maxdepth 1 -name site-packages | head -1) +: "${BUILD_DIR:=$(pwd)/build/pyinstaller}" +[ -n "${BUILD_DIR}" ] -RTI_DIST_INFO=$(find ${VENV_LIB} -name "rti.connext-*.dist-info" -mindepth 1 -maxdepth 1 | head -1) +( + set -x + rm -rf ${BUILD_DIR} +) -. ${VENV_PYINST}/bin/activate -pyinstaller \ - --noconfirm \ - --onedir \ - --clean \ - --distpath ${DIST_DIR} \ - --specpath build/ \ - --add-data $(pwd)/uno:uno \ - --hidden-import rti.connextdds \ - --hidden-import rti.idl_impl \ - --hidden-import rti.idl \ - --hidden-import rti.logging \ - --hidden-import rti.request \ - --hidden-import rti.rpc \ - --hidden-import rti.types \ - --add-data ${VENV_LIB}/rti:rti \ - --add-data ${VENV_LIB}/rti.connext.libs:rti.connext.libs \ - --add-data ${RTI_DIST_INFO}:$(basename ${RTI_DIST_INFO}) \ - -p ${VENV_LIB} \ +: "${SCRIPTS:=\ ./scripts/bundle/uno + ./uno/test/integration/runner.py}" +[ -n "${SCRIPTS}" ] + +: "${VENV_PYINST:=${BUILD_DIR}/venv-pyinst}" +[ -n "${VENV_PYINST}" ] + +: "${VENV_UNO:=${BUILD_DIR}/venv-uno}" +[ -n "${VENV_UNO}" ] + +if [ ! -d "${VENV_PYINST}" ]; then + ( + set -x + python3 -m venv ${VENV_PYINST} + set +x + . ${VENV_PYINST}/bin/activate + set -x + pip install pyinstaller + set +x + deactivate + ) +fi -pyinstaller \ - --noconfirm \ - --onedir \ - --clean \ - --distpath ${DIST_DIR}-runner \ - --specpath build/ \ - --add-data "../uno:uno" \ - --hidden-import rti.connextdds \ - --hidden-import rti.idl_impl \ - --hidden-import rti.idl \ - --hidden-import rti.logging \ - --hidden-import rti.request \ - --hidden-import rti.rpc \ - --hidden-import rti.types \ - --add-data ${VENV_LIB}/rti:rti \ - --add-data ${VENV_LIB}/rti.connext.libs:rti.connext.libs \ - --add-data ${RTI_DIST_INFO}:$(basename ${RTI_DIST_INFO}) \ - -p ${VENV_LIB} \ - ./uno/test/integration/runner.py +if [ "${CLEAN}" = yes -o ! -d "${VENV_UNO}" ]; then + ( + set -x + rm -rf ${VENV_UNO} + python3 -m venv ${VENV_UNO} + set +x + . ${VENV_UNO}/bin/activate + set -x + pip install . + [ "${FLAVOR}" != default ] || pip install rti.connext + pip uninstall --yes pip setuptools + set +x + deactivate + ) +fi + +VENV_LIB=$(find ${VENV_UNO}/lib/*/ -mindepth 1 -maxdepth 1 -name site-packages | head -1) +[ -n "${VENV_LIB}" ] + +RTI_DIST_INFO=$(find ${VENV_LIB} -mindepth 1 -maxdepth 1 -name "rti.connext-*.dist-info" | head -1) +[ -n "${RTI_DIST_INFO}" ] + +( + . ${VENV_PYINST}/bin/activate + set -x + for script in ${SCRIPTS}; do + pyinstaller \ + --noconfirm \ + --onedir \ + --clean \ + --workpath ${BUILD_DIR} \ + --distpath ${DIST_DIR} \ + --specpath build/ \ + --add-data $(pwd)/uno:uno \ + --hidden-import rti.connextdds \ + --hidden-import rti.idl_impl \ + --hidden-import rti.idl \ + --hidden-import rti.logging \ + --hidden-import rti.request \ + --hidden-import rti.rpc \ + --hidden-import rti.types \ + --add-data ${VENV_LIB}/rti:rti \ + --add-data ${VENV_LIB}/rti.connext.libs:rti.connext.libs \ + --add-data ${RTI_DIST_INFO}:$(basename ${RTI_DIST_INFO}) \ + -p ${VENV_LIB} \ + ${script} + done +) diff --git a/scripts/cleanup_closed_pull_request.sh b/scripts/cleanup_closed_pull_request.sh new file mode 100755 index 00000000..88b5f292 --- /dev/null +++ b/scripts/cleanup_closed_pull_request.sh @@ -0,0 +1,165 @@ +#!/bin/sh -e +if [ $# -ne 3 ]; then + printf -- "ERROR: invalid arguments\n" >&2 + printf -- "Usage: %s (true|false)\n" "$(basename $0)" >&2 + exit 1 +fi + +REPO="${1}" +PR_NO="${2}" +MERGED="${3:=false}" +UNO_DIR=$(cd $(dirname $0)/.. && pwd) + +if [ -n "${NOOP}" ]; then + OPT_NOOP="-e NOOP=y" +fi + +: "${GH_TOKEN:?GH_TOKEN is required but missing} +: "${ADMIN_IMAGE:=mentalsmash/uno-ci-admin:latest} + +log_msg() +{ + local lvl="${1}" + shift + printf -- "${lvl}: $@\n" >&2 +} + +RC=0 +case "${MERGED}" in + false) + PR_STATE=unmerged + log_msg INFO "deleting all runs for ${PR_STATE} PR #${PR_NO} of ${REPO}" + rc=0 + docker run --rm \ + -v ${UNO_DIR}:/uno \ + -e GH_TOKEN=${GH_TOKEN} \ + ${OPT_NOOP} \ + ${ADMIN_IMAGE} \ + /uno/scripts/cleanup_workflows.sh ${REPO} \ + "'PR #${REPO_NO} [" || rc=0 + if [ "${rc}" -ne 0 ]; then + log_msg ERROR "failed to delete all runs for ${PR_STATE} PR #${PR_NO} of ${REPO}" + RC=${rc} + else + log_msg INFO "deleted all runs for ${PR_STATE} PR #${PR_NO} of ${REPO}" + fi + ;; + true) + PR_STATE=merged + log_msg INFO "listing good 'basic validation' runs for ${PR_STATE} PR #${PR_NO} of ${REPO}" + BASIC_VALIDATION_ALL=$( + docker run --rm \ + -v ${UNO_DIR}:/uno \ + -e GH_TOKEN=${GH_TOKEN} \ + -e NOOP=y \ + ${ADMIN_IMAGE} \ + /uno/scripts/cleanup_workflows.sh ${REPO} \ + "^GOOD PR #${PR_NO} [changed]" + ) + log_msg INFO "$(echo "${BASIC_VALIDATION_ALL}" | grep -v '^$' | wc -l) good 'basic validation' runs detected for ${PR_STATE} PR #${PR_NO} of ${REPO}" + log_msg INFO "----------------------------------------------------------------" + echo "${BASIC_VALIDATION_ALL}" | grep -v '^$' >&2 + log_msg INFO "----------------------------------------------------------------" + BASIC_VALIDATION_RUN="$(echo "${BASIC_VALIDATION_ALL}" | grep -v '^$' | tail -1)" + if [ -z "${BASIC_VALIDATION_RUN}" ]; then + log_msg ERROR "no good 'basic validation' run detected for ${PR_STATE} PR #${PR_NO} of ${REPO}" + exit 1 + else + BASIC_VALIDATION_DELETE="$(echo "${BASIC_VALIDATION_ALL}" | grep -v '^$' | head -n -1)" + log_msg INFO "- $(echo "${BASIC_VALIDATION_DELETE}" | wc -l) extra runs will be deleted" + fi + + log_msg INFO "listing good 'full validation' runs for ${PR_STATE} PR #${PR_NO} of ${REPO}" + FULL_VALIDATION_ALL=$( + docker run --rm \ + -v ${UNO_DIR}:/uno \ + -e GH_TOKEN=${GH_TOKEN} \ + -e NOOP=y \ + ${ADMIN_IMAGE} \ + sh -c "/uno/scripts/cleanup_workflows.sh ${REPO} '^GOOD PR #${PR_NO} [reviewed, approved]'" + ) + log_msg INFO "$(echo "${FULL_VALIDATION_ALL}" | grep -v '^$' | wc -l) good 'full validation' runs detected for ${PR_STATE} PR #${PR_NO} of ${REPO}" + log_msg INFO "----------------------------------------------------------------" + echo "${FULL_VALIDATION_ALL}" | grep -v '^$' >&2 + log_msg INFO "----------------------------------------------------------------" + FULL_VALIDATION_RUN="$(echo "${FULL_VALIDATION_ALL}" | grep -v '^$' | tail -1)" + if [ -z "${FULL_VALIDATION_RUN}" ]; then + log_msg ERROR "no good 'full validation' run detected for ${PR_STATE} PR #${PR_NO} of ${REPO}" + exit 1 + else + FULL_VALIDATION_DELETE="$(echo "${FULL_VALIDATION_ALL}" | grep -v '^$' | head -n -1)" + log_msg INFO "- $(echo "${FULL_VALIDATION_DELETE}" | wc -l) extra runs will be deleted" + fi + + log_msg INFO "BASIC VALIDATION run: '${BASIC_VALIDATION_RUN}'" + log_msg INFO "FULL VALIDATION run: '${FULL_VALIDATION_RUN}'" + + log_msg INFO "deleting failed runs for ${PR_STATE} PR #${PR_NO} of ${REPO}" + rc=0 + docker run --rm \ + -v ${UNO_DIR}:/uno \ + -e GH_TOKEN=${GH_TOKEN} \ + ${OPT_NOOP} \ + ${ADMIN_IMAGE} \ + /uno/scripts/cleanup_workflows.sh ${REPO} \ + "^FAIL | ^cancelled 'PR #${PR_NO} [" || rc=$? + if [ "${rc}" -ne 0 ]; then + log_msg WARNING "failed to delete failed runs for ${PR_STATE} PR #${PR_NO} of ${REPO}" + RC=${rc} + else + log_msg INFO "DELETED failed runs for ${PR_STATE} PR #${PR_NO} of ${REPO}" + fi + + if [ -n "${BASIC_VALIDATION_DELETE}" ]; then + log_msg INFO "deleting extra 'basic validation' runs for ${PR_STATE} PR #${PR_NO} of ${REPO}" + echo "${BASIC_VALIDATION_DELETE}" > .delete_runs.log + rc=0 + docker run --rm \ + -v $(pwd)/.delete_runs.log:/delete_runs.log \ + -v ${UNO_DIR}:/uno \ + -e GH_TOKEN=${GH_TOKEN} \ + ${OPT_NOOP} \ + ${ADMIN_IMAGE} \ + sh -c "cat /delete_runs.log | head -n -1 | RAW=y /uno/scripts/cleanup_workflows.sh ${REPO}" || rc=$? + rm .delete_runs.log + if [ "${rc}" -ne 0 ]; then + log_msg WARNING "failed to delete extra 'basic validation' runs for ${PR_STATE} PR #${PR_NO} of ${REPO}" + RC=${rc} + else + log_msg INFO "DELETED extra 'basic validation' runs for ${PR_STATE} PR #${PR_NO} of ${REPO}" + fi + fi + + if [ -n "${FULL_VALIDATION_DELETE}" ]; then + log_msg INFO "deleting extra 'full validation' runs for ${PR_STATE} PR #${PR_NO} of ${REPO}" + echo "${FULL_VALIDATION_DELETE}" > .delete_runs.log + rc=0 + docker run --rm \ + -v $(pwd)/.delete_runs.log:/delete_runs.log \ + -v ${UNO_DIR}:/uno \ \ + -e GH_TOKEN=${GH_TOKEN} \ + ${OPT_NOOP} \ + ${ADMIN_IMAGE} \ + sh -c "cat /delete_runs.log | head -n -1 | RAW=y /uno/scripts/cleanup_workflows.sh ${REPO}" + rm .delete_runs.log + if [ "${rc}" -ne 0 ]; then + log_msg WARNING "failed to delete extra 'full validation' runs for ${PR_STATE} PR #${PR_NO} of ${REPO}" + RC=${rc} + else + log_msg INFO "DELETED extra 'full validation' runs for ${PR_STATE} PR #${PR_NO} of ${REPO}" + fi + fi + ;; + *) + printf -- "ERROR: invalid MERGED value: '%s' (expected either 'true' or 'false')\n" "${MERGED}" >&2 + exit 1 + ;; +esac + +if [ "${RC}" -ne 0 ]; then + log_msg ERROR "errors encountered while processing ${PR_STATE} PR #${PR_NO} of ${REPO}" +else + log_msg INFO "finished processing ${PR_STATE} PR #${PR_NO} of ${REPO}" +fi + +exit ${RC} diff --git a/scripts/cleanup_workflows.sh b/scripts/cleanup_workflows.sh index 8603ea21..7c319ecc 100755 --- a/scripts/cleanup_workflows.sh +++ b/scripts/cleanup_workflows.sh @@ -10,10 +10,14 @@ # First version +# (asorbini) Modified to take an optional filter argument to run in noninteractive mode + +declare REPO=${1:?No owner/repo specified} +FILTER="${2}" + set -o errexit set -o pipefail -declare repo=${1:?No owner/repo specified} jqscript() { @@ -42,22 +46,32 @@ EOF } selectruns() { - - gh api --paginate "/repos/$repo/actions/runs" \ - | jq -r -f <(jqscript) \ - | fzf --multi - + if [ -n "${RAW}" ]; then + # expect entries to be piped via stdin + cat - + else + gh api --paginate "/repos/${REPO}/actions/runs" | + jq -r -f <(jqscript) | + if [ -z "${FILTER}" ]; then + fzf --multi + else + fzf --multi --filter "${FILTER}" + fi + fi } deleterun() { local run id result - run=$1 + run="${1}" id="$(cut -f 3 <<< "$run")" - gh api -X DELETE "/repos/$repo/actions/runs/$id" - [[ $? = 0 ]] && result="OK!" || result="BAD" - printf "%s\t%s\n" "$result" "$run" - + if [ -z "${NOOP}" ]; then + gh api -X DELETE "/repos/${REPO}/actions/runs/$id" + [[ $? = 0 ]] && result="OK!" || result="BAD" + printf "%s\t%s\n" "$result" "$run" + else + printf -- '%s\n' "${run}" + fi } deleteruns() { diff --git a/test/install/test_install_docker.py b/test/install/test_install_docker.py index eb8862f8..2ec5b1e6 100644 --- a/test/install/test_install_docker.py +++ b/test/install/test_install_docker.py @@ -65,7 +65,7 @@ def test_install_docker(): f"{test_dir.parent.parent}:/uno", f"--platform=linux/{platform}", uno_image, - "fix-root-permissions", + "fix-file-ownership", f"{os.getuid()}:{os.getgid()}", "/uno", ], diff --git a/test/local_deploy/experiment_lib.sh b/test/local_deploy/experiment_lib.sh index 577cd2da..77883917 100755 --- a/test/local_deploy/experiment_lib.sh +++ b/test/local_deploy/experiment_lib.sh @@ -59,7 +59,7 @@ docker_network() net_gw="${3}" \ net_masquerade="${4}" - [ -n "${net_masquerade}" ] || net_masquerade=false + [ -n "${net_masquerade}" ] || net_masquerade=true log_debug " deleting docker network: ${net_name}" ( @@ -106,7 +106,7 @@ docker_network_w_bridge() --gateway=${bridge_gw} \ --ip-range=${net_client_range} \ --subnet=${net_subnet} \ - -o com.docker.network.bridge.enable_ip_masquerade=false \ + -o com.docker.network.bridge.enable_ip_masquerade=true \ -o com.docker.network.bridge.name=${bridge_name} \ ${net_name} \ >> ${EXPERIMENT_LOG} 2>&1 diff --git a/uno/__init__.py b/uno/__init__.py index 3e2f46a3..d69d16e9 100644 --- a/uno/__init__.py +++ b/uno/__init__.py @@ -1 +1 @@ -__version__ = "0.9.0" +__version__ = "0.9.1" diff --git a/uno/core/exec.py b/uno/core/exec.py index 2fb3425f..3f7ce152 100644 --- a/uno/core/exec.py +++ b/uno/core/exec.py @@ -78,7 +78,7 @@ def exec_command( stdout = subprocess.PIPE stderr = subprocess.PIPE elif debug: - stdout = sys.stdout + stdout = sys.stderr stderr = sys.stderr else: stdout = subprocess.DEVNULL diff --git a/uno/middleware/connext/connext_middleware.py b/uno/middleware/connext/connext_middleware.py index d6996741..838eae8f 100644 --- a/uno/middleware/connext/connext_middleware.py +++ b/uno/middleware/connext/connext_middleware.py @@ -86,14 +86,14 @@ def _search_dir(root: Path): else: log.warning("invalid RTI_LICENSE_FILE := {}", rti_license_env) - default_path = [Path.cwd()] - connext_home_env = os.getenv("CONNEXTDDS_DIR", os.getenv("NDDSHOME")) - if connext_home_env: - default_path.add(connext_home_env) - for root in default_path: - rti_license = _search_dir(root) - if rti_license: - return rti_license + # default_path = [Path.cwd()] + # connext_home_env = os.getenv("CONNEXTDDS_DIR", os.getenv("NDDSHOME")) + # if connext_home_env: + # default_path.add(connext_home_env) + # for root in default_path: + # rti_license = _search_dir(root) + # if rti_license: + # return rti_license return None diff --git a/uno/test/integration/experiment.py b/uno/test/integration/experiment.py index f374213c..ccb2c69b 100644 --- a/uno/test/integration/experiment.py +++ b/uno/test/integration/experiment.py @@ -35,7 +35,7 @@ import uno -_uno_dir = Path(uno.__file__).resolve().parent.parent +_UnoDir = Path(uno.__file__).resolve().parent.parent # Make "info" the minimum default verbosity when this module is loaded # if Logger.Level.warning >= Logger.level: @@ -56,13 +56,28 @@ class ExperimentLoader(Protocol): def __call__(self, **experiment_args) -> "Experiment": ... +_RtiLicenseFile = os.environ.get("RTI_LICENSE_FILE") +if _RtiLicenseFile: + _RtiLicenseFile = Path(_RtiLicenseFile).resolve() + +_ExternalTestDir = os.environ.get("TEST_DIR") +if _ExternalTestDir: + _ExternalTestDir = Path(_ExternalTestDir).resolve() + +# Don't resolve this path, because it might be "runner", +# passed by the Makefile +_RunnerScript = Path(os.environ.get("TEST_RUNNER", "/uno/uno/test/integration/runner.py")) + + class Experiment: Dev = bool(os.environ.get("DEV", False)) InsideTestRunner = bool(os.environ.get("UNO_TEST_RUNNER", False)) TestImage = os.environ.get("TEST_IMAGE", "mentalsmash/uno-test-runner:latest") - RunnerScript = os.environ.get("TEST_RUNNER", "/uno/uno/test/integration/runner.py") + RunnerScript = _RunnerScript + ExternalTestDir = _ExternalTestDir + RtiLicenseFile = _RtiLicenseFile BuiltImages = set() - UnoDir = _uno_dir + UnoDir = _UnoDir # Load the selected uno middleware plugin UnoMiddlewareEnv = os.environ.get("UNO_MIDDLEWARE") RunnerTestDir = Path("/experiment-tmp") @@ -101,7 +116,7 @@ def define( config = cls.load_config(config) # Check if the user specified a non-temporary test directory # Otherwise the experiment will allocate a temporary directory - test_dir = os.environ.get("TEST_DIR", test_dir) + test_dir = cls.ExternalTestDir or test_dir test_dir_tmp = None if test_dir is not None: test_dir = Path(test_dir) / name @@ -253,7 +268,7 @@ def registry(self) -> Registry: self.log.info("initializing UVN") self.define_uvn() self.log.info("initialized UVN") - self.log.activity("opening UVN registry from {}", self.registry_root) + self.log.info("opening UVN registry from {}", self.registry_root) registry = Registry.open(self.registry_root, readonly=True) self.log.info("opened UVN registry {}: {}", registry.root, registry) return registry @@ -306,13 +321,13 @@ def create(self) -> None: # (otherwise we will fail to recreate networks) self.wipe_containers() for net in self.networks: - self.log.debug("creating network: {}", net) + self.log.info("creating network: {}", net) net.create() - self.log.activity("network created: {}", net) + self.log.info("network created: {}", net) for host in self.hosts: - self.log.debug("creating host: {}", host) + self.log.info("creating host: {}", host) host.create() - self.log.activity("host created: {}", host) + self.log.info("host created: {}", host) self.log.info("created {} networks and {} containers", len(self.networks), len(self.hosts)) def fix_root_permissions(self) -> None: @@ -324,31 +339,12 @@ def fix_root_permissions(self) -> None: "--rm", *(tkn for hvol, vol in dirs.items() for tkn in ("-v", f"{hvol}:{vol}")), self.TestImage, - "fix-root-permissions", + "fix-file-ownership", f"{os.getuid()}:{os.getgid()}", *dirs.values(), ] ) - @cached_property - def rti_license(self) -> Path | None: - for license in [ - os.environ.get("RTI_LICENSE_FILE"), - self.root / "rti_license.dat", - self.test_dir / "rti_license.dat", - Path.cwd() / "rti_license.dat", - ]: - if not license: - continue - elif not isinstance(license, Path): - license = Path(license) - if license.exists(): - license = license.resolve() - self.log.debug("found RTI license: {}", license) - return license - self.log.warning("no RTI license file found") - return None - def uno(self, *args, **exec_args): verbose_flag = Logger.verbose_flag try: @@ -392,10 +388,10 @@ def uno(self, *args, **exec_args): *( [ "-v", - f"{self.rti_license}:/rti_license.dat", + f"{self.RtiLicenseFile}:/rti_license.dat", ] - if self.rti_license - else [] + if self.RtiLicenseFile + else ["-e", "RTI_LICENSE_FILE="] ), "-e", f"VERBOSITY={self.log.level.name}", @@ -413,14 +409,14 @@ def uno(self, *args, **exec_args): def tear_down(self, assert_stopped: bool = False) -> None: self.log.info("tearing down {} networks and {} containers", len(self.networks), len(self.hosts)) for host in self.hosts: - self.log.debug("tearing down host: {}", host) + self.log.info("tearing down host: {}", host) host.delete(ignore_errors=assert_stopped) - self.log.activity("host deleted: {}", host) + self.log.info("host deleted: {}", host) self.hosts.clear() for net in self.networks: - self.log.debug("tearing down net: {}", net) + self.log.info("tearing down net: {}", net) net.delete(ignore_errors=assert_stopped) - self.log.activity("network deleted: {}", net) + self.log.info("network deleted: {}", net) self.networks.clear() self.fix_root_permissions() self.log.info("removed all networks and containers", len(self.networks), len(self.hosts)) diff --git a/uno/test/integration/host.py b/uno/test/integration/host.py index 515134f7..4cc866ee 100644 --- a/uno/test/integration/host.py +++ b/uno/test/integration/host.py @@ -354,9 +354,9 @@ def create(self) -> None: ), *(["-v", f"{self.cell_package}:/package.uvn-agent"] if self.role == HostRole.CELL else []), *( - ["-v", f"{self.experiment.rti_license}:/rti_license.dat"] - if self.experiment.rti_license - else [] + ["-v", f"{self.experiment.RtiLicenseFile}:/rti_license.dat"] + if self.experiment.RtiLicenseFile + else ["-e", "RTI_LICENSE_FILE="] ), *( [ @@ -406,23 +406,35 @@ def start(self, wait: bool = False) -> subprocess.Popen: self.log.activity("started") return result - def print_logs(self, output_file: Path | None = None) -> None: + def print_logs(self) -> None: + self.log.debug("generating host logs... (DEBUG={})", self.log.DEBUG) if self.log.DEBUG: import sys - print("{} {} [host logs] {}".format("=" * 20, self.container_name, "=" * 20), file=sys.stderr) - exec_command(["docker", "logs", self.container_name]) - print( - "{} //{} [host logs] {}".format("=" * 19, self.container_name, "=" * 19), file=sys.stderr - ) - if output_file is None: + def _print(name, out): + print( + "{} {} [host logs - {}] {}".format("=" * 20, self.container_name, name, "=" * 20), + file=sys.stderr, + ) + if out: + print(out.decode(), file=sys.stderr) + else: + print("", file=sys.stderr) + print( + "{} //{} [host logs - {}] {}".format("=" * 19, self.container_name, name, "=" * 19), + file=sys.stderr, + ) + + result = exec_command(["docker", "logs", self.container_name], capture_output=True) + _print("stdout", result.stdout) + _print("stderr", result.stderr) + if self.experiment.ExternalTestDir: output_file = self.test_dir / "container.log" - exec_command(["docker", "logs", self.container_name], output_file=output_file) - self.log.debug("generated host logs: {}", output_file) + exec_command(["docker", "logs", self.container_name], output_file=output_file) + self.log.debug("generated host logs: {}", output_file) def stop(self) -> subprocess.Popen: - if self.log.DEBUG: - self.print_logs() + self.print_logs() self.log.debug("stopping container") return subprocess.Popen( [ diff --git a/uno/test/integration/network.py b/uno/test/integration/network.py index 5d06881c..29304d77 100644 --- a/uno/test/integration/network.py +++ b/uno/test/integration/network.py @@ -37,7 +37,7 @@ def __init__( i: int, private_lan: bool = False, transit_wan: bool = False, - masquerade_docker: bool = False, + masquerade_docker: bool = True, ) -> None: self.experiment = experiment self.name = name