From 0bc0b00c062848cea248f60d0de3029e80b45fda Mon Sep 17 00:00:00 2001 From: Jussi Kukkonen Date: Wed, 24 Jan 2024 14:10:16 +0200 Subject: [PATCH 1/6] actions: Add integrated issue management Goals: * If workflow fails and issue does not exist, file an issue * If issue for the workflow exists, add a comment * If workflow succeeds and issue exists for workflow, close issue This adds a new action update-issue which we expect all workflows (except possibly signing-event) to call as the last job. --- actions/update-issue/action.yml | 91 +++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 actions/update-issue/action.yml diff --git a/actions/update-issue/action.yml b/actions/update-issue/action.yml new file mode 100644 index 00000000..25368f85 --- /dev/null +++ b/actions/update-issue/action.yml @@ -0,0 +1,91 @@ +name: 'Update TUF-on-CI issue' +description: 'Create, close or add a comment in a GitHub issue for a workflow failure' +# * This action will open an issue per workflow if that workflow fails. +# * If an issue is open for that workflow already, the action will add a comment. +# * If an issue is open and the workflow succeeds, the action will close the issue. +# * The issue is identified using a label that is the workflow filename. +# * Required permissions: +# issues: write + +inputs: + token: + description: 'GitHub token' + required: true + + success: + description: '"true" if workflow is succeeding' + required: true + +runs: + using: "composite" + steps: + - name: Update issue + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + env: + SUCCESS: ${{ inputs.success }} + with: + github-token: ${{ inputs.token }} + script: | + var path = require("path") + + success = (process.env.SUCCESS == "true") + + // Find issue labeled with the forkflow filename + const label = path.basename(context.payload.workflow) + const issues = await github.rest.issues.listForRepo({ + owner: context.repo.owner, + repo: context.repo.repo, + labels: [label], + }) + if (issues.data.length == 0) { + issue_number = 0 + } else { + issue_number = issues.data[0].number + } + + if (success && !issue_number) { + console.log("update-issue: Nothing to do (success, no issue open)") + return + } + + // Build comment body + const run_url = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}` + if (success) { + body = `### Workflow run succeeded for '${context.workflow}'.\n` + + `Succesful run: ${run_url}\n\n` + + `Closing issue based on this success.` + } else { + body = `### Workflow run failed for '${context.workflow}'.\n` + + `Failed run: ${run_url}\n\n` + + `This issue will be automatically closed if a later run succeeds` + } + + // open, comment on, and close issue as needed + if (!success && !issue_number) { + console.log("update-issue: Opening a new issue on failure") + await github.rest.issues.create({ + owner: context.repo.owner, + repo: context.repo.repo, + title: `Failure in ${context.workflow}`, + labels: [label], + body: body, + }) + } + if (issue_number) { + console.log(`update-issue: Adding a comment (issue: ${issue_number})`) + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issue_number, + body: body, + }) + } + if (success) { + console.log(`update-issue: Closing issue on success (issue: ${issue_number})`) + await github.rest.issues.update({ + issue_number: issue_number, + owner: context.repo.owner, + repo: context.repo.repo, + state: "closed", + }) + } From 696198cf30e123dcce6a8b39eaa207d302275929 Mon Sep 17 00:00:00 2001 From: Jussi Kukkonen Date: Wed, 24 Jan 2024 15:18:17 +0200 Subject: [PATCH 2/6] actions: Use "--quiet" with pip The pip install log is massively long in the logs: make it a little more readable. --- actions/create-signing-events/action.yml | 2 +- actions/online-sign/action.yml | 2 +- actions/signing-event/action.yml | 2 +- actions/test-repository/action.yml | 2 +- actions/upload-repository/action.yml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/actions/create-signing-events/action.yml b/actions/create-signing-events/action.yml index 62e2e005..7913694f 100644 --- a/actions/create-signing-events/action.yml +++ b/actions/create-signing-events/action.yml @@ -18,7 +18,7 @@ runs: with: python-version: "3.11" - - run: pip install $GITHUB_ACTION_PATH/../../repo/ + - run: pip --quiet install $GITHUB_ACTION_PATH/../../repo/ shell: bash - name: Create signing event branches for expiring roles diff --git a/actions/online-sign/action.yml b/actions/online-sign/action.yml index e004efff..89342026 100644 --- a/actions/online-sign/action.yml +++ b/actions/online-sign/action.yml @@ -49,7 +49,7 @@ runs: with: python-version: "3.11" - - run: pip install $GITHUB_ACTION_PATH/../../repo/ + - run: pip --quiet install $GITHUB_ACTION_PATH/../../repo/ shell: bash - id: online-sign diff --git a/actions/signing-event/action.yml b/actions/signing-event/action.yml index 595be4ee..49fe645e 100644 --- a/actions/signing-event/action.yml +++ b/actions/signing-event/action.yml @@ -33,7 +33,7 @@ runs: with: python-version: "3.11" - - run: pip install $GITHUB_ACTION_PATH/../../repo/ + - run: pip --quiet install $GITHUB_ACTION_PATH/../../repo/ shell: bash - id: update_targets diff --git a/actions/test-repository/action.yml b/actions/test-repository/action.yml index 5ddcbc13..75cb6d05 100644 --- a/actions/test-repository/action.yml +++ b/actions/test-repository/action.yml @@ -21,7 +21,7 @@ runs: with: python-version: "3.11" - - run: pip install $GITHUB_ACTION_PATH/../../repo/ + - run: pip --quiet install $GITHUB_ACTION_PATH/../../repo/ shell: bash - env: diff --git a/actions/upload-repository/action.yml b/actions/upload-repository/action.yml index d902d254..d8e163b2 100644 --- a/actions/upload-repository/action.yml +++ b/actions/upload-repository/action.yml @@ -30,7 +30,7 @@ runs: with: python-version: "3.11" - - run: pip install $GITHUB_ACTION_PATH/../../repo/ + - run: pip --quiet install $GITHUB_ACTION_PATH/../../repo/ shell: bash - id: build-repository From 5563954ae9d96927f1c76a57a1cd35636fc0a4ba Mon Sep 17 00:00:00 2001 From: Jussi Kukkonen Date: Wed, 24 Jan 2024 18:54:28 +0200 Subject: [PATCH 3/6] actions: Improve issue comment in update-issue --- actions/update-issue/action.yml | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/actions/update-issue/action.yml b/actions/update-issue/action.yml index 25368f85..87de3d24 100644 --- a/actions/update-issue/action.yml +++ b/actions/update-issue/action.yml @@ -51,13 +51,24 @@ runs: // Build comment body const run_url = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}` if (success) { - body = `### Workflow run succeeded for '${context.workflow}'.\n` + + body = `### Workflow run succeeded for ${context.workflow}.\n` + `Succesful run: ${run_url}\n\n` + `Closing issue based on this success.` + } else if (issue_number){ + body = `### Workflow run failed for ${context.workflow}.\n` + + `Failed run: ${run_url}\n\n` } else { - body = `### Workflow run failed for '${context.workflow}'.\n` + + retry_line = "" + if (label == "online-sign.yml" || + label == "create-signing-events.yml" || + label == "test-repository.yml") { + retry_line = "* This workflow will be retried automatically (by default in 6 hours)\n" + } + body = `### Workflow run failed for ${context.workflow}.\n` + `Failed run: ${run_url}\n\n` + - `This issue will be automatically closed if a later run succeeds` + retry_line + + "* Maintainers can re-run the failing job manually\n" + + "* This issue will be automatically closed if a later run succeeds" } // open, comment on, and close issue as needed From c9d91e56361c7c4fb95bfaa16159fd871e688ee1 Mon Sep 17 00:00:00 2001 From: Jussi Kukkonen Date: Thu, 25 Jan 2024 19:28:34 +0200 Subject: [PATCH 4/6] actions: Always run test on publish branch This is the only thing that makes sense: * publish branch documents what should be published * test action compares the published metadata to the "expected metadata" in sources on this branch This fixes the potential issue where cron workflows always run on main and could then use the wrong content as "expected metadata". --- actions/test-repository/action.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/actions/test-repository/action.yml b/actions/test-repository/action.yml index 75cb6d05..2dbc362c 100644 --- a/actions/test-repository/action.yml +++ b/actions/test-repository/action.yml @@ -16,6 +16,8 @@ runs: using: "composite" steps: - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + ref: "publish" - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0 with: From 43912c7f498490776e1db82ad5eda50a553f2b7b Mon Sep 17 00:00:00 2001 From: Jussi Kukkonen Date: Sat, 27 Jan 2024 11:58:20 +0200 Subject: [PATCH 5/6] actions: Fix bug with labeling context.payload does not contain 'workflow' for all event types. Use the actual workflow name instead: it looks like GitHub labels allow just about anything. Remove the custom message for specific workflows: this was on the verge of too complicated already, now it's over the line. Fix a typo in issue comment content. --- actions/update-issue/action.yml | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/actions/update-issue/action.yml b/actions/update-issue/action.yml index 87de3d24..af06d7b7 100644 --- a/actions/update-issue/action.yml +++ b/actions/update-issue/action.yml @@ -3,7 +3,7 @@ description: 'Create, close or add a comment in a GitHub issue for a workflow fa # * This action will open an issue per workflow if that workflow fails. # * If an issue is open for that workflow already, the action will add a comment. # * If an issue is open and the workflow succeeds, the action will close the issue. -# * The issue is identified using a label that is the workflow filename. +# * The issue is identified using a label that is the workflow name. # * Required permissions: # issues: write @@ -30,12 +30,11 @@ runs: success = (process.env.SUCCESS == "true") - // Find issue labeled with the forkflow filename - const label = path.basename(context.payload.workflow) + // Find issue labeled with the forkflow name const issues = await github.rest.issues.listForRepo({ owner: context.repo.owner, repo: context.repo.repo, - labels: [label], + labels: [context.workflow], }) if (issues.data.length == 0) { issue_number = 0 @@ -52,21 +51,14 @@ runs: const run_url = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}` if (success) { body = `### Workflow run succeeded for ${context.workflow}.\n` + - `Succesful run: ${run_url}\n\n` + + `Successful run: ${run_url}\n\n` + `Closing issue based on this success.` } else if (issue_number){ body = `### Workflow run failed for ${context.workflow}.\n` + `Failed run: ${run_url}\n\n` } else { - retry_line = "" - if (label == "online-sign.yml" || - label == "create-signing-events.yml" || - label == "test-repository.yml") { - retry_line = "* This workflow will be retried automatically (by default in 6 hours)\n" - } body = `### Workflow run failed for ${context.workflow}.\n` + `Failed run: ${run_url}\n\n` + - retry_line + "* Maintainers can re-run the failing job manually\n" + "* This issue will be automatically closed if a later run succeeds" } @@ -78,7 +70,7 @@ runs: owner: context.repo.owner, repo: context.repo.repo, title: `Failure in ${context.workflow}`, - labels: [label], + labels: [context.workflow], body: body, }) } From 71368595431520812eb12e7cb6c413e7cdb53047 Mon Sep 17 00:00:00 2001 From: Jussi Kukkonen Date: Mon, 29 Jan 2024 10:19:13 +0200 Subject: [PATCH 6/6] Lint fixes * These are unrelated to the PR, they are from black 24.1 * We should start pinning test deps but I'm not doing it here --- signer/tuf_on_ci_sign/_signer_repository.py | 36 ++++++++++----------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/signer/tuf_on_ci_sign/_signer_repository.py b/signer/tuf_on_ci_sign/_signer_repository.py index 66675af4..e2beb599 100644 --- a/signer/tuf_on_ci_sign/_signer_repository.py +++ b/signer/tuf_on_ci_sign/_signer_repository.py @@ -397,18 +397,18 @@ def set_online_config(self, online_config: OnlineConfig): root.add_key(online_config.key, "snapshot") # set online role periods - timestamp.unrecognized_fields[ - "x-tuf-on-ci-expiry-period" - ] = online_config.timestamp_expiry - timestamp.unrecognized_fields[ - "x-tuf-on-ci-signing-period" - ] = online_config.timestamp_signing - snapshot.unrecognized_fields[ - "x-tuf-on-ci-expiry-period" - ] = online_config.snapshot_expiry - snapshot.unrecognized_fields[ - "x-tuf-on-ci-signing-period" - ] = online_config.snapshot_signing + timestamp.unrecognized_fields["x-tuf-on-ci-expiry-period"] = ( + online_config.timestamp_expiry + ) + timestamp.unrecognized_fields["x-tuf-on-ci-signing-period"] = ( + online_config.timestamp_signing + ) + snapshot.unrecognized_fields["x-tuf-on-ci-expiry-period"] = ( + online_config.snapshot_expiry + ) + snapshot.unrecognized_fields["x-tuf-on-ci-signing-period"] = ( + online_config.snapshot_signing + ) def get_role_config(self, rolename: str) -> OfflineConfig | None: """Read configuration for delegation and role from metadata""" @@ -546,12 +546,12 @@ def set_role_config( if expiry == config.expiry_period and signing == config.signing_period: raise AbortEdit(f"No changes to {rolename}") - signed.unrecognized_fields[ - "x-tuf-on-ci-expiry-period" - ] = config.expiry_period - signed.unrecognized_fields[ - "x-tuf-on-ci-signing-period" - ] = config.signing_period + signed.unrecognized_fields["x-tuf-on-ci-expiry-period"] = ( + config.expiry_period + ) + signed.unrecognized_fields["x-tuf-on-ci-signing-period"] = ( + config.signing_period + ) state_file_path = os.path.join(self._dir, ".signing-event-state") if self._invites: