From 09759b0a8ec6e2eaf550be67cf8b99951e49b7c5 Mon Sep 17 00:00:00 2001 From: Nick Date: Sat, 26 Mar 2022 14:28:44 -0400 Subject: [PATCH 01/13] Workflow to build and release the current tag. This only runs on tags that include this file, so it won't run on any upstream tags, but only on tags into which this has been merged/cherry-picked. --- .github/workflows/release.yml | 60 +++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 000000000000..9fea838a4ef2 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,60 @@ +name: Release + +on: + push: + tags: + - "v*.*.*" + + # This second trigger covers the case where you + # delete and recreate from an existing tag + release: + types: + - created + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Install toolchain + run: | + sudo apt-get update && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y make git xz-utils + # per README.md, building needs Go 1.17 and Node LTS + - uses: actions/setup-go@v2 + with: + go-version: '^1.17' # The Go version to download (if necessary) and use. + - uses: actions/setup-node@v2 + with: + node-version: 'lts/*' + + - name: Build Release Assets + # The officially releases use 'make release' (https://github.com/neuropoly/gitea/blob/65e42f83e916af771a51af6a3f8db483ffa05c05/.drone.yml#L772) + # but that does cross-compilation (via docker (via https://github.com/techknowlogick/xgo)) + # which is overhead and complication I don't need or want to deal with. + # + # Instead, just do native Linux compilation then pretend we did 'make release'. + run: | + TAGS="bindata sqlite sqlite_unlock_notify" make build + mkdir -p dist/release + cp -p gitea dist/release/gitea-$(git describe --tags --always)-linux-amd64 + + - name: Compress Release Assets + run: | + xz -k dist/release/* + + - name: Checksum Release Assets + # each release asset in the official build process gets a separate .sha256 file + # which means we need a loop to emulate it + run: | + (cd dist/release; for asset in *; do sha256sum $asset > $asset.sha256; done) + + - name: Upload Release + # this Action creates the release if not yet created + uses: softprops/action-gh-release@v1 + with: + # We don't have .drone.yml's pretty changelog + # generator, so just zero-out the release notes + body: '' + files: 'dist/release/*' + fail_on_unmatched_files: true From 75aba369da73d07bf9e59872179451397bdca0a6 Mon Sep 17 00:00:00 2001 From: Nick Guenther Date: Sat, 26 Mar 2022 15:54:15 -0400 Subject: [PATCH 02/13] Workflow to automatically sync with upstream. We plan to keep this fork alive for a significant amount of time, because it will hold an extra feature (https://github.com/neuropoly/gitea/pull/1) that upstream doesn't have. Automate keeping up to date with upstream to reduce our overhead. The scheme I chose for this is a little bit convoluted. There are three branches involved, but this means that the patch that we might potentially upstream is clean. Also, this scheme can't fix merge conflicts automatically, but it can find and email about them automatically, which is about as good as we can hope for. --- .github/workflows/sync-upstream.yml | 145 ++++++++++++++++++++++++++++ 1 file changed, 145 insertions(+) create mode 100644 .github/workflows/sync-upstream.yml diff --git a/.github/workflows/sync-upstream.yml b/.github/workflows/sync-upstream.yml new file mode 100644 index 000000000000..a258e22dfa70 --- /dev/null +++ b/.github/workflows/sync-upstream.yml @@ -0,0 +1,145 @@ +# This soft-fork of Gitea adds git-annex support (https://git-annex.branchable.com/) +# git-annex is like git-lfs, which Gitea already supports, but more complicated, +# except that it doesn't need an extra port open. +# +# We maintain three branches and N tags: +# - main - a mirror of upstream's main +# - git-annex - our patch (see it at: https://github.com/neuropoly/gitea/pull/1) +# - release-action - release scripts + our front page +# - $X-git-annex for each upstream tag $X (each created after we started tracking upstream, that is) +# which = $X + release-action + git-annex +# +# This branch, release-action, contains: +# - sync-upstream.yml (this) - try to update the branches/tags +# - release.yml - build and push to https://github.com/neuropoly/gitea/releases/ +# and it is our default branch because cronjobs are +# only allowed to run on the default branch + +name: 'Sync Upstream' + +on: + workflow_dispatch: + schedule: + # 08:00 Montreal time, every day + - cron: '0 13 * * *' + +jobs: + sync_upstream: + name: 'Sync Upstream' + runs-on: ubuntu-latest + steps: + + #- name: debug - github object + # run: | + # echo '${{ tojson(github) }}' + + - name: Git Identity + run: | + set -ex + git config --global user.name "Actions Bot" + # or 41898282+github-actions[bot]@users.noreply.github.com ? + git config --global user.email action@github.com + + #- name: Git config + # run: | + # set -ex + # # disambiguates 'git checkout' so it always uses this repo + # #git config --global checkout.defaultRemote origin + + - uses: actions/checkout@v3 + + - name: Add upstream + run: | + set -ex + + PARENT=$(curl -s https://api.github.com/repos/${{github.repository}} | jq -r '.parent.clone_url // empty') + git remote add upstream "$PARENT" + + + - name: Fetch current origin + run: | + set -ex + # Because actions/checkout does a lazy, shallow checkout + # we need to use --shallow-since to make sure there's + # enough common history that git can tell how the two + # branches relate. + # + # We *could* do a full checkout by setting depth: 0 above, + # but this is faster, especially on a large repo like this one. + # + # Since this runs daily, 1 week should be plenty. + git fetch '--shallow-since=1 week' origin main "${{ github.ref_name }}" git-annex + git fetch '--shallow-since=1 week' upstream main + + - name: Sync main + # force main to be identical to upstream + # This throws away any commits to our local main + # so don't commit anything to that branch. + run: | + set -ex + git checkout -B main upstream/main + + - name: Sync ${{ github.ref_name }} + run: | + set -ex + git checkout "${{ github.ref_name }}" + git rebase main + + - name: Rebase git-annex, the feature branch + # This is the meatiest part of this script: rebase git-annex on top of upstream. + # Occasionally this step will fail -- when there's a merge conflict with upstream. + # In that case, you will get an email about it, and you should run these steps + # manually, and fix the merge conflicts that way. + run: | + set -ex + git checkout git-annex + git rebase main + + - name: Construct latest version with git-annex on top + run: | + # for the latest tag vX.Y.Z, construct tag vX.Y.Z-git-annex. + # Only construct the *latest* release to reduce the risk of conflicts + # (we have to ask 'git tag' instead of the more elegant method of syncing tags + # and using Github Actions' `on: push: tags: ...` because those upstream tags + # *don't contain this workflow*, so there would be no way to trigger this) + # + # This will trigger release.yml to build and publish the latest version, too + set -e + + # git fetch is supposed to get any tags corresponding to commits it downloads, + # but this behaviour is ignored when combined with --shallow, and there doesn't + # seem to be any other way to get a list of tags without downloading all of them, + # which effectively does --unshallow. But the GitHub API provides a shortcut, and + # using this saves about 30s over downloading the unshallow repo: + PARENT_API=$(curl -s https://api.github.com/repos/${{github.repository}} | jq -r '.parent.url // empty') + PARENT_TAGS=$(curl -s "$PARENT_API"| jq -r '.tags_url // empty') + RELEASE=$(curl -s "$PARENT_TAGS" | jq -r 'map(.name | select(test("dev") | not)) | first // empty') + # https://stackoverflow.com/questions/26617862/git-shallow-fetch-of-a-new-tag + git fetch --depth 1 upstream tag "$RELEASE" + + # But if we decide to just unshallow the entire repo from the start, + # then you can use this instead: + #RELEASE="$(git tag -l --sort=-v:refname | egrep -v 'git-annex$' | head -n 1)" + + if git fetch -q --depth 1 origin tag "$RELEASE"-git-annex 2>/dev/null; then + echo "$RELEASE-git-annex already published :tada:" + else + set -x + git checkout -q "$RELEASE" + git cherry-pick main.."${{ github.ref_name }}" # Make sure release.yml is in the tag, so it triggers a build + git cherry-pick main..git-annex + git tag "$RELEASE"-git-annex + + # If this step fails due to merge conflicts, + # it's probably because the most recent merge conflict + # occurred somewhere after the most recent release but + # before the current upstream/main. + # + # You should just manually create the tag, and fix the merge conflicts. + # This won't try to overwrite a pre-existing tag. + fi + + - name: Upload everything back to Github + run: | + git push -f --all + git push -f --tags From d2beaa8fb1f1d2e02458aae3ddedaa9fff18ce5b Mon Sep 17 00:00:00 2001 From: kousu Date: Fri, 8 Apr 2022 03:50:38 -0400 Subject: [PATCH 03/13] Enable actuallying syncing all tags. fetching the latest tag with --depth 1 fails: that makes a shallow commit (aka a 'grafted' commit) and github refuses to accept it when being pushed back. Instead, use --shallow-exclude=HEAD which seems to get enough of the history without getting *all* of the history that github will accept the upload --- .github/workflows/sync-upstream.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/sync-upstream.yml b/.github/workflows/sync-upstream.yml index a258e22dfa70..607ffdd6ecbd 100644 --- a/.github/workflows/sync-upstream.yml +++ b/.github/workflows/sync-upstream.yml @@ -114,8 +114,11 @@ jobs: PARENT_API=$(curl -s https://api.github.com/repos/${{github.repository}} | jq -r '.parent.url // empty') PARENT_TAGS=$(curl -s "$PARENT_API"| jq -r '.tags_url // empty') RELEASE=$(curl -s "$PARENT_TAGS" | jq -r 'map(.name | select(test("dev") | not)) | first // empty') + # https://stackoverflow.com/questions/26617862/git-shallow-fetch-of-a-new-tag - git fetch --depth 1 upstream tag "$RELEASE" + # --shallow-exclude=HEAD seems to get enough history that we can push back + # all necessary tags, but I'm a bit unclear if that's correct + git fetch --shallow-exclude=HEAD upstream tag "$RELEASE" # But if we decide to just unshallow the entire repo from the start, # then you can use this instead: From b2d8c710ed81bfd8168a0fcc4775f108521dfe0f Mon Sep 17 00:00:00 2001 From: kousu Date: Fri, 8 Apr 2022 22:40:10 -0400 Subject: [PATCH 04/13] Trigger a build when pushing a new *-git-annex tag. We need a special token, given to us by tibdex/github-app-token, because the default token that workflows are given is constrained: > When you use the repository's GITHUB_TOKEN to perform tasks, events > triggered by the GITHUB_TOKEN will not create a new workflow run. > This prevents you from accidentally creating recursive workflow runs. > For example, if a workflow run pushes code using the repository's > GITHUB_TOKEN, a new workflow will not run even when the repository > contains a workflow configured to run when push events occur. - https://docs.github.com/en/actions/security-guides/automatic-token-authentication#using-the-github_token-in-a-workflow See also https://github.community/t/push-from-workflow-not-doesnt-trigger-on-push-tags-workflow/149151/2 GitHub recommends https://docs.github.com/en/actions/using-workflows/reusing-workflows#calling-a-reusable-workflow for solving this, but that seems a lot less flexible to me. I'd rather be able to choose whether to trigger a build by pushing a tag manually or having the bot do it. --- .github/workflows/sync-upstream.yml | 58 +++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/.github/workflows/sync-upstream.yml b/.github/workflows/sync-upstream.yml index 607ffdd6ecbd..42949a051975 100644 --- a/.github/workflows/sync-upstream.yml +++ b/.github/workflows/sync-upstream.yml @@ -46,7 +46,64 @@ jobs: # # disambiguates 'git checkout' so it always uses this repo # #git config --global checkout.defaultRemote origin + - id: generate_token + # The default token provided doesn't have enough rights + # for 'git push --tags' to trigger the build in release.yml: + # + # > When you use the repository's GITHUB_TOKEN to perform tasks, events + # > triggered by the GITHUB_TOKEN will not create a new workflow run. + # > This prevents you from accidentally creating recursive workflow runs. + # + # ref: https://docs.github.com/en/actions/security-guides/automatic-token-authentication#using-the-github_token-in-a-workflow + # + # But we're not making a recursive workflow, and really do want to + # trigger the release.yml workflow. + # + # https://github.com/tibdex/github-app-token works around this by + # trading OAuth credentials (created at the Organization level) for + # a token credential (which can't be created except at the user level) + # + # Two alternate solutions are: + # + # 1. Provide a personal token (https://github.com/settings/tokens) but then + # whoever does exposes their account to everyone in the organization, + # and the organization is exposed to DoS if the person ever leaves. + # 2. Use workflow_call: (https://docs.github.com/en/actions/using-workflows/reusing-workflows) + # but this is a substantial amount of intricate code + # for something that should be simple. + # + # If you need to reconfigure this, know that you need to: + # + # 1. Create an OAuth credential at https://github.com/organizations/neuropoly/settings/apps + # a. Set 'Name' = 'gitea-sync' + # b. Set 'Homepage URL' = 'https://github.com/neuropoly/github-app-token' + # c. Uncheck 'Webhook' + # d. Set 'Repository permissions / Contents' = 'Access: Read & write'. + # e. Set 'Where can this GitHub App be installed' = 'Only on this account' + # 2. Click 'Generate Private Key'; it will download a .pem file to your computer. + # 3. Store the credential in the repo at https://github.com/neuropoly/gitea/settings/secrets/actions + # a. Set 'APP_ID' = the app ID displayed on https://github.com/organizations/neuropoly/settings/apps/gitea-sync + # b. Set 'APP_KEY' = the contents of the .pem file it downloaded. + # c. Now you can throw away the .pem file. + # 4. Install the app: + # a. Go to https://github.com/organizations/neuropoly/settings/apps/gitea-sync/installations + # b. Click 'Install' + # c. Pick this repo + # d. Click 'Install' for real this time + # + # ref: https://github.com/peter-evans/create-pull-request/blob/main/docs/concepts-guidelines.md#authenticating-with-github-app-generated-tokens + # + # Notice too: we've **forked** a copy of tibdex/github-app-token, + # to avoid passing our tokens through potentially untrusted code. + # Even if it is safe now, it might become malicious in the future. + uses: neuropoly/github-app-token@v1.5.1 + with: + app_id: ${{ secrets.APP_ID }} + private_key: ${{ secrets.APP_KEY }} + - uses: actions/checkout@v3 + with: + token: ${{ steps.generate_token.outputs.token }} - name: Add upstream run: | @@ -96,6 +153,7 @@ jobs: git rebase main - name: Construct latest version with git-annex on top + id: fork run: | # for the latest tag vX.Y.Z, construct tag vX.Y.Z-git-annex. # Only construct the *latest* release to reduce the risk of conflicts From 005fce19b6a96eb962f2c80ebe0e2db9c5f5e7fa Mon Sep 17 00:00:00 2001 From: kousu Date: Sat, 9 Apr 2022 02:32:53 -0400 Subject: [PATCH 05/13] Actually zero the release notes --- .github/workflows/release.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9fea838a4ef2..ecdcbf8a2345 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -54,7 +54,8 @@ jobs: uses: softprops/action-gh-release@v1 with: # We don't have .drone.yml's pretty changelog - # generator, so just zero-out the release notes - body: '' + # generator, so just empty the release notes + # ('' doesn't work, it needs at least one character) + body: '.' files: 'dist/release/*' fail_on_unmatched_files: true From 41ff3af4a15dfdfb3d399fea2afad14f33877716 Mon Sep 17 00:00:00 2001 From: kousu Date: Fri, 24 Jun 2022 19:03:04 -0400 Subject: [PATCH 06/13] --unshallow the upstream tags Fixes this failure: https://github.com/neuropoly/gitea/runs/7042433953?check_suite_focus=true Also, expand the comments, now that I understand this better. --- .github/workflows/sync-upstream.yml | 59 +++++++++++++++++++++-------- 1 file changed, 44 insertions(+), 15 deletions(-) diff --git a/.github/workflows/sync-upstream.yml b/.github/workflows/sync-upstream.yml index 42949a051975..ffb0b15880e1 100644 --- a/.github/workflows/sync-upstream.yml +++ b/.github/workflows/sync-upstream.yml @@ -112,7 +112,6 @@ jobs: PARENT=$(curl -s https://api.github.com/repos/${{github.repository}} | jq -r '.parent.clone_url // empty') git remote add upstream "$PARENT" - - name: Fetch current origin run: | set -ex @@ -173,11 +172,6 @@ jobs: PARENT_TAGS=$(curl -s "$PARENT_API"| jq -r '.tags_url // empty') RELEASE=$(curl -s "$PARENT_TAGS" | jq -r 'map(.name | select(test("dev") | not)) | first // empty') - # https://stackoverflow.com/questions/26617862/git-shallow-fetch-of-a-new-tag - # --shallow-exclude=HEAD seems to get enough history that we can push back - # all necessary tags, but I'm a bit unclear if that's correct - git fetch --shallow-exclude=HEAD upstream tag "$RELEASE" - # But if we decide to just unshallow the entire repo from the start, # then you can use this instead: #RELEASE="$(git tag -l --sort=-v:refname | egrep -v 'git-annex$' | head -n 1)" @@ -186,20 +180,55 @@ jobs: echo "$RELEASE-git-annex already published :tada:" else set -x + # BEWARE: the releases are tagged off of *release* branches: + # https://github.com/go-gitea/gitea/tree/release/v1.18 + # https://github.com/go-gitea/gitea/tree/release/v1.17 + # + # These were branched from 'main' at some point, and since + # have been added to with backport patches. For example, + # https://github.com/go-gitea/gitea/pull/19567 is the backport + # into v1.16 of https://github.com/go-gitea/gitea/pull/19566. + # In that case, the two patches were identical, but it's possible + # a merge conflict could force edits to the backport. + # + # To fit into their scheme, we would have to manuually maintain + # our own git-annex branch (based on upstream/main), another + # release/v1.18-git-annex (based on upstream/release/v1.18), and a + # release/v1.17-git-annex (based on upstream/release/v1.17), etc. + # + # That seems like a lot of work, so we're taking a shortcut: + # just cherry-pick main..git-annex on top of their releases + # and hope for the best. + # + # The only trouble I'm aware of with this is that a merge conflict + # will show up against upstream/main but might not exist against + # v1.19.1 or whatever the latest tag is; so fixing it for main will + # cause a different merge conflict with the release. + # + # But we only need each release to get tagged and published once + # (see: the if statement above), so the best way to handle that case + # is to manually fix the conflict against upstream/main, and at the + # same time manually tag the latest release without the additional fixes. + # This is basically the same work we would do anyway if we were manually + # backporting every update to a separate release branch anyway, but + # only needs to be investigated when the automated build fails. + # + # tl;dr: If this step fails due to merge conflicts, you should + # manually fix them and then manually create the tag, + # sidestepping this + + # Because the tags don't share close history with 'main', GitHub would reject us + # pushing them as dangling commit histories. So this part has to be --unshallow. + # It takes longer but oh well, GitHub can afford it. + git fetch --unshallow upstream tag "$RELEASE" + git checkout -q "$RELEASE" + git cherry-pick main.."${{ github.ref_name }}" # Make sure release.yml is in the tag, so it triggers a build git cherry-pick main..git-annex - git tag "$RELEASE"-git-annex - # If this step fails due to merge conflicts, - # it's probably because the most recent merge conflict - # occurred somewhere after the most recent release but - # before the current upstream/main. - # - # You should just manually create the tag, and fix the merge conflicts. - # This won't try to overwrite a pre-existing tag. + git tag "$RELEASE"-git-annex fi - - name: Upload everything back to Github run: | git push -f --all From 5b803f6359e694fadc46d873a325a9a5d68cfdb6 Mon Sep 17 00:00:00 2001 From: Nick Date: Fri, 19 Aug 2022 01:40:59 -0400 Subject: [PATCH 07/13] Allow blank issues For us, as a fork working on our own bubble, these are useful --- .github/ISSUE_TEMPLATE/config.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index e769873f470c..93f2c244fc7d 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -1,4 +1,3 @@ -blank_issues_enabled: false contact_links: - name: Security Concern url: https://tinyurl.com/security-gitea From b6ac4a0355a9ffea34d2284366caa92e6bf6a2b0 Mon Sep 17 00:00:00 2001 From: Nick Guenther Date: Tue, 18 Oct 2022 17:24:03 -0400 Subject: [PATCH 08/13] Bump to github-app-token 1.7.0 Fixes the warning from https://github.blog/changelog/2022-10-11-github-actions-deprecating-save-state-and-set-output-commands/ --- .github/workflows/sync-upstream.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sync-upstream.yml b/.github/workflows/sync-upstream.yml index ffb0b15880e1..6d929fee0211 100644 --- a/.github/workflows/sync-upstream.yml +++ b/.github/workflows/sync-upstream.yml @@ -96,7 +96,7 @@ jobs: # Notice too: we've **forked** a copy of tibdex/github-app-token, # to avoid passing our tokens through potentially untrusted code. # Even if it is safe now, it might become malicious in the future. - uses: neuropoly/github-app-token@v1.5.1 + uses: neuropoly/github-app-token@v1.7.0 with: app_id: ${{ secrets.APP_ID }} private_key: ${{ secrets.APP_KEY }} From 1d362d5e4862e176baa06aba1706cba9462e67cc Mon Sep 17 00:00:00 2001 From: Nick Guenther Date: Mon, 23 Jan 2023 15:44:52 -0500 Subject: [PATCH 09/13] Bump actions. This is re https://github.blog/changelog/2022-09-22-github-actions-all-actions-will-begin-running-on-node16-instead-of-node12/ --- .github/workflows/release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ecdcbf8a2345..08ef684e8736 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -21,10 +21,10 @@ jobs: run: | sudo apt-get update && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y make git xz-utils # per README.md, building needs Go 1.17 and Node LTS - - uses: actions/setup-go@v2 + - uses: actions/setup-go@v3 with: go-version: '^1.17' # The Go version to download (if necessary) and use. - - uses: actions/setup-node@v2 + - uses: actions/setup-node@v3 with: node-version: 'lts/*' From 2155044da8ff9a83d09be68f7056953282d157e5 Mon Sep 17 00:00:00 2001 From: Nick Guenther Date: Sun, 5 Feb 2023 19:31:52 -0500 Subject: [PATCH 10/13] Bump to go 1.19 --- .github/workflows/release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 08ef684e8736..62a70924c1ae 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -20,10 +20,10 @@ jobs: - name: Install toolchain run: | sudo apt-get update && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y make git xz-utils - # per README.md, building needs Go 1.17 and Node LTS + # per README.md, building needs Go 1.19 and Node LTS - uses: actions/setup-go@v3 with: - go-version: '^1.17' # The Go version to download (if necessary) and use. + go-version: '^1.19' # The Go version to download (if necessary) and use. - uses: actions/setup-node@v3 with: node-version: 'lts/*' From f99476e79f39fbba499b28d3592aff97383cb59c Mon Sep 17 00:00:00 2001 From: Nick Guenther Date: Fri, 3 Mar 2023 18:47:24 -0500 Subject: [PATCH 11/13] Enable PAM builds Gitea's official builds have this disabled. I'm not sure why. But I would like to use it at least on some neurogitea systems, so turn it on. See https://docs.gitea.io/en-us/authentication/#pam-pluggable-authentication-module --- .github/workflows/release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 62a70924c1ae..60344cf60c1d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -19,7 +19,7 @@ jobs: - name: Install toolchain run: | - sudo apt-get update && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y make git xz-utils + sudo apt-get update && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y make git xz-utils libpam0g-dev # per README.md, building needs Go 1.19 and Node LTS - uses: actions/setup-go@v3 with: @@ -35,7 +35,7 @@ jobs: # # Instead, just do native Linux compilation then pretend we did 'make release'. run: | - TAGS="bindata sqlite sqlite_unlock_notify" make build + TAGS="bindata sqlite sqlite_unlock_notify pam" make build mkdir -p dist/release cp -p gitea dist/release/gitea-$(git describe --tags --always)-linux-amd64 From fb21b9232dd7963deef2d5a8c41524bc9ce66e5d Mon Sep 17 00:00:00 2001 From: Nick Guenther Date: Thu, 8 Jun 2023 14:25:27 -0400 Subject: [PATCH 12/13] lint --- .github/workflows/release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 60344cf60c1d..320c913b6e80 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -37,7 +37,7 @@ jobs: run: | TAGS="bindata sqlite sqlite_unlock_notify pam" make build mkdir -p dist/release - cp -p gitea dist/release/gitea-$(git describe --tags --always)-linux-amd64 + cp -p gitea dist/release/gitea-"$(git describe --tags --always)"-linux-amd64 - name: Compress Release Assets run: | @@ -47,7 +47,7 @@ jobs: # each release asset in the official build process gets a separate .sha256 file # which means we need a loop to emulate it run: | - (cd dist/release; for asset in *; do sha256sum $asset > $asset.sha256; done) + (cd dist/release; for asset in *; do sha256sum "$asset" > "$asset".sha256; done) - name: Upload Release # this Action creates the release if not yet created From e50657ec2f95577f2ad0ad4d7fff9b4f9f47a866 Mon Sep 17 00:00:00 2001 From: Nick Guenther Date: Wed, 20 Sep 2023 16:27:16 -0400 Subject: [PATCH 13/13] YAML lint --- .github/workflows/sync-upstream.yml | 398 ++++++++++++++-------------- 1 file changed, 199 insertions(+), 199 deletions(-) diff --git a/.github/workflows/sync-upstream.yml b/.github/workflows/sync-upstream.yml index 6d929fee0211..5649e3efa0dc 100644 --- a/.github/workflows/sync-upstream.yml +++ b/.github/workflows/sync-upstream.yml @@ -21,7 +21,7 @@ on: workflow_dispatch: schedule: # 08:00 Montreal time, every day - - cron: '0 13 * * *' + - cron: '0 13 * * *' jobs: sync_upstream: @@ -29,207 +29,207 @@ jobs: runs-on: ubuntu-latest steps: - #- name: debug - github object - # run: | - # echo '${{ tojson(github) }}' - - - name: Git Identity - run: | - set -ex - git config --global user.name "Actions Bot" - # or 41898282+github-actions[bot]@users.noreply.github.com ? - git config --global user.email action@github.com - - #- name: Git config - # run: | - # set -ex - # # disambiguates 'git checkout' so it always uses this repo - # #git config --global checkout.defaultRemote origin - - - id: generate_token - # The default token provided doesn't have enough rights - # for 'git push --tags' to trigger the build in release.yml: - # - # > When you use the repository's GITHUB_TOKEN to perform tasks, events - # > triggered by the GITHUB_TOKEN will not create a new workflow run. - # > This prevents you from accidentally creating recursive workflow runs. - # - # ref: https://docs.github.com/en/actions/security-guides/automatic-token-authentication#using-the-github_token-in-a-workflow - # - # But we're not making a recursive workflow, and really do want to - # trigger the release.yml workflow. - # - # https://github.com/tibdex/github-app-token works around this by - # trading OAuth credentials (created at the Organization level) for - # a token credential (which can't be created except at the user level) - # - # Two alternate solutions are: - # - # 1. Provide a personal token (https://github.com/settings/tokens) but then - # whoever does exposes their account to everyone in the organization, - # and the organization is exposed to DoS if the person ever leaves. - # 2. Use workflow_call: (https://docs.github.com/en/actions/using-workflows/reusing-workflows) - # but this is a substantial amount of intricate code - # for something that should be simple. - # - # If you need to reconfigure this, know that you need to: - # - # 1. Create an OAuth credential at https://github.com/organizations/neuropoly/settings/apps - # a. Set 'Name' = 'gitea-sync' - # b. Set 'Homepage URL' = 'https://github.com/neuropoly/github-app-token' - # c. Uncheck 'Webhook' - # d. Set 'Repository permissions / Contents' = 'Access: Read & write'. - # e. Set 'Where can this GitHub App be installed' = 'Only on this account' - # 2. Click 'Generate Private Key'; it will download a .pem file to your computer. - # 3. Store the credential in the repo at https://github.com/neuropoly/gitea/settings/secrets/actions - # a. Set 'APP_ID' = the app ID displayed on https://github.com/organizations/neuropoly/settings/apps/gitea-sync - # b. Set 'APP_KEY' = the contents of the .pem file it downloaded. - # c. Now you can throw away the .pem file. - # 4. Install the app: - # a. Go to https://github.com/organizations/neuropoly/settings/apps/gitea-sync/installations - # b. Click 'Install' - # c. Pick this repo - # d. Click 'Install' for real this time - # - # ref: https://github.com/peter-evans/create-pull-request/blob/main/docs/concepts-guidelines.md#authenticating-with-github-app-generated-tokens - # - # Notice too: we've **forked** a copy of tibdex/github-app-token, - # to avoid passing our tokens through potentially untrusted code. - # Even if it is safe now, it might become malicious in the future. - uses: neuropoly/github-app-token@v1.7.0 - with: - app_id: ${{ secrets.APP_ID }} - private_key: ${{ secrets.APP_KEY }} - - - uses: actions/checkout@v3 - with: - token: ${{ steps.generate_token.outputs.token }} - - - name: Add upstream - run: | - set -ex - - PARENT=$(curl -s https://api.github.com/repos/${{github.repository}} | jq -r '.parent.clone_url // empty') - git remote add upstream "$PARENT" - - - name: Fetch current origin - run: | - set -ex - # Because actions/checkout does a lazy, shallow checkout - # we need to use --shallow-since to make sure there's - # enough common history that git can tell how the two - # branches relate. + #- name: debug - github object + # run: | + # echo '${{ tojson(github) }}' + + - name: Git Identity + run: | + set -ex + git config --global user.name "Actions Bot" + # or 41898282+github-actions[bot]@users.noreply.github.com ? + git config --global user.email action@github.com + + #- name: Git config + # run: | + # set -ex + # # disambiguates 'git checkout' so it always uses this repo + # #git config --global checkout.defaultRemote origin + + - id: generate_token + # The default token provided doesn't have enough rights + # for 'git push --tags' to trigger the build in release.yml: # - # We *could* do a full checkout by setting depth: 0 above, - # but this is faster, especially on a large repo like this one. + # > When you use the repository's GITHUB_TOKEN to perform tasks, events + # > triggered by the GITHUB_TOKEN will not create a new workflow run. + # > This prevents you from accidentally creating recursive workflow runs. # - # Since this runs daily, 1 week should be plenty. - git fetch '--shallow-since=1 week' origin main "${{ github.ref_name }}" git-annex - git fetch '--shallow-since=1 week' upstream main - - - name: Sync main - # force main to be identical to upstream - # This throws away any commits to our local main - # so don't commit anything to that branch. - run: | - set -ex - git checkout -B main upstream/main - - - name: Sync ${{ github.ref_name }} - run: | - set -ex - git checkout "${{ github.ref_name }}" - git rebase main - - - name: Rebase git-annex, the feature branch - # This is the meatiest part of this script: rebase git-annex on top of upstream. - # Occasionally this step will fail -- when there's a merge conflict with upstream. - # In that case, you will get an email about it, and you should run these steps - # manually, and fix the merge conflicts that way. - run: | - set -ex - git checkout git-annex - git rebase main - - - name: Construct latest version with git-annex on top - id: fork - run: | - # for the latest tag vX.Y.Z, construct tag vX.Y.Z-git-annex. - # Only construct the *latest* release to reduce the risk of conflicts - # (we have to ask 'git tag' instead of the more elegant method of syncing tags - # and using Github Actions' `on: push: tags: ...` because those upstream tags - # *don't contain this workflow*, so there would be no way to trigger this) + # ref: https://docs.github.com/en/actions/security-guides/automatic-token-authentication#using-the-github_token-in-a-workflow # - # This will trigger release.yml to build and publish the latest version, too - set -e - - # git fetch is supposed to get any tags corresponding to commits it downloads, - # but this behaviour is ignored when combined with --shallow, and there doesn't - # seem to be any other way to get a list of tags without downloading all of them, - # which effectively does --unshallow. But the GitHub API provides a shortcut, and - # using this saves about 30s over downloading the unshallow repo: - PARENT_API=$(curl -s https://api.github.com/repos/${{github.repository}} | jq -r '.parent.url // empty') - PARENT_TAGS=$(curl -s "$PARENT_API"| jq -r '.tags_url // empty') - RELEASE=$(curl -s "$PARENT_TAGS" | jq -r 'map(.name | select(test("dev") | not)) | first // empty') - - # But if we decide to just unshallow the entire repo from the start, - # then you can use this instead: - #RELEASE="$(git tag -l --sort=-v:refname | egrep -v 'git-annex$' | head -n 1)" - - if git fetch -q --depth 1 origin tag "$RELEASE"-git-annex 2>/dev/null; then - echo "$RELEASE-git-annex already published :tada:" - else - set -x - # BEWARE: the releases are tagged off of *release* branches: - # https://github.com/go-gitea/gitea/tree/release/v1.18 - # https://github.com/go-gitea/gitea/tree/release/v1.17 - # - # These were branched from 'main' at some point, and since - # have been added to with backport patches. For example, - # https://github.com/go-gitea/gitea/pull/19567 is the backport - # into v1.16 of https://github.com/go-gitea/gitea/pull/19566. - # In that case, the two patches were identical, but it's possible - # a merge conflict could force edits to the backport. - # - # To fit into their scheme, we would have to manuually maintain - # our own git-annex branch (based on upstream/main), another - # release/v1.18-git-annex (based on upstream/release/v1.18), and a - # release/v1.17-git-annex (based on upstream/release/v1.17), etc. - # - # That seems like a lot of work, so we're taking a shortcut: - # just cherry-pick main..git-annex on top of their releases - # and hope for the best. + # But we're not making a recursive workflow, and really do want to + # trigger the release.yml workflow. + # + # https://github.com/tibdex/github-app-token works around this by + # trading OAuth credentials (created at the Organization level) for + # a token credential (which can't be created except at the user level) + # + # Two alternate solutions are: + # + # 1. Provide a personal token (https://github.com/settings/tokens) but then + # whoever does exposes their account to everyone in the organization, + # and the organization is exposed to DoS if the person ever leaves. + # 2. Use workflow_call: (https://docs.github.com/en/actions/using-workflows/reusing-workflows) + # but this is a substantial amount of intricate code + # for something that should be simple. + # + # If you need to reconfigure this, know that you need to: + # + # 1. Create an OAuth credential at https://github.com/organizations/neuropoly/settings/apps + # a. Set 'Name' = 'gitea-sync' + # b. Set 'Homepage URL' = 'https://github.com/neuropoly/github-app-token' + # c. Uncheck 'Webhook' + # d. Set 'Repository permissions / Contents' = 'Access: Read & write'. + # e. Set 'Where can this GitHub App be installed' = 'Only on this account' + # 2. Click 'Generate Private Key'; it will download a .pem file to your computer. + # 3. Store the credential in the repo at https://github.com/neuropoly/gitea/settings/secrets/actions + # a. Set 'APP_ID' = the app ID displayed on https://github.com/organizations/neuropoly/settings/apps/gitea-sync + # b. Set 'APP_KEY' = the contents of the .pem file it downloaded. + # c. Now you can throw away the .pem file. + # 4. Install the app: + # a. Go to https://github.com/organizations/neuropoly/settings/apps/gitea-sync/installations + # b. Click 'Install' + # c. Pick this repo + # d. Click 'Install' for real this time + # + # ref: https://github.com/peter-evans/create-pull-request/blob/main/docs/concepts-guidelines.md#authenticating-with-github-app-generated-tokens + # + # Notice too: we've **forked** a copy of tibdex/github-app-token, + # to avoid passing our tokens through potentially untrusted code. + # Even if it is safe now, it might become malicious in the future. + uses: neuropoly/github-app-token@v1.7.0 + with: + app_id: ${{ secrets.APP_ID }} + private_key: ${{ secrets.APP_KEY }} + + - uses: actions/checkout@v3 + with: + token: ${{ steps.generate_token.outputs.token }} + + - name: Add upstream + run: | + set -ex + + PARENT=$(curl -s https://api.github.com/repos/${{github.repository}} | jq -r '.parent.clone_url // empty') + git remote add upstream "$PARENT" + + - name: Fetch current origin + run: | + set -ex + # Because actions/checkout does a lazy, shallow checkout + # we need to use --shallow-since to make sure there's + # enough common history that git can tell how the two + # branches relate. # - # The only trouble I'm aware of with this is that a merge conflict - # will show up against upstream/main but might not exist against - # v1.19.1 or whatever the latest tag is; so fixing it for main will - # cause a different merge conflict with the release. + # We *could* do a full checkout by setting depth: 0 above, + # but this is faster, especially on a large repo like this one. # - # But we only need each release to get tagged and published once - # (see: the if statement above), so the best way to handle that case - # is to manually fix the conflict against upstream/main, and at the - # same time manually tag the latest release without the additional fixes. - # This is basically the same work we would do anyway if we were manually - # backporting every update to a separate release branch anyway, but - # only needs to be investigated when the automated build fails. + # Since this runs daily, 1 week should be plenty. + git fetch '--shallow-since=1 week' origin main "${{ github.ref_name }}" git-annex + git fetch '--shallow-since=1 week' upstream main + + - name: Sync main + # force main to be identical to upstream + # This throws away any commits to our local main + # so don't commit anything to that branch. + run: | + set -ex + git checkout -B main upstream/main + + - name: Sync ${{ github.ref_name }} + run: | + set -ex + git checkout "${{ github.ref_name }}" + git rebase main + + - name: Rebase git-annex, the feature branch + # This is the meatiest part of this script: rebase git-annex on top of upstream. + # Occasionally this step will fail -- when there's a merge conflict with upstream. + # In that case, you will get an email about it, and you should run these steps + # manually, and fix the merge conflicts that way. + run: | + set -ex + git checkout git-annex + git rebase main + + - name: Construct latest version with git-annex on top + id: fork + run: | + # for the latest tag vX.Y.Z, construct tag vX.Y.Z-git-annex. + # Only construct the *latest* release to reduce the risk of conflicts + # (we have to ask 'git tag' instead of the more elegant method of syncing tags + # and using Github Actions' `on: push: tags: ...` because those upstream tags + # *don't contain this workflow*, so there would be no way to trigger this) # - # tl;dr: If this step fails due to merge conflicts, you should - # manually fix them and then manually create the tag, - # sidestepping this - - # Because the tags don't share close history with 'main', GitHub would reject us - # pushing them as dangling commit histories. So this part has to be --unshallow. - # It takes longer but oh well, GitHub can afford it. - git fetch --unshallow upstream tag "$RELEASE" - - git checkout -q "$RELEASE" - - git cherry-pick main.."${{ github.ref_name }}" # Make sure release.yml is in the tag, so it triggers a build - git cherry-pick main..git-annex - - git tag "$RELEASE"-git-annex - fi - - name: Upload everything back to Github - run: | - git push -f --all - git push -f --tags + # This will trigger release.yml to build and publish the latest version, too + set -e + + # git fetch is supposed to get any tags corresponding to commits it downloads, + # but this behaviour is ignored when combined with --shallow, and there doesn't + # seem to be any other way to get a list of tags without downloading all of them, + # which effectively does --unshallow. But the GitHub API provides a shortcut, and + # using this saves about 30s over downloading the unshallow repo: + PARENT_API=$(curl -s https://api.github.com/repos/${{github.repository}} | jq -r '.parent.url // empty') + PARENT_TAGS=$(curl -s "$PARENT_API"| jq -r '.tags_url // empty') + RELEASE=$(curl -s "$PARENT_TAGS" | jq -r 'map(.name | select(test("dev") | not)) | first // empty') + + # But if we decide to just unshallow the entire repo from the start, + # then you can use this instead: + #RELEASE="$(git tag -l --sort=-v:refname | egrep -v 'git-annex$' | head -n 1)" + + if git fetch -q --depth 1 origin tag "$RELEASE"-git-annex 2>/dev/null; then + echo "$RELEASE-git-annex already published :tada:" + else + set -x + # BEWARE: the releases are tagged off of *release* branches: + # https://github.com/go-gitea/gitea/tree/release/v1.18 + # https://github.com/go-gitea/gitea/tree/release/v1.17 + # + # These were branched from 'main' at some point, and since + # have been added to with backport patches. For example, + # https://github.com/go-gitea/gitea/pull/19567 is the backport + # into v1.16 of https://github.com/go-gitea/gitea/pull/19566. + # In that case, the two patches were identical, but it's possible + # a merge conflict could force edits to the backport. + # + # To fit into their scheme, we would have to manuually maintain + # our own git-annex branch (based on upstream/main), another + # release/v1.18-git-annex (based on upstream/release/v1.18), and a + # release/v1.17-git-annex (based on upstream/release/v1.17), etc. + # + # That seems like a lot of work, so we're taking a shortcut: + # just cherry-pick main..git-annex on top of their releases + # and hope for the best. + # + # The only trouble I'm aware of with this is that a merge conflict + # will show up against upstream/main but might not exist against + # v1.19.1 or whatever the latest tag is; so fixing it for main will + # cause a different merge conflict with the release. + # + # But we only need each release to get tagged and published once + # (see: the if statement above), so the best way to handle that case + # is to manually fix the conflict against upstream/main, and at the + # same time manually tag the latest release without the additional fixes. + # This is basically the same work we would do anyway if we were manually + # backporting every update to a separate release branch anyway, but + # only needs to be investigated when the automated build fails. + # + # tl;dr: If this step fails due to merge conflicts, you should + # manually fix them and then manually create the tag, + # sidestepping this + + # Because the tags don't share close history with 'main', GitHub would reject us + # pushing them as dangling commit histories. So this part has to be --unshallow. + # It takes longer but oh well, GitHub can afford it. + git fetch --unshallow upstream tag "$RELEASE" + + git checkout -q "$RELEASE" + + git cherry-pick main.."${{ github.ref_name }}" # Make sure release.yml is in the tag, so it triggers a build + git cherry-pick main..git-annex + + git tag "$RELEASE"-git-annex + fi + - name: Upload everything back to Github + run: | + git push -f --all + git push -f --tags