chore(ci): fix release script

.github/workflows/build-and-deploy.yml

@@ -19,12 +19,11 @@ defaults:
     shell: bash -euxo pipefail {0}
 
 env:
-  GITHUB_REPOSITORY_URL: ${{ github.server_url }}/${{ github.repository }}
   VERBOSE: 1
 
 jobs:
   build-and-deploy-datasets:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-20.04
 
     environment:
       name: ${{ github.ref }}
@@ -37,6 +36,7 @@ jobs:
       AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
       AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
       AWS_DEFAULT_REGION: us-east-2
+      GH_TOKEN: ${{ github.token }}
 
     steps:
       - name: "Checkout code"
@@ -45,31 +45,6 @@ jobs:
           fetch-depth: 0
           submodules: true
 
-      - name: "Authenticate git"
-        run: |
-          export GITHUB_TOKEN="${{ secrets.GH_TOKEN_NEXTSTRAIN_BOT_REPO }}"
-          export BOT_GIT_USER_EMAIL="${{ secrets.BOT_GIT_USER_EMAIL }}"
-          export BOT_GIT_USER_NAME="${{ secrets.BOT_GIT_USER_NAME }}"
-
-          : ${GITHUB_TOKEN?"The env var GITHUB_TOKEN is required"}
-          : ${BOT_GIT_USER_EMAIL?"The env var BOT_GIT_USER_EMAIL is required"}
-          : ${BOT_GIT_USER_NAME?"The env var BOT_GIT_USER_NAME is required"}
-
-          git config --global user.email "${BOT_GIT_USER_EMAIL}"
-          git config --global user.name "${BOT_GIT_USER_NAME}"
-
-          gh auth setup-git >/dev/null
-
-      - name: "Extract branch name"
-        run: |
-          echo "current_branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> $GITHUB_OUTPUT
-        id: branch-name
-
-      - name: "Checkout branch"
-        run: |
-          git fetch --all
-          git switch ${{ steps.branch-name.outputs.current_branch }}
-
       - name: "Install system dependencies"
         run: |
           sudo apt-get install brotli pigz parallel python3 rename --yes -qq >/dev/null
@@ -95,21 +70,24 @@ jobs:
       - name: "Rebuild, commit, push and make a release"
         if: github.ref == 'refs/heads/release'
         run: |
-          ./scripts/rebuild --input-dir 'data/' --output-dir 'data_output/' --release --repo="${GITHUB_REPOSITORY}"
-
-      - name: "Deploy dataset server"
-        if: ${{ endsWith(github.ref, '/master') || endsWith(github.ref, '/staging') || endsWith(github.ref, '/release') }}
-        run: |
-          ./scripts/upload 'data_output/'
+          git config --global user.email "${{ secrets.BOT_GIT_USER_EMAIL }}"
+          git config --global user.name "${{ secrets.BOT_GIT_USER_NAME }}"
 
-      - name: "Upload build artifacts: dataset server"
-        uses: actions/upload-artifact@v3
-        with:
-          name: server
-          path: ./data_output/*
+          ./scripts/rebuild --input-dir 'data/' --output-dir 'data_output/' --release --repo="${GITHUB_REPOSITORY}"
 
-      - name: "Upload build artifacts: zip archives"
-        uses: actions/upload-artifact@v3
-        with:
-          name: zips
-          path: ./data_temp/*
+#      - name: "Deploy dataset server"
+#        if: ${{ endsWith(github.ref, '/master') || endsWith(github.ref, '/staging') || endsWith(github.ref, '/release') }}
+#        run: |
+#          ./scripts/upload 'data_output/'
+#
+#      - name: "Upload build artifacts: dataset server"
+#        uses: actions/upload-artifact@v3
+#        with:
+#          name: server
+#          path: ./data_output/*
+#
+#      - name: "Upload build artifacts: zip archives"
+#        uses: actions/upload-artifact@v3
+#        with:
+#          name: zips
+#          path: ./data_temp/*

scripts/lib/changelog.py

@@ -3,7 +3,8 @@
 
 
 def changelog_prepare(dataset, updated_at, changelog_path):
-  path = dataset["path"]
+  path = dict_get(dataset, ["path"])
+  name = dict_get(dataset, ["attributes", "name"])
   release_notes = changelog_get_unreleased_section(changelog_path)
   if len(release_notes) == 0:
     raise ValueError(
@@ -13,10 +14,10 @@ def changelog_prepare(dataset, updated_at, changelog_path):
   full_changelog = file_read(changelog_path).replace(f"## Unreleased", f"## {updated_at}")
   file_write(full_changelog, changelog_path)
 
-  attr_table = format_dataset_attributes_md_table(dict_get_required(dataset, ["attributes"]))
+  # attr_table = format_dataset_attributes_md_table(dict_get_required(dataset, ["attributes"]))
   release_notes = release_notes.replace(
     "## Unreleased",
-    f"""### {path}\n\n{attr_table}""".strip("\n ")
+    f"""## {name} ({path})""".strip("\n ")
   )
 
   return release_notes
@@ -39,10 +40,8 @@ def changelog_get_unreleased_section(changelog_path: str):
 
 
 def format_dataset_attributes_md_table(attributes):
-  attr_table = f"| {'attribute':20} | {'value':20} | {'value friendly':40} |\n"
-  attr_table += f"| {'-' * 20} | {'-' * 20} | {'-' * 40} |\n"
+  attr_table = f"| {'attribute':20} | {'value':40} |\n"
+  attr_table += f"| {'-' * 20} | {'-' * 40} |\n"
   for attr_name, attr_val in attributes.items():
-    value = attr_val["value"]
-    value_friendly = dict_get(attr_val, ["valueFriendly"]) or ""
-    attr_table += f'| {attr_name:20} | {value:20} | {value_friendly:40} |\n'
+    attr_table += f'| {attr_name:20} | {attr_val:40} |\n'
   return attr_table

scripts/rebuild

@@ -5,6 +5,7 @@ Builds a fresh data repo from source data
 """
 import argparse
 import json
+import time
 from collections import defaultdict
 from copy import deepcopy
 from os import getcwd
@@ -254,10 +255,14 @@ def main():
     commit_hash = commit_changes(args, tag, release_infos)
 
     if args.push:
+      l.info("Pushing commited changes to GitHub")
       git_push()
 
       if args.release:
+        l.info("Releasing to GitHub")
         release_notes = aggregate_release_notes(release_infos)
+        l.info(f"Release notes:\n-------\n{release_notes}\n-------\nEnd of release notes\n")
+        time.sleep(5)
         publish_to_github_releases(args, tag, commit_hash, release_notes)
 
 
@@ -351,26 +356,35 @@ def prepare_dataset_release_infos(args, datasets, collection_dir, tag, updated_a
 
 
 def aggregate_release_notes(release_infos):
-  dataset_names_friendly = format_list(
-    unique([get_dataset_name_friendly(release_info["dataset"]) for release_info in release_infos]),
-    sep="\n", marker="- ", quote=False
-  )
-  release_notes = f"This release contains changes for datasets:\n\n{dataset_names_friendly}\n\n\n"
+  dataset_list = format_dataset_list(release_infos)
+  release_notes = f"This release contains changes for datasets:\n\n{dataset_list}\n\n\n"
   for release_info in release_infos:
     release_notes += f'\n{release_info["release_notes"]}\n\n'
   return release_notes
 
 
+def format_dataset_list(release_infos):
+  entries = [format_dataset_list_entry(release_info) for release_info in release_infos]
+  entries = unique(entries)
+  entries = list(sorted(entries))
+  return format_list(entries, sep="\n", marker="- ", quote=False)
+
+
+def format_dataset_list_entry(release_info):
+  path = dict_get_required(release_info, ['dataset', 'path'])
+  name = get_dataset_name_friendly(release_info['dataset'])
+  return f"{path} ({name})"
+
+
 def commit_changes(args, tag, release_infos):
-  l.info(f"Commiting changes for '{tag}'")
+  l.info(f"Committing changes for '{tag}'")
 
   commit_message = "chore: rebuild [skip ci]"
   if args.release:
-    dataset_names = format_list(
-      unique([get_dataset_name(release_info["dataset"]) for release_info in release_infos]),
-      sep="\n", marker="- ", quote=False
-    )
-    commit_message = f"chore: release '{tag}'\n\nUpdated datasets:\n\n{dataset_names}"
+    dataset_list = format_dataset_list(release_infos)
+    commit_message = f"chore: release '{tag}'\n\nUpdated datasets:\n\n{dataset_list}"
+
+  l.info(f"Commit message:\n--------\n{commit_message}\n--------\nEnd of commit message\n\n")
 
   return git_commit_all(commit_message)
 
@@ -438,12 +452,8 @@ def create_dataset_package(args, dataset, pathogen_json, tag, dataset_dir):
   file_write("User-agent: *\nDisallow: /\n", join(args.output_dir, "robots.txt"))
 
 
-def get_dataset_name(dataset):
-  return dict_get_required(dataset, ["attributes", "name", "value"])
-
-
 def get_dataset_name_friendly(dataset):
-  return dict_get(dataset, ["attributes", "name", "valueFriendly"]) or get_dataset_name(dataset)
+  return dict_get_required(dataset, ["attributes", "name"])
 
 
 if __name__ == '__main__':