[CI] add automatic draft release generation (#1022)

* add test script

* first try

remove compiler

temp change to haerdibg

fix parsing

fixes

fix change.md

Update README.md

fixes (#3)

remove pallet changes

fix change.md

fix printing changes

remove docker image for now

fix comments

add global challenge level

set min level

fix gloabl challenge level

update challenge sentence

Update README.md (#4)

fix variables

fix meta.C -> meta.E

fix misc - include all

Update README.md (#5)

fix prios

Update README.md (#6)

test

modify readme

maybe?

test nr 2

test

again try

fix

* change owner back to integritee

* readd master branch

* remove extra .yml file

* remove outdated scripts

* rename priority to challenge

* fix readme.md

* remove docker file

.github/workflows/publish-draft-release.yml

@@ -0,0 +1,69 @@
+name: Release - Publish draft
+
+on:
+  push:
+    tags:
+      # Catches v1.2.3 and v1.2.3-rc1
+      - v[0-9]+.[0-9]+.[0-9]+*
+
+jobs:
+  publish-draft-release:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          path: worker
+
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: 3.0.0
+
+      - name: Download srtool json output
+        uses: actions/download-artifact@v3
+
+      - name: Prepare tooling
+        run: |
+          cd worker/scripts/changelog
+          gem install bundler changelogerator:0.9.1
+          bundle install
+          changelogerator --help
+          URL=https://github.com/chevdor/tera-cli/releases/download/v0.2.1/tera-cli_linux_amd64.deb
+          wget $URL -O tera.deb
+          sudo dpkg -i tera.deb
+          tera --version
+
+      - name: Generate release notes
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          DEBUG: 1
+          PRE_RELEASE: ${{ github.event.inputs.pre_release }}
+        run: |
+          find ${{env.GITHUB_WORKSPACE}} -type f -name "*_srtool_output.json"
+
+          cd worker/scripts/changelog
+
+          ./bin/changelog ${GITHUB_REF}
+          ls -al release-notes.md
+          ls -al context.json
+
+      - name: Archive artifact context.json
+        uses: actions/upload-artifact@v3
+        with:
+          name: release-notes-context
+          path: |
+            worker/scripts/changelog/context.json
+            **/*_srtool_output.json
+
+      - name: Create draft release
+        id: create-release
+        uses: actions/create-release@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          tag_name: ${{ github.ref }}
+          release_name: SDK ${{ github.ref }}
+          body_path: ./worker/scripts/changelog/release-notes.md
+          draft: true

scripts/changelog/.gitignore

@@ -0,0 +1,4 @@
+changelog.md
+*.json
+release*.md
+.env

scripts/changelog/Gemfile

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
+
+gem 'octokit', '~> 4'
+
+gem 'git_diff_parser', '~> 3'
+
+gem 'toml', '~> 0.3.0'
+
+gem 'rake', group: :dev
+
+gem 'optparse', '~> 0.1.1'
+
+gem 'logger', '~> 1.4'
+
+gem 'test-unit', group: :dev
+
+gem 'rubocop', group: :dev, require: false

scripts/changelog/Gemfile.lock

@@ -0,0 +1,79 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    addressable (2.8.0)
+      public_suffix (>= 2.0.2, < 5.0)
+    ast (2.4.2)
+    faraday (1.8.0)
+      faraday-em_http (~> 1.0)
+      faraday-em_synchrony (~> 1.0)
+      faraday-excon (~> 1.1)
+      faraday-httpclient (~> 1.0.1)
+      faraday-net_http (~> 1.0)
+      faraday-net_http_persistent (~> 1.1)
+      faraday-patron (~> 1.0)
+      faraday-rack (~> 1.0)
+      multipart-post (>= 1.2, < 3)
+      ruby2_keywords (>= 0.0.4)
+    faraday-em_http (1.0.0)
+    faraday-em_synchrony (1.0.0)
+    faraday-excon (1.1.0)
+    faraday-httpclient (1.0.1)
+    faraday-net_http (1.0.1)
+    faraday-net_http_persistent (1.2.0)
+    faraday-patron (1.0.0)
+    faraday-rack (1.0.0)
+    git_diff_parser (3.2.0)
+    logger (1.4.4)
+    multipart-post (2.1.1)
+    octokit (4.21.0)
+      faraday (>= 0.9)
+      sawyer (~> 0.8.0, >= 0.5.3)
+    optparse (0.1.1)
+    parallel (1.21.0)
+    parser (3.0.2.0)
+      ast (~> 2.4.1)
+    parslet (2.0.0)
+    power_assert (2.0.1)
+    public_suffix (4.0.6)
+    rainbow (3.0.0)
+    rake (13.0.6)
+    regexp_parser (2.1.1)
+    rexml (3.2.5)
+    rubocop (1.23.0)
+      parallel (~> 1.10)
+      parser (>= 3.0.0.0)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.8, < 3.0)
+      rexml
+      rubocop-ast (>= 1.12.0, < 2.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 1.4.0, < 3.0)
+    rubocop-ast (1.13.0)
+      parser (>= 3.0.1.1)
+    ruby-progressbar (1.11.0)
+    ruby2_keywords (0.0.5)
+    sawyer (0.8.2)
+      addressable (>= 2.3.5)
+      faraday (> 0.8, < 2.0)
+    test-unit (3.5.1)
+      power_assert
+    toml (0.3.0)
+      parslet (>= 1.8.0, < 3.0.0)
+    unicode-display_width (2.1.0)
+
+PLATFORMS
+  x86_64-darwin-20
+
+DEPENDENCIES
+  git_diff_parser (~> 3)
+  logger (~> 1.4)
+  octokit (~> 4)
+  optparse (~> 0.1.1)
+  rake
+  rubocop
+  test-unit
+  toml (~> 0.3.0)
+
+BUNDLED WITH
+   2.2.22

scripts/changelog/README.md

@@ -0,0 +1,3 @@
+## License
+
+Everything in this folder is GPL 3.0 licensed. The original has been authored by parity and was taken from here: https://github.com/paritytech/polkadot/tree/master/scripts/ci/changelog. 

scripts/changelog/bin/changelog

@@ -0,0 +1,84 @@
+#!/usr/bin/env ruby
+
+# frozen_string_literal: true
+
+# call for instance as:
+# ./bin/changelog <to> [<from>] [<output_file>]
+# for instance, for the release notes of v1.2.3:
+# ./bin/changelog v1.2.3
+# or
+# ./bin/changelog v1.2.3 v1.2.2
+#
+# You may set the ENV NO_CACHE to force fetching from Github
+# You should also ensure you set the ENV: GITHUB_TOKEN
+
+require_relative '../lib/changelog'
+require 'logger'
+
+logger = Logger.new($stdout)
+logger.level = Logger::DEBUG
+logger.debug('Starting')
+
+owner = 'integritee-network'
+repo = 'worker'
+
+gh_worker = SubRef.new(format('%<owner>s/%<repo>s', { owner: owner, repo: repo }))
+last_release_ref = gh_worker.get_last_ref()
+
+worker_ref2 = ARGV[0] || 'HEAD'
+worker_ref1 = ARGV[1] || last_release_ref
+
+output = ARGV[2] || 'release-notes.md'
+
+ENV['REF1'] = worker_ref1
+ENV['REF2'] = worker_ref2
+
+pallets_ref1 = gh_worker.get_dependency_reference(worker_ref1, 'pallet-teerex')
+pallets_ref2 = gh_worker.get_dependency_reference(worker_ref2, 'pallet-teerex')
+
+logger.debug("Worker from:   #{worker_ref1}")
+logger.debug("Worker to:     #{worker_ref2}")
+
+logger.debug("Pallets from:  #{pallets_ref1}")
+logger.debug("Pallets to:    #{pallets_ref2}")
+
+pallets_data = 'pallets.json'
+worker_data = 'worker.json'
+
+logger.debug("Using PALLETS: #{pallets_data}")
+logger.debug("Using WORKER:  #{worker_data}")
+
+logger.warn('NO_CACHE set') if ENV['NO_CACHE']
+
+if ENV['NO_CACHE'] || !File.file?(worker_data)
+  logger.debug(format('Fetching data for Worker into %s', worker_data))
+  cmd = format('changelogerator %<owner>s/%<repo>s -f %<from>s -t %<to>s > %<output>s',
+               { owner: owner, repo: 'worker', from: worker_ref1, to: worker_ref2, output: worker_data })
+  system(cmd)
+else
+  logger.debug("Re-using:#{worker_data}")
+end
+
+if ENV['NO_CACHE'] || !File.file?(pallets_data)
+  logger.debug(format('Fetching data for Pallets into %s', pallets_data))
+  cmd = format('changelogerator %<owner>s/%<repo>s -f %<from>s -t %<to>s > %<output>s',
+               { owner: owner, repo: 'pallets', from: pallets_ref1, to: pallets_ref2, output: pallets_data })
+  system(cmd)
+else
+  logger.debug("Re-using:#{pallets_data}")
+end
+
+# Here we compose all the pieces together into one
+# single big json file.
+cmd = format('jq \
+    --slurpfile pallets %s \
+    --slurpfile worker %s \
+    -n \'{
+            pallets: $pallets[0],
+            worker: $worker[0],
+    }\' > context.json', pallets_data, worker_data)
+system(cmd)
+
+cmd = format('tera --env --env-key env --include-path templates \
+        --template templates/template.md.tera context.json > %s', output)
+system(cmd)

scripts/changelog/digests/.gitignore

@@ -0,0 +1 @@
+*.json

scripts/changelog/lib/changelog.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+# A Class to find Substrate references
+class SubRef
+  require 'octokit'
+  require 'toml'
+
+  attr_reader :client, :repository
+
+  def initialize(github_repo)
+    @client = Octokit::Client.new(
+      access_token: ENV['GITHUB_TOKEN']
+    )
+    @repository = @client.repository(github_repo)
+  end
+
+  # This function checks the Cargo.lock of a given
+  # Rust project, for a given package, and fetches
+  # the dependency git ref.
+  def get_dependency_reference(ref, package)
+    cargo = TOML::Parser.new(
+      Base64.decode64(
+        @client.contents(
+          @repository.full_name,
+          path: 'Cargo.lock',
+          query: { ref: ref.to_s }
+        ).content
+      )
+    ).parsed
+    cargo['package'].find { |p| p['name'] == package }['source'].split('#').last
+  end
+
+  # Get the git ref of the last release for the repo.
+  # repo is given in the form integritee-network/worker
+  def get_last_ref()
+    'refs/tags/' + @client.latest_release(@repository.full_name).tag_name
+  end
+end

scripts/changelog/templates/_free_notes.md.tera

@@ -0,0 +1,10 @@
+
+{# This file uses the Markdown format with additional templating such as this comment. -#}
+{# Such a comment will not show up in the rendered release notes. -#}
+{# The content of this file (if any) will be inserted at the top of the release notes -#}
+{# and generated for each new release candidate. -#}
+{# Ensure you leave an empty line at both top and bottom of this file. -#}
+
+<!-- Such a comment will be rendered but remain invisible in the rendered markdown -->
+<!-- Edit below this line -->
+<!-- Edit above this line -->

scripts/changelog/templates/challenge_level.md.tera

@@ -0,0 +1,37 @@
+{%- import "change.md.tera" as m_c -%}
+
+{# This macro convert a merge challenge level into readable output #}
+{%- macro challenge_level(e, changes) -%}
+
+{%- if e >= 5 -%}
+    {%- set level = "‼️ Breaking Changes" -%}
+    {%- set text = "This release contains **breaking changes**. Be sure to upgrade the affected interfaces." -%}
+{%- elif e >= 3 -%}
+    {%- set level = "❗️ Attention" -%}
+    {%- set text = "This release contains some non-trivial updates. Be mindful when upgrading." -%}
+{%- else -%}
+    {%- set level = "Trivial" -%}
+    {%- set text = "This release contains relatively small updates." -%}
+{%- endif %}
+
+
+<!-- detected max level is: {{e}} -->
+
+{%- if level %}
+{{level}}: {{text}}
+
+{% if e >= 3 %}
+The changes motivating this challenge level are:
+{% for pr in changes | sort(attribute="merged_at") -%}
+    {%- if pr.meta.E -%}
+        {%- if pr.meta.E.value == e %}
+- {{ m_c::change(c=pr) }}
+        {%- endif -%}
+    {%- endif -%}
+{%- endfor -%}
+{%- else -%}
+<!-- No relevant Level label as been detected for e={{ e }} -->
+{%- endif -%}
+{%- endif -%}
+
+{%- endmacro level -%}