From 352019dcf1be49300b5f2b1ab6849c9aac958b51 Mon Sep 17 00:00:00 2001 From: jgraham Date: Tue, 12 Mar 2019 11:56:45 +0000 Subject: [PATCH] Use GitHub actions to build manifest, tag and release. --- .github/main.workflow | 16 ++++ tools/ci/action_manifest_build.sh | 7 ++ tools/ci/tag_master.py | 145 ++++++++++++++++++++++-------- tools/docker/github/Dockerfile | 27 ++++++ 4 files changed, 159 insertions(+), 36 deletions(-) create mode 100644 .github/main.workflow create mode 100755 tools/ci/action_manifest_build.sh create mode 100644 tools/docker/github/Dockerfile diff --git a/.github/main.workflow b/.github/main.workflow new file mode 100644 index 0000000000..b1b61554f9 --- /dev/null +++ b/.github/main.workflow @@ -0,0 +1,16 @@ +workflow "Build & Release Manifest" { + on = "push" + resolves = ["tag-master"] +} + +action "build-manifest" { + uses = "./tools/docker/github" + runs = ["bash", "-c", "tools/ci/action_manifest_build.sh"] +} + +action "tag-master" { + needs = "build-manifest" + uses = "./tools/docker/github" + runs = ["python", "tools/ci/tag_master.py"] + secrets = ["GITHUB_TOKEN"] +} diff --git a/tools/ci/action_manifest_build.sh b/tools/ci/action_manifest_build.sh new file mode 100755 index 0000000000..bd5c2b20f5 --- /dev/null +++ b/tools/ci/action_manifest_build.sh @@ -0,0 +1,7 @@ +#!/bin/bash +mkdir -p ~/meta + +WPT_MANIFEST_FILE=~/meta/MANIFEST.json + +./wpt manifest -p $WPT_MANIFEST_FILE +gzip -f $WPT_MANIFEST_FILE diff --git a/tools/ci/tag_master.py b/tools/ci/tag_master.py index 0ae2801dfe..15c9c48e8a 100644 --- a/tools/ci/tag_master.py +++ b/tools/ci/tag_master.py @@ -1,9 +1,9 @@ -import base64 import json import logging import os import sys -import urllib2 + +import requests here = os.path.abspath(os.path.dirname(__file__)) wpt_root = os.path.abspath(os.path.join(here, os.pardir, os.pardir)) @@ -17,26 +17,54 @@ logger = logging.getLogger(__name__) -def get_pr(owner, repo, sha): - url = ("https://api.github.com/search/issues?q=type:pr+is:merged+repo:%s/%s+sha:%s" % - (owner, repo, sha)) +def request(url, desc, data=None, json_data=None, params=None, headers=None): + github_token = os.environ.get("GITHUB_TOKEN") + default_headers = { + "Authorization": "token %s" % github_token, + "Accept": "application/vnd.github.machine-man-preview+json" + } + + _headers = default_headers + if headers is not None: + _headers.update(headers) + + kwargs = {"params": params, + "headers": _headers} try: - resp = urllib2.urlopen(url) - body = resp.read() + logger.info("Loading URL %s" % url) + if json_data is not None: + method = requests.post + kwargs["json"] = json_data + kwargs["data"] = data + else: + method = requests.get + + resp = method(url, **kwargs) + except Exception as e: - logger.error(e) + logger.error("%s failed:\n%s" % (desc, e)) return None - if resp.code != 200: - logger.error("Got HTTP status %s. Response:" % resp.code) - logger.error(body) + try: + resp.raise_for_status() + except requests.HTTPError: + logger.error("%s failed: Got HTTP status %s. Response:" % + (desc, resp.status_code)) + logger.error(resp.text) return None try: - data = json.loads(body) + return resp.json() except ValueError: - logger.error("Failed to read response as JSON:") - logger.error(body) + logger.error("%s failed: Returned data was not JSON Response:" % + (desc, resp.status_code)) + logger.error(resp.text) + + +def get_pr(owner, repo, sha): + data = request("https://api.github.com/search/issues?q=type:pr+is:merged+repo:%s/%s+sha:%s" % + (owner, repo, sha), "Getting PR") + if data is None: return None items = data["items"] @@ -52,50 +80,95 @@ def get_pr(owner, repo, sha): def tag(owner, repo, sha, tag): - data = json.dumps({"ref": "refs/tags/%s" % tag, - "sha": sha}) - try: - url = "https://api.github.com/repos/%s/%s/git/refs" % (owner, repo) - req = urllib2.Request(url, data=data) + data = {"ref": "refs/tags/%s" % tag, + "sha": sha} + url = "https://api.github.com/repos/%s/%s/git/refs" % (owner, repo) - base64string = base64.b64encode(os.environ["GH_TOKEN"]) - req.add_header("Authorization", "Basic %s" % base64string) + resp_data = request(url, "Tag creation", json_data=data) + if not resp_data: + return False - opener = urllib2.build_opener(urllib2.HTTPSHandler()) + logger.info("Tagged %s as %s" % (sha, tag)) + return True - resp = opener.open(req) - except Exception as e: - logger.error("Tag creation failed:\n%s" % e) + +def create_release(owner, repo, sha, tag, summary, body): + if body: + body = "%s\n%s" % (summary, body) + else: + body = summary + + create_url = "https://api.github.com/repos/%s/%s/releases" % (owner, repo) + create_data = {"tag_name": tag, + "name": tag, + "body": body} + create_data = request(create_url, "Release creation", json_data=create_data) + if not create_data: return False - if resp.code != 201: - logger.error("Got HTTP status %s. Response:" % resp.code) - logger.error(resp.read()) + # Upload URL contains '{?name,label}' at the end which we want to remove + upload_url = create_data["upload_url"].split("{", 1)[0] + + upload_filename = "MANIFEST-%s.json.gz" % sha + params = {"name": upload_filename, + "label": "MANIFEST.json.gz"} + + with open(os.path.expanduser("~/meta/MANIFEST.json.gz"), "rb") as f: + upload_data = f.read() + + logger.info("Uploading %s bytes" % len(upload_data)) + + upload_resp = request(upload_url, "Manifest upload", data=upload_data, params=params, + headers={'Content-Type': 'application/octet-stream'}) + if not upload_resp: return False - logger.info("Tagged %s as %s" % (sha, tag)) return True -def main(): - owner, repo = os.environ["TRAVIS_REPO_SLUG"].split("/", 1) - if os.environ["TRAVIS_PULL_REQUEST"] != "false": +def should_run_action(): + with open(os.environ["GITHUB_EVENT_PATH"]) as f: + event = json.load(f) + + if "pull_request" in event: logger.info("Not tagging for PR") - return - if os.environ["TRAVIS_BRANCH"] != "master": + return False + if event.get("ref") != "refs/heads/master": logger.info("Not tagging for non-master branch") + return False + return True + + +def main(): + repo_key = "GITHUB_REPOSITORY" + should_run = should_run_action() + + if not should_run: + logger.info("Not tagging master for this push") return + owner, repo = os.environ[repo_key].split("/", 1) + git = get_git_cmd(wpt_root) head_rev = git("rev-parse", "HEAD") pr = get_pr(owner, repo, head_rev) if pr is None: - sys.exit(1) - tagged = tag(owner, repo, head_rev, "merge_pr_%s" % pr) + # This should only really happen during testing + tag_name = "merge_commit_%s" % head_rev + else: + tag_name = "merge_pr_%s" % pr + + tagged = tag(owner, repo, head_rev, tag_name) if not tagged: sys.exit(1) + summary = git("show", "--no-patch", '--format="%s"', "HEAD") + body = git("show", "--no-patch", '--format="%b"', "HEAD") + + if not create_release(owner, repo, head_rev, tag_name, summary, body): + sys.exit(1) + if __name__ == "__main__": main() diff --git a/tools/docker/github/Dockerfile b/tools/docker/github/Dockerfile new file mode 100644 index 0000000000..5a6dc866cd --- /dev/null +++ b/tools/docker/github/Dockerfile @@ -0,0 +1,27 @@ +FROM ubuntu:18.04 + +# No interactive frontend during docker build +ENV DEBIAN_FRONTEND=noninteractive \ + DEBCONF_NONINTERACTIVE_SEEN=true + +RUN apt-get -qqy update \ + && apt-get -qqy install \ + git \ + locales \ + python \ + python-pip \ + tzdata + +RUN pip install --upgrade pip +RUN pip install virtualenv +RUN pip install requests + +ENV TZ "UTC" +RUN echo "${TZ}" > /etc/timezone \ + && dpkg-reconfigure --frontend noninteractive tzdata + +# Set the locale +RUN locale-gen en_US.UTF-8 +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US:en +ENV LC_ALL en_US.UTF-8