diff --git a/bids-hook.go b/bids-hook.go index a792ec6..fbcefb0 100644 --- a/bids-hook.go +++ b/bids-hook.go @@ -10,7 +10,6 @@ import ( "errors" "fmt" "io" - "io/fs" "log" "net/http" "net/url" @@ -66,13 +65,13 @@ var ( // * 0 = "success" (green checkmark) // * 1 = "failure" (red "X" mark) // * 2 = "warning" (yellow "!" mark) - // stdout will be saved to the Gitea url "/assets/${BH_UUID}.html" and linked from the commit status - // stderr will be appended to the log file "{{workerLogPath}}/${BH_UUID}.log" + // * 3+ = "error" (red "!" mark, no link to the result page) + // stdout will be saved to the Gitea url "/assets/bids-validator/XX/YY/${BH_UUID}.html" and linked from the commit status + // stderr will be appended to the log file "{{workerLogPath}}/XX/YY/${BH_UUID}.log" workerScript string // the path to a log directory for worker stderr output // read from environment variable WORKER_LOG_PATH - // it should already exist workerLogPath string // channel used to ferry jobs from the server to the worker @@ -279,18 +278,18 @@ type job struct { // web link to the results page for this job // see also j.resultPath() func (j job) resultUrl() string { - return giteaRootUrl.JoinPath("assets", fmt.Sprintf("%s.html", j.uuid)).String() + return giteaRootUrl.JoinPath("assets", "bids-validator", j.uuid[:2], j.uuid[2:4], fmt.Sprintf("%s.html", j.uuid)).String() } // file path to the results page for this job // see also j.resultUrl() func (j job) resultPath() string { - return filepath.Join(giteaCustom, "public", fmt.Sprintf("%s.html", j.uuid)) + return filepath.Join(giteaCustom, "public", "bids-validator", j.uuid[:2], j.uuid[2:4], fmt.Sprintf("%s.html", j.uuid)) } // file path to the log file for this job func (j job) logPath() string { - return filepath.Join(workerLogPath, fmt.Sprintf("%s.log", j.uuid)) + return filepath.Join(workerLogPath, j.uuid[:2], j.uuid[2:4], fmt.Sprintf("%s.log", j.uuid)) } // postStatus posts a commit status to Gitea @@ -360,7 +359,12 @@ func (j job) run() (state string, _ error) { ) // redirect stdout to the result file - stdout, err := os.OpenFile(j.resultPath(), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644) + resultPath := j.resultPath() + err := os.MkdirAll(filepath.Dir(resultPath), 0750) + if err != nil { + return stateError, err + } + stdout, err := os.OpenFile(resultPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0640) if err != nil { return stateError, err } @@ -373,7 +377,12 @@ func (j job) run() (state string, _ error) { cmd.Stdout = stdout // redirect stderr to the log file - stderr, err := os.OpenFile(j.logPath(), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644) + logPath := j.logPath() + err = os.MkdirAll(filepath.Dir(logPath), 0750) + if err != nil { + return stateError, err + } + stderr, err := os.OpenFile(j.logPath(), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0640) if err != nil { return stateError, err } @@ -420,10 +429,9 @@ func worker() { // readConfig sets up global variables from environment values func readConfig() { var ( - val string - ok bool - err error - info fs.FileInfo + val string + ok bool + err error ) val, ok = os.LookupEnv("BIDS_HOOK_URL") @@ -464,7 +472,7 @@ func readConfig() { if err != nil { log.Fatalf("invalid GITEA_CUSTOM: %v", err) } - err = os.MkdirAll(filepath.Join(giteaCustom, "public"), 0750) + err = os.MkdirAll(filepath.Join(giteaCustom, "public", "bids-validator"), 0750) if err != nil { log.Fatalf("error creating output folder: %v", err) } @@ -486,12 +494,9 @@ func readConfig() { if err != nil { log.Fatalf("invalid WORKER_LOG_PATH: %v", err) } - info, err = os.Stat(workerLogPath) + err = os.MkdirAll(workerLogPath, 0750) if err != nil { - log.Fatalf("error opening WORKER_LOG_PATH: %v", err) - } - if !info.IsDir() { - log.Fatal("WORKER_LOG_PATH is not a directory") + log.Fatalf("error creating log folder: %v", err) } val, ok = os.LookupEnv("WORKER_QUEUE_CAPACITY") diff --git a/start b/start index 89be690..992da9f 100755 --- a/start +++ b/start @@ -22,7 +22,7 @@ export BIDS_HOOK_SECRET='blabla' export GITEA_ROOT_URL='http://127.0.0.1:3000' export GITEA_TOKEN='69e45fa9cfa75a7497633c6be8dd2347226e2f62' -export WORKER_SCRIPT='./worker' +export WORKER_SCRIPT='./worker-script' export WORKER_LOG_PATH='./log' export WORKER_QUEUE_CAPACITY=20 diff --git a/worker b/worker deleted file mode 100755 index 5992a77..0000000 --- a/worker +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash -set -o nounset -o pipefail -trap 'exit 3' ERR - -(exec 1>&2 -date -cat < - -a title -

a paragraph -EOF - -exit 0 diff --git a/worker-script b/worker-script new file mode 100755 index 0000000..3035b81 --- /dev/null +++ b/worker-script @@ -0,0 +1,147 @@ +#!/bin/bash +# ------------------------------------------------------------------- +# Default bids-hook worker script +# ------------------------------------------------------------------- +# +# If bids-hook is started with the environment variable WORKER_SCRIPT +# pointing to this script, then it will be executed by the worker +# once for each accepted job. +# +# In addition to inheriting all environment variables used to launch +# bids-hook, the script will be given the details of the job in the +# following environment variables: +# +# BH_USER, BH_REPO, BH_COMMIT, BH_UUID +# +# Any output generated by the script on stdout (file descriptor 1) +# is saved as a result page, overwriting the output of any previous +# run for the same UUID. The result page is linked from the commit +# status posted on Gitea, and will be visible to anyone who has a +# link. It should be a complete, properly formatted HTML document. +# +# Any output generated by the script on stderr (file descriptor 2) +# is appended to a log file, after the log output of any previous +# run for the same UUID, visible at the filesystem path: +# +# "${WORKER_LOG_PATH}/ab/cd/${BH_UUID}.log" +# +# where "abcd" are the first four characters of BH_UUID. +# +# The exit code of the script is interpreted as follows: +# +# 0 = "success" (green checkmark) +# 1 = "failure" (red "X" mark) +# 2 = "warning" (yellow "!" mark) +# 3+ = "internal error" (red "!" mark, no link to the result page) +# +# ------------------------------------------------------------------- + +# Exit with the "internal error" status if anything goes wrong. +set -o nounset -o pipefail +trap 'exit 3' ERR + +# Log the job metadata to stderr. +(exec 1>&2 +echo '=== job start ===' +date +cat <&2 "GITEA_REPO=${GITEA_REPO}" + +# Do all the work in a private temporary directory. +WORKDIR=$(mktemp --directory --tmpdir bids-hook.XXXXXXXXXX) +echo 1>&2 "WORKDIR=${WORKDIR}" +# We have an ERR trap to account for this: +# shellcheck disable=SC2164 +cd 1>&2 "$WORKDIR" +# We have an EXIT trap which makes this code reachable: +# shellcheck disable=SC2317 +function cleanup() { + echo "# cleaning ${WORKDIR}" + chmod --recursive u+w "$WORKDIR" + rm --recursive "$WORKDIR" +} 1>&2 +trap cleanup EXIT + +# Check out the repository, redirecting all output to stderr. +(exec 1>&2 +# We'd like to do 'git clone -b "$BH_COMMIT" --depth 1 "$GITEA_REPO"' +# but maybe "$BH_COMMIT" is a commit ID instead of a branch name, so +# we do it this way instead. +echo "# cloning ${GITEA_REPO}" +git init +git remote add origin "$GITEA_REPO" +git fetch --depth 1 origin "$BH_COMMIT" +git checkout "$BH_COMMIT" + +# If this is a git-annex repository, we need to get the contents. +if git ls-remote --exit-code origin refs/heads/git-annex >/dev/null; then + echo '# getting git-annexed files' + # this reduces copies; always overrides annex.hardlink even if that is set system-wide + git config annex.thin true + # make sure we don't corrupt origin accidentally + git config remote.origin.annex-readonly true + git config annex.private true # XXX this doesn't do anything until git-annex 10 + git annex init + git annex dead here # this is like annex.private, but has to be run + # grab the git-annex branch (since we did a shallow clone above) + git annex sync --only-annex --no-content + # NB: using 'copy --from origin' and not 'git annex get; to ensure we're + # validating the contents of origin and not any special remotes + git annex copy --from origin +fi +) + +echo 1>&2 '# running bids-validator' +OUTPUT=$(bids-validator .) && STATUS=$? || STATUS=$? +WARNINGS=$(bids-validator . --json | jq '.issues.warnings | length') || true +cat 1>&2 <&2 '# formatting output' +HTML=$(echo "$OUTPUT" | ansifilter --html --fragment --line-numbers --anchors=self) + +# Produce the HTML result page on stdout. +cat < + +bids-validator results for ${BH_USER}/${BH_REPO}@${BH_COMMIT} +

Here are the bids-validator results for ${BH_USER}/${BH_REPO}@${BH_COMMIT}:

+
+${HTML}
+
+ +EOF + +# Our exit code means: +# 0 = "success" (green checkmark) +# 1 = "failure" (red "X" mark) +# 2 = "warning" (yellow "!" mark) +# 3+ = "internal error" (red "!" mark, no link to the result page) +case $STATUS in +(0) + if ((WARNINGS)); then + exit 2 + else + exit 0 + fi + ;; +(1) + exit 1 + ;; +(*) + exit 3 + ;; +esac