From bdd9670b13fcb84c3a2a7eef138584a859533983 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Minh=20Nh=E1=BA=ADt?= <86871862+minhnhatnoe@users.noreply.github.com> Date: Sat, 14 Sep 2024 06:56:24 -0700 Subject: [PATCH] Add support for isolate v2 (#112) * fix: change isolate daemon command in scripts/start_container.sh * feat: Remove isolate in docker/Dockerfile * chore: update go-sqlite3 to fix musl build https://github.com/mattn/go-sqlite3/issues/1164 * feat: Dockerfile for gcc-only * feat: accept arbitrary command for isolate daemon * feat(isolate): print logs real-time and add Stop() * chore: update golangci-lint to make tests passable --- cmd/kjudge/main.go | 1 + docker/Dockerfile | 22 +------ docker/gcc-only.dockerfile | 18 +----- go.mod | 2 +- go.sum | 10 +-- scripts/install_tools.sh | 2 +- scripts/start_container.sh | 6 +- worker/sandbox.go | 6 +- worker/sandbox/isolate/sandbox.go | 100 ++++++++++++++++++++++++++---- worker/sandbox/raw/sandbox.go | 6 ++ worker/sandbox/sandbox.go | 2 + 11 files changed, 113 insertions(+), 62 deletions(-) diff --git a/cmd/kjudge/main.go b/cmd/kjudge/main.go index 36fe1101..f6bcd1d9 100644 --- a/cmd/kjudge/main.go +++ b/cmd/kjudge/main.go @@ -56,6 +56,7 @@ func main() { log.Println("Starting kjudge. Press Ctrl+C to stop") + go sandbox.Start() go queue.Start() go startServer(server) diff --git a/docker/Dockerfile b/docker/Dockerfile index 2f798177..3698a149 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,4 +1,4 @@ -# Stage -1: Update apt-get +# Stage 0: Update apt-get FROM ubuntu:jammy AS base-ubuntu # Mount apt's cache folders to cache install but maintain @@ -10,19 +10,6 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ --mount=type=cache,target=/var/lib/apt,sharing=locked \ apt-get update && apt-get upgrade -y -# Stage 0: Compile isolate -FROM base-ubuntu AS isolate - -RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ - --mount=type=cache,target=/var/lib/apt,sharing=locked \ - apt-get update && apt-get install -y libcap-dev gcc git make - -WORKDIR /isolate - -RUN git clone --branch v1.10.1 --single-branch https://github.com/ioi/isolate.git . - -RUN make isolate - # Stage 1: Generate front-end FROM node:18-alpine AS frontend @@ -51,7 +38,7 @@ RUN scripts/install_tools.sh RUN go generate && go build -tags production -o kjudge cmd/kjudge/main.go # Stage 3: Create awesome output image -FROM base-ubuntu +FROM ghcr.io/minhnhatnoe/isolate:v2.1.5 RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ --mount=type=cache,target=/var/lib/apt,sharing=locked \ @@ -60,11 +47,6 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ python3.10 python2.7 rustc golang libcap-dev \ openssl -COPY --from=isolate /isolate/ /isolate - -WORKDIR /isolate -RUN make install - COPY --from=backend /kjudge/kjudge /usr/local/bin COPY --from=backend /kjudge/scripts /scripts diff --git a/docker/gcc-only.dockerfile b/docker/gcc-only.dockerfile index 3071e1f0..47fa31c3 100644 --- a/docker/gcc-only.dockerfile +++ b/docker/gcc-only.dockerfile @@ -1,14 +1,3 @@ -# Stage 0: Compile isolate -FROM alpine:3 AS isolate - -RUN apk add --no-cache libcap gcc make git g++ libcap-dev - -WORKDIR /isolate - -RUN git clone --branch v1.10.1 --single-branch https://github.com/ioi/isolate.git . - -RUN make isolate - # Stage 1: Generate front-end FROM node:18-alpine AS frontend @@ -40,15 +29,10 @@ RUN sh scripts/install_tools.sh RUN go generate && go build -tags production -o kjudge cmd/kjudge/main.go # Stage 3: Create awesome output image -FROM alpine:3 +FROM ghcr.io/minhnhatnoe/isolate:v2.1.5-alpine RUN apk add --no-cache libcap make g++ openssl bash -COPY --from=isolate /isolate/ /isolate - -WORKDIR /isolate -RUN make install - COPY --from=backend /kjudge/kjudge /usr/local/bin COPY --from=backend /kjudge/scripts /scripts diff --git a/go.mod b/go.mod index a4be115b..b628ecb8 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/labstack/echo-contrib v0.9.0 github.com/labstack/echo/v4 v4.9.0 github.com/mattn/go-colorable v0.1.12 // indirect - github.com/mattn/go-sqlite3 v1.14.0 + github.com/mattn/go-sqlite3 v1.14.19 github.com/pkg/errors v0.9.1 golang.org/x/crypto v0.14.0 golang.org/x/net v0.17.0 // indirect diff --git a/go.sum b/go.sum index ef91d5f8..27ce7500 100644 --- a/go.sum +++ b/go.sum @@ -1,10 +1,8 @@ github.com/BurntSushi/toml v1.1.0 h1:ksErzDEI1khOiGPgpwuI7x2ebx/uXQNw7xJpn9Eq1+I= github.com/BurntSushi/toml v1.1.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= -github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= github.com/appleboy/gofight/v2 v2.1.2/go.mod h1:frW+U1QZEdDgixycTj4CygQ48yLTUhplt43+Wczp3rw= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= @@ -62,8 +60,8 @@ github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= -github.com/mattn/go-sqlite3 v1.14.0 h1:mLyGNKR8+Vv9CAU7PphKa2hkEqxxhn8i32J6FPj1/QA= -github.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus= +github.com/mattn/go-sqlite3 v1.14.19 h1:fhGleo2h1p8tVChob4I9HpmVFIAkKGpiukdrgQbWfGI= +github.com/mattn/go-sqlite3 v1.14.19/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -116,7 +114,6 @@ golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -124,8 +121,6 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20190607181551-461777fb6f67/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= @@ -146,7 +141,6 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190602015325-4c4f7f33c9ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190609082536-301114b31cce/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/scripts/install_tools.sh b/scripts/install_tools.sh index 3eb68540..a89acc82 100755 --- a/scripts/install_tools.sh +++ b/scripts/install_tools.sh @@ -8,4 +8,4 @@ do done # Install golangci-lint -curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b "$(go env GOPATH)"/bin v1.52.2 +curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b "$(go env GOPATH)"/bin v1.58.0 diff --git a/scripts/start_container.sh b/scripts/start_container.sh index dc50c738..2a9c89ae 100755 --- a/scripts/start_container.sh +++ b/scripts/start_container.sh @@ -44,7 +44,7 @@ case ${HTTPS} in esac if [ "${useHTTPS}" = true ]; then - kjudge -port 443 -file /data/kjudge.db -https /certs "$@" + KJUDGE_ISOLATE_DAEMON="start_isolate" kjudge -port 443 -file /data/kjudge.db -https /certs "$@" else - kjudge -port 80 -file /data/kjudge.db "$@" -fi + KJUDGE_ISOLATE_DAEMON="start_isolate" kjudge -port 80 -file /data/kjudge.db "$@" +fi \ No newline at end of file diff --git a/worker/sandbox.go b/worker/sandbox.go index 41a68654..79b48c6d 100644 --- a/worker/sandbox.go +++ b/worker/sandbox.go @@ -12,8 +12,12 @@ func NewSandbox(name string, options ...sandbox.Option) (sandbox.Runner, error) switch name { case "raw": return raw.New(setting), nil + case "isolate_v1": + return isolate.New(1, setting), nil case "isolate": - return isolate.New(setting), nil + fallthrough + case "isolate_v2": + return isolate.New(2, setting), nil default: return nil, errors.Errorf("Sandbox %s doesn't exists or not yet implemented.", name) } diff --git a/worker/sandbox/isolate/sandbox.go b/worker/sandbox/isolate/sandbox.go index 57125e04..0f0e0dd3 100644 --- a/worker/sandbox/isolate/sandbox.go +++ b/worker/sandbox/isolate/sandbox.go @@ -6,8 +6,11 @@ package isolate import ( + "bufio" "bytes" "fmt" + "io" + "log" "os" "os/exec" "path/filepath" @@ -19,40 +22,115 @@ import ( ) var ( - // The isolate command. Can be overridden with KJUDGE_ISOLATE environment variable. - isolateCommand = "isolate" + // The isolate command. Can be overridden with KJUDGE_ISOLATE_V1 environment variable. + isolateCommandV1 = "isolate" + isolateCommandV2 = "isolate" + isolateDaemonCommand = "systemctl status isolate.service" ) func init() { + if v, ok := os.LookupEnv("KJUDGE_ISOLATE_V1"); ok { + isolateCommandV1 = v + } if v, ok := os.LookupEnv("KJUDGE_ISOLATE"); ok { - isolateCommand = v + isolateCommandV2 = v + } + if v, ok := os.LookupEnv("KJUDGE_ISOLATE_DAEMON"); ok { + isolateDaemonCommand = v } } // Runner implements worker.Runner. type Runner struct { + version int // 1 or 2 + cmd *exec.Cmd settings sandbox.Settings private struct{} // Makes the sandbox not simply constructible } var _ sandbox.Runner = (*Runner)(nil) +func (s *Runner) isolateCommand() string { + if s.version == 1 { + return isolateCommandV1 + } else if s.version == 2 { + return isolateCommandV2 + } else { + log.Panicf("Invalid isolate version: %d", s.version) + return "" + } +} + // Panics on not having "isolate" accessible. -func mustHaveIsolate() { - output, err := exec.Command(isolateCommand, "--version").CombinedOutput() +func (s *Runner) mustHaveIsolate() { + output, err := exec.Command(s.isolateCommand(), "--version").CombinedOutput() if err != nil { panic(errors.Wrap(err, "trying to run isolate")) } if !strings.Contains(string(output), "The process isolator") { - panic("Wrong isolate command found. Override the KJUDGE_ISOLATE environment variable to set a different path.") + panic("Wrong isolate command found. Override the KJUDGE_ISOLATE_V1/KJUDGE_ISOLATE environment variable to set a different path.") } } // New returns a new sandbox. // Panics if isolate is not installed. -func New(settings sandbox.Settings) *Runner { - mustHaveIsolate() - return &Runner{settings: settings, private: struct{}{}} +func New(version int, settings sandbox.Settings) *Runner { + runner := &Runner{version: version, cmd: nil, settings: settings, private: struct{}{}} + runner.mustHaveIsolate() + return runner +} + +func (s *Runner) Start() { + if s.version == 1 { + return + } else if s.version != 2 { + log.Panicf("Invalid isolate version: %v", s.version) + } + + s.cmd = exec.Command("/bin/sh", "-c", isolateDaemonCommand) + + stdout, err := s.cmd.StdoutPipe() + if err != nil { + log.Panic(errors.Wrap(err, "getting stdout pipe")) + } + + stderr, err := s.cmd.StderrPipe() + if err != nil { + log.Panic(errors.Wrap(err, "getting stderr pipe")) + } + + multi := io.MultiReader(stdout, stderr) + reader := bufio.NewScanner(multi) + + if err := s.cmd.Start(); err != nil { + log.Panic(errors.Wrap(err, "starting isolate daemon")) + } + + for reader.Scan() { + log.Printf("[isolate v2 daemon]: %s", reader.Text()) + } + + if err := reader.Err(); err != nil { + log.Panic(errors.Wrapf(err, "isolate daemon dead. Is daemon installed and, if installed as a systemd unit, started?")) + } + + if err := s.cmd.Wait(); err != nil { + log.Panic(errors.Wrap(err, "waiting for isolate daemon")) + } +} + +func (s *Runner) Stop() error { + if s.cmd != nil { + if err := s.cmd.Process.Kill(); err != nil { + return errors.Wrap(err, "killing isolate daemon") + } + + if err := s.cmd.Process.Release(); err != nil { + return errors.Wrap(err, "releasing isolate daemon") + } + } + + return nil } func (s *Runner) Settings() *sandbox.Settings { @@ -63,7 +141,7 @@ func (s *Runner) Settings() *sandbox.Settings { func (s *Runner) Run(input *sandbox.Input) (*sandbox.Output, error) { // Init the sandbox defer s.cleanup() - dirBytes, err := exec.Command(isolateCommand, "--init", "--cg").Output() + dirBytes, err := exec.Command(s.isolateCommand(), "--init", "--cg").Output() if err != nil { return nil, errors.WithStack(err) } @@ -153,5 +231,5 @@ func buildCmd(dir, metaFile string, input *sandbox.Input) *exec.Cmd { } func (s *Runner) cleanup() { - _ = exec.Command(isolateCommand, "--cleanup", "--cg").Run() + _ = exec.Command(s.isolateCommand(), "--cleanup", "--cg").Run() } diff --git a/worker/sandbox/raw/sandbox.go b/worker/sandbox/raw/sandbox.go index 9089f074..7d169b9a 100644 --- a/worker/sandbox/raw/sandbox.go +++ b/worker/sandbox/raw/sandbox.go @@ -35,6 +35,12 @@ func New(settings sandbox.Settings) *Runner { return &Runner{settings: settings} } +func (s *Runner) Start() {} + +func (s *Runner) Stop() error { + return nil +} + func (s *Runner) Settings() *sandbox.Settings { return &s.settings } diff --git a/worker/sandbox/sandbox.go b/worker/sandbox/sandbox.go index 20acb01d..68c2c7a7 100644 --- a/worker/sandbox/sandbox.go +++ b/worker/sandbox/sandbox.go @@ -16,6 +16,8 @@ import ( // and "raw" (NOT RECOMMENDED, RUN AT YOUR OWN RISK). // Which sandbox is used can be set at runtime with a command-line switch. type Runner interface { + Start() + Stop() error Settings() *Settings Run(*Input) (*Output, error) }