Skip to content

Commit

Permalink
DockerAPIを使うように修正 #162 (#164)
Browse files Browse the repository at this point in the history
* DockerAPIを操作するためのクライアントを追加 #162

* jesterがdevelだと動かなかったのでバージョン固定 #162

* 不要になったテストコードを削除 #162

* Nimコンパイラのバージョンアップ #162

* ビルド用のコンテナでdevelを使うように変更 #162

* テスト方法を修正 #162

* bigendianで読み取るように修正

* ディレクトリ作成

* コンテナを削除

* 削除をremoverで

* fix

* 削除のログを追加

* 一旦無効化

* ネットワーク制限を追加

* 非同期処理に変更

* Revert "非同期処理に変更"

This reverts commit a7a8bb3.

* getLogs follow=falseにして自前でタイムアウトするように変更

* デバッグコードを削除

* ciフラグありのときはテストをスキップ

* ciでのテスト時はciスキップ
  • Loading branch information
jiro4989 authored Jun 11, 2020
1 parent 9c491e0 commit 6645109
Show file tree
Hide file tree
Showing 9 changed files with 291 additions and 150 deletions.
13 changes: 10 additions & 3 deletions .github/workflows/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -29,12 +29,19 @@ jobs:
- uses: actions/checkout@v1
- name: Pull docker image
run: docker pull jiro4989/websh:latest
- uses: jiro4989/setup-nim-action@v1
with:
nim-version: devel
- name: Test server
run: docker-compose -f docker-compose-ci.yml run server test -Y
run: |
pushd websh_server
echo -e '\nswitch("d", "ci")' > tests/config.nims
nimble test -Y
- name: Build server
run: docker-compose -f docker-compose-ci.yml run server
- name: Test API
run: ./.github/scripts/api_test.sh
# FIXME: GitHubActionsでDockerAPIを使用可能にする必要がある
# - name: Test API
# run: ./.github/scripts/api_test.sh
- name: Build front
run: docker-compose -f docker-compose-ci.yml run front
- name: Archive release files
Expand Down
2 changes: 1 addition & 1 deletion Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -11,4 +11,4 @@ RUN apt-get update -yqq \
ENV PATH /root/.nimble/bin:$PATH
RUN curl https://nim-lang.org/choosenim/init.sh -sSf > init.sh
RUN sh init.sh -y \
&& choosenim update stable
&& choosenim update devel
153 changes: 153 additions & 0 deletions websh_server/src/dockerclient.nim
Original file line number Diff line number Diff line change
@@ -0,0 +1,153 @@
import httpclient, os, json, strformat, streams, endians
from strutils import join

import status

type
DockerClient = ref object
client: HttpClient
url: string
Mount = object
Target, Source, Type: string
ReadOnly: bool
HostConfig = object
Memory: int64
OomKillDisable: bool
PidsLimit: int64
Mounts: seq[Mount]
NetworkMode: string

proc newClient*(): DockerClient =
let
client = newHttpClient(timeout = 10 * 1000)
url = "http://localhost:2376"
return DockerClient(client: client, url: url)

proc createContainer*(self: DockerClient, name: string, image: string, cmds: seq[string], script = "", mediaDir = "", imageDir = ""): Response =
var self = self
self.client.headers = newHttpHeaders({ "Content-Type": "application/json" })
let url = &"{self.url}/containers/create?name={name}"

var body = %*{
"Image": image,
"Cmd": cmds,
"Tty": false,
}

var hostconf = HostConfig(
Memory: 256 * 1024 * 1024, # 256MB
OomKillDisable: true,
PidsLimit: 1024,
NetworkMode: "none",
)
var mounts: seq[Mount]
if script != "":
mounts.add(Mount(Target: "/tmp/exec.sh", Source: script, Type: "bind", ReadOnly: true))
if mediaDir != "":
mounts.add(Mount(Target: "/media", Source: mediaDir, Type: "bind", ReadOnly: true))
if imageDir != "":
mounts.add(Mount(Target: "/images", Source: imageDir, Type: "bind", ReadOnly: false))

if 1 <= mounts.len:
hostconf.Mounts = mounts

body["HostConfig"] = % hostconf

self.client.post(url = url, body = $body)

proc startContainer*(self: DockerClient, name: string): Response =
var self = self
self.client.headers = newHttpHeaders({ "Content-Type": "application/json" })
let url = &"{self.url}/containers/{name}/start"
self.client.post(url = url)

proc killContainer*(self: DockerClient, name: string): Response =
var self = self
let url = &"{self.url}/containers/{name}/kill"
self.client.post(url = url)

proc removeContainer*(self: DockerClient, name: string): Response =
var self = self
let url = &"{self.url}/containers/{name}?v=true&force=true"
self.client.delete(url = url)

proc inspectContainer*(self: DockerClient, name: string): Response =
var self = self
let url = &"{self.url}/containers/{name}/json"
self.client.get(url = url)

proc parseLog(s: string): string =
var strm = newStringStream(s)
defer: strm.close
var lines: seq[string]
while not strm.atEnd:
# 1 = stdout, 2 = stderr
if strm.readUint8() notin [1'u8, 2]:
break

# 3byteは使わないので捨てる
discard strm.readUint8()
discard strm.readUint8()
discard strm.readUint8()

# Bigendianで読み取る
var src = strm.readUint32().int
var n: int
bigEndian32(addr(n), addr(src))

lines.add(strm.readStr(n))
result = lines.join

proc getLog(self: DockerClient, name: string, stdout = false, stderr = false): Response =
let url = &"{self.url}/containers/{name}/logs?stdout={stdout}&stderr={stderr}&follow=false"
self.client.get(url = url)

proc getStdoutLog*(self: DockerClient, name: string): Response =
self.getLog(name = name, stdout = true, stderr = false)

proc getStderrLog*(self: DockerClient, name: string): Response =
self.getLog(name = name, stdout = false, stderr = true)

proc waitFinish(self: DockerClient, name: string) =
const timeout = 10000
const unit = 200
var elapsed: int
while true:
let resp = self.inspectContainer(name = name)
if not resp.code.is2xx:
return
let running = resp.body.parseJson["State"]["Running"].getBool
if not running: break

sleep unit
elapsed += unit
if timeout <= elapsed: break

proc runContainer*(self: DockerClient, name: string, image: string, cmds: seq[string], script = "", mediaDir = "", imageDir = ""): (string, string, int, string) =
var resp: Response
resp = self.createContainer(name = name, image = image, cmds = cmds, script = script, mediaDir = mediaDir, imageDir = imageDir)
if not resp.code.is2xx:
return ("", "", statusSystemError, &"failed to call 'createContainer': cmds={cmds} resp.body={resp.body}")

resp = self.startContainer(name = name)
if not resp.code.is2xx:
return ("", "", statusSystemError, &"failed to call 'startContainer': cmds={cmds} resp.body={resp.body}")

self.waitFinish(name = name)

var stdoutStr: string
resp = self.getStdoutLog(name = name)
if not resp.code.is2xx:
return ("", "", statusSystemError, &"failed to call 'getStdoutLog': cmds={cmds} resp.body={resp.body}")
stdoutStr = resp.body.parseLog

var stderrStr: string
resp = self.getStderrLog(name = name)
if not resp.code.is2xx:
return ("", "", statusSystemError, &"failed to call 'getStderrLog': cmds={cmds} resp.body={resp.body}")
stderrStr = resp.body.parseLog

discard self.killContainer(name = name)
discard self.removeContainer(name = name)

return (stdoutStr, stderrStr, statusOk, "")
5 changes: 5 additions & 0 deletions websh_server/src/status.nim
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
const
statusOk* = 0
statusTimeout* = 1
statusSystemError* = 100

25 changes: 2 additions & 23 deletions websh_server/src/websh_remover.nim
Original file line number Diff line number Diff line change
Expand Up @@ -13,28 +13,7 @@ when isMainModule and not defined modeTest:
let rmflagDir = containerDir/"removes"
if not existsDir(rmflagDir): continue

let (_, containerName, _) = containerDir.splitFile()
discard execShellCmd(&"docker kill {containerName}")

# lockだけは一番最後に削除する必要がある
for dir in walkDirs(containerDir/"*"):
let (_, base, _) = splitFile(dir)
echo %*{"time": $now(), "level": "info", "msg": "remove " & base}
if base == "lock": continue
removeDir(dir)
if base in ["images", "media", "script"]:
createDir(dir)
dir.setFilePermissions({
fpUserRead,
fpUserWrite,
fpUserExec,
fpGroupRead,
fpGroupWrite,
fpGroupExec,
fpOthersRead,
fpOthersWrite,
fpOthersExec,
})
removeDir(containerDir/"lock")
removeDir(containerDir)
echo %*{"time": $now(), "level": "info", "msg": &"{containerDir} was removed"}

sleep(500) # ミリ秒
112 changes: 26 additions & 86 deletions websh_server/src/websh_server.nim
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ from algorithm import sorted
# 外部ライブラリ
import jester, uuids

import status, dockerclient

type
ReqShellgeiJSON* = object
code*: string
Expand All @@ -14,62 +16,12 @@ type
filesize*: int

const
statusOk = 0
statusTimeout = 1
statusSystemError = 100
scriptName = "exec.sh"
targetScript = "/tmp/" & scriptName
containerPrefix = "shellgeibot"

proc getTmpDir(): string = getCurrentDir() / "tmp"

proc readStream(strm: var Stream): string =
defer: strm.close()
result = strm.readAll()

proc runCommand(command: string, args: openArray[string], timeout: int = 3): (string, string, int, string) =
## ``command`` を実行し、標準出力と標準エラー出力を返す。
## timeout は秒を指定する。
var
p = startProcess(command, args = args, options = {poUsePath})
stdoutStr, stderrStr: string
defer: p.close()

let
timeoutMilSec = timeout * 1000
exitCode = waitForExit(p, timeout = timeoutMilSec)

# 処理結果の判定
var
status: int
msg: string
if exitCode == 0:
status = statusOk
elif exitCode == 137:
status = statusTimeout
msg = &"timeout: {timeout} second"
else:
status = statusSystemError
msg = &"failed to run command: command={command}, args={args}"

# 出力の取得
block:
var strm = p.outputStream
stdoutStr = strm.readStream()
block:
var strm = p.errorStream
stderrStr = strm.readStream()

result = (stdoutStr, stderrStr, status, msg)

proc runCommandOnContainer(scriptDir, containerName: string): (string, string, int, string) =
let vScript = &"/tmp/script/{scriptName}"
let args = [
"exec",
"-i", containerName,
"bash", "-c", &"sync && cp {vScript} {vScript}.1 && chmod +x {vScript}.1 && {vScript}.1 | stdbuf -o0 head -c 100K",
]
let timeout = getEnv("WEBSH_REQUEST_TIMEOUT", "3").parseInt
result = runCommand("docker", args, timeout)

proc getImages(dir: string): seq[ImageObj] =
## 画像ディレクトリから画像ファイルを取得。
Expand All @@ -81,19 +33,6 @@ proc getImages(dir: string): seq[ImageObj] =
let img = ImageObj(image: base64.encode(content), filesize: content.len)
result.add(img)

proc fetchContainerName(count: int): string =
## 一番起動時間の長いコンテナ名を返す
##
## TODO: コマンドラインで実行した結果をパースしていてとても気に入らないけれど
## 、かと言ってAPIリクエストを調べるのも面倒なのでとりあえずはコレで...
let conts = toSeq(1..count).mapIt(&"{containerPrefix}_{it}")
var args = @["inspect", "-f", "{{.State.StartedAt}} {{.State.Status}} {{.Name}}"]
args = args.concat(conts)
let stdoutstr = execProcess("docker",
args = args,
options = {poUsePath})
result = stdoutstr.strip().split("\n").filterIt(" running " in it).sorted()[0].split(" ")[^1]

proc createMediaFiles(dir: string, medias: seq[string]) =
## 入力の画像ファイルをディレクトリ配下に出力。
## 画像ファイルはbase64エンコードされたデータで渡されるので
Expand All @@ -120,40 +59,38 @@ router myrouter:
echo %*{xForHeader: xFor, "time": $now(), "level": "info", "uuid": uuid, "code": respJson.code, "msg": "request begin"}

let
containersCount = getEnv("WEBSH_CONTAINERS_COUNT", "4").parseInt()
containerName = fetchContainerName(containersCount)
contDir = getTmpDir() / containerName
contDir = getTmpDir() / uuid
scriptDir = contDir / "script"
imageDir = contDir / "images"
mediaDir = contDir / "media"
lockDir = contDir / "lock"
rmflagDir = contDir / "removes"

# ロックファイルの生成
# trueが反るときはすでにディレクトリが存在する(ロック中)なので終了
if existsOrCreateDir(lockDir):
resp %*{
"status": statusSystemError,
"system_message": "System is busy. Please wait a second.",
"stdout": "",
"stderr": "",
"images": [],
"elapsed_time": "0milsec",
}

defer:
# 削除フラグファイルの作成
discard existsOrCreateDir(rmflagDir)
removeFlag = contDir / "removes"

createDir(imageDir)

# コンテナ内で実行するスクリプトの生成
createDir(scriptDir)
let shellScriptPath = scriptDir/scriptName
writeFile(shellScriptPath, respJson.code)

# Mediaの配置
createMediaFiles(mediaDir, respJson.images)

# コンテナ上でシェルを実行
let (stdoutStr, stderrStr, status, systemMsg) = runCommandOnContainer(scriptDir, containerName)
const image = "theoldmoon0602/shellgeibot"
let cmds = @[
"bash",
"-c",
&"sync && cp {targetScript} {targetScript}.1 && chmod +x {targetScript}.1 && {targetScript}.1 | stdbuf -o0 head -c 100K",
]
var client = newClient()
let (stdoutStr, stderrStr, status, systemMsg) =
client.runContainer(
name = uuid,
image = image,
cmds = cmds,
script = shellScriptPath,
mediaDir = mediaDir,
imageDir = imageDir)

# TODO: ここ邪魔だなぁ
case status
Expand All @@ -165,6 +102,9 @@ router myrouter:

let images = getImages(imageDir)

# 削除フラグをたてる
createDir(removeFlag)

let elapsedTime = (now() - now).inMilliseconds
echo %*{xForHeader: xFor, "time": $now(), "level": "info", "uuid": uuid, "elapsedTime": elapsedTime, "msg": "request end"}

Expand Down
Loading

0 comments on commit 6645109

Please sign in to comment.