From eb0bf36ed930ff10df8dd7b6611af0685892dcef Mon Sep 17 00:00:00 2001 From: Kirill Zaitsev Date: Thu, 29 Dec 2016 19:41:30 +0300 Subject: [PATCH] Add runbook docker driver Docker is now the default driver, subprocess is used to communicate with the docker itself. runbook_runner image is used as a base image for all the runs and each run spawns a new image, that is later removed. --- runbook/drivers/base.py | 2 +- runbook/drivers/docker.py | 120 ++++++++++++++++++++++++++++++++++++++ runbook/drivers/shell.py | 4 +- runbook/runner.py | 24 ++++---- 4 files changed, 138 insertions(+), 12 deletions(-) create mode 100644 runbook/drivers/docker.py diff --git a/runbook/drivers/base.py b/runbook/drivers/base.py index 2b79e8d..2d2ba65 100644 --- a/runbook/drivers/base.py +++ b/runbook/drivers/base.py @@ -18,7 +18,7 @@ class Driver(object): interpreters = { "bash": "/bin/bash", - "python": "/bin/python", + "python": "/usr/bin/python", } @classmethod diff --git a/runbook/drivers/docker.py b/runbook/drivers/docker.py new file mode 100644 index 0000000..936311c --- /dev/null +++ b/runbook/drivers/docker.py @@ -0,0 +1,120 @@ +# Copyright 2016: Mirantis Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging +import os +import shutil +import subprocess +import tempfile +import uuid + +from runbook.drivers import base + +LOG = logging.getLogger("runner.docker") + + +RUNNER_DOCKERFILE = """ +FROM ubuntu:14.04 +RUN apt-get update +RUN apt-get install python -y +""" + +RUN_DOCKERFILE = """ +FROM runbook_runner +COPY . /runbook +WORKDIR /runbook +ENTRYPOINT ["{interpreter}"] +CMD ["runbook"] +""" + + +class Driver(base.Driver): + + @classmethod + def initialise(cls): + dirname = tempfile.mkdtemp() + with open(os.path.join(dirname, 'Dockerfile'), "w") as f: + f.write(RUNNER_DOCKERFILE) + output = subprocess.check_output( + ["docker", "build", "-t", "runbook_runner", dirname], + stderr=subprocess.STDOUT,) + LOG.info("Built 'runbook_runner' image, {}".format(output)) + + shutil.rmtree(dirname) + + @classmethod + def run(cls, runbook, parameters): + LOG.info("Running runbook '{}' with parameters '{}'".format( + runbook, parameters)) + + interpreter = cls.interpreters.get(runbook.get("type")) + if not interpreter: + return { + "return_code": -1, + "output": "Don't know how to run '{}' type runbook".format( + runbook.get("type")), + } + + dirname = tempfile.mkdtemp() + + with open(os.path.join(dirname, 'runbook'), "w") as f: + f.write(runbook["runbook"]) + + with open(os.path.join(dirname, 'Dockerfile'), "w") as f: + f.write(RUN_DOCKERFILE.format( + interpreter=interpreter)) + + run_name = "run-{}".format(uuid.uuid4().hex) + output = subprocess.check_output( + ["docker", "build", "-t", run_name, dirname], + stderr=subprocess.STDOUT,) + LOG.info("Built 'runbook_runner' image, {}".format(output)) + + returncode = 0 + try: + output = subprocess.check_output( + ["docker", "run", "--name", run_name, run_name], + stderr=subprocess.STDOUT, + ) + except subprocess.CalledProcessError as e: + output = e.output + returncode = e.returncode + + # cleanup + try: + subprocess.check_output( + ["docker", "rm", run_name], + stderr=subprocess.STDOUT, + ) + LOG.info("Removed container {}".format(run_name)) + except subprocess.CalledProcessError as e: + LOG.warning("Could not remove container {}".format( + run_name, e.output)) + + try: + subprocess.check_output( + ["docker", "rmi", run_name], + stderr=subprocess.STDOUT, + ) + LOG.info("Removed image {}".format(run_name)) + except subprocess.CalledProcessError as e: + LOG.warning("Could not remove image {}: {}".format( + run_name, e.output)) + + shutil.rmtree(dirname, ignore_errors=True) + return { + "return_code": returncode, + "output": output, + } diff --git a/runbook/drivers/shell.py b/runbook/drivers/shell.py index 1d9b8b1..5ca573c 100644 --- a/runbook/drivers/shell.py +++ b/runbook/drivers/shell.py @@ -21,6 +21,8 @@ from runbook.drivers import base +LOG = logging.getLogger("runner.shell") + class Driver(base.Driver): @@ -30,7 +32,7 @@ def initialise(cls): @classmethod def run(cls, runbook, parameters): - logging.info("Running runbook '{}' with parameters '{}'".format( + LOG.info("Running runbook '{}' with parameters '{}'".format( runbook, parameters)) f = tempfile.NamedTemporaryFile() f.write(runbook["runbook"]) diff --git a/runbook/runner.py b/runbook/runner.py index c986a2f..7e7f933 100644 --- a/runbook/runner.py +++ b/runbook/runner.py @@ -32,6 +32,7 @@ API_TYPE = "runner" CONF = config.get_config(API_TYPE) POOL = eventlet.GreenPool() +DRIVER = None LOG = logging.getLogger("runner") LOG.setLevel(logging.INFO) @@ -97,7 +98,6 @@ def handle_hit(hit, index_name, driver): LOG.info("Finished run '{}': '{}'".format(hit['_id'], run_result)) end_status = "finished" if run_result.get("return_code") == 0 else "failed" - # end_status = "scheduled" # FIXME now = datetime.datetime.now() try: @@ -122,15 +122,7 @@ def handle_hit(hit, index_name, driver): def job(): - driver_name = CONF.get("driver", "shell") - try: - driver_module = importlib.import_module( - "runbook.drivers." + driver_name) - except ImportError: - LOG.critical("No driver named '{}'".format(driver_name)) - return - - driver = driver_module.Driver + driver = DRIVER es = storage.get_elasticsearch(API_TYPE) for region in CONF["regions"]: @@ -146,6 +138,18 @@ def job(): def main(): + driver_name = CONF.get("driver", "docker") + try: + driver_module = importlib.import_module( + "runbook.drivers." + driver_name) + except ImportError: + LOG.critical("No driver named '{}'".format(driver_name)) + return + + driver = driver_module.Driver + driver.initialise() + global DRIVER + DRIVER = driver run_every_seconds = CONF.get("run_every_seconds", 30) schedule.every(run_every_seconds).seconds.do(job)