From 7035abe188835dd027c244c84d1d338a87f2a9da Mon Sep 17 00:00:00 2001 From: Maarten Hoogendoorn Date: Mon, 13 Mar 2017 19:42:56 +0100 Subject: [PATCH 001/123] Move statefile into state.file module --- nixops/deployment.py | 1 - nixops/state/__init__.py | 0 nixops/state/etcd.py | 0 nixops/{statefile.py => state/file.py} | 0 scripts/nixops | 16 ++++++++-------- tests/__init__.py | 6 +++--- tests/functional/__init__.py | 4 ++-- tests/functional/generic_deployment_test.py | 1 - 8 files changed, 13 insertions(+), 15 deletions(-) create mode 100644 nixops/state/__init__.py create mode 100644 nixops/state/etcd.py rename nixops/{statefile.py => state/file.py} (100%) diff --git a/nixops/deployment.py b/nixops/deployment.py index ddf9b8fb6..4bd325073 100644 --- a/nixops/deployment.py +++ b/nixops/deployment.py @@ -12,7 +12,6 @@ import errno from collections import defaultdict from xml.etree import ElementTree -import nixops.statefile import nixops.backends import nixops.logger import nixops.parallel diff --git a/nixops/state/__init__.py b/nixops/state/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/nixops/state/etcd.py b/nixops/state/etcd.py new file mode 100644 index 000000000..e69de29bb diff --git a/nixops/statefile.py b/nixops/state/file.py similarity index 100% rename from nixops/statefile.py rename to nixops/state/file.py diff --git a/scripts/nixops b/scripts/nixops index 1ba5b4bab..2e561601a 100755 --- a/scripts/nixops +++ b/scripts/nixops @@ -5,7 +5,7 @@ from nixops import deployment from nixops.nix_expr import py2nix from nixops.parallel import MultipleExceptions, run_tasks -import nixops.statefile +import nixops.state.file import prettytable import argparse import os @@ -48,14 +48,14 @@ def sort_deployments(depls): # $NIXOPS_DEPLOYMENT. def one_or_all(): if args.all: - sf = nixops.statefile.StateFile(args.state_file) + sf = nixops.state.file.StateFile(args.state_file) return sf.get_all_deployments() else: return [open_deployment()] def op_list_deployments(): - sf = nixops.statefile.StateFile(args.state_file) + sf = nixops.state.file.StateFile(args.state_file) tbl = create_table([("UUID", 'l'), ("Name", 'l'), ("Description", 'l'), ("# Machines", 'r'), ("Type", 'c')]) for depl in sort_deployments(sf.get_all_deployments()): tbl.add_row( @@ -67,7 +67,7 @@ def op_list_deployments(): def open_deployment(): - sf = nixops.statefile.StateFile(args.state_file) + sf = nixops.state.file.StateFile(args.state_file) depl = sf.open_deployment(uuid=args.deployment) depl.extra_nix_path = sum(args.nix_path or [], []) @@ -101,7 +101,7 @@ def modify_deployment(depl): def op_create(): - sf = nixops.statefile.StateFile(args.state_file) + sf = nixops.state.file.StateFile(args.state_file) depl = sf.create_deployment() sys.stderr.write("created deployment ‘{0}’\n".format(depl.uuid)) modify_deployment(depl) @@ -189,7 +189,7 @@ def op_info(): ]) if args.all: - sf = nixops.statefile.StateFile(args.state_file) + sf = nixops.state.file.StateFile(args.state_file) if not args.plain: tbl = create_table([('Deployment', 'l')] + table_headers) for depl in sort_deployments(sf.get_all_deployments()): @@ -491,7 +491,7 @@ def op_export(): def op_import(): - sf = nixops.statefile.StateFile(args.state_file) + sf = nixops.state.file.StateFile(args.state_file) existing = set(sf.query_deployments()) dump = json.loads(sys.stdin.read()) @@ -695,7 +695,7 @@ subparsers = parser.add_subparsers(help='sub-command help') def add_subparser(name, help): subparser = subparsers.add_parser(name, help=help) subparser.add_argument('--state', '-s', dest='state_file', metavar='FILE', - default=nixops.statefile.get_default_state_file(), help='path to state file') + default=nixops.state.file.get_default_state_file(), help='path to state file') subparser.add_argument('--deployment', '-d', dest='deployment', metavar='UUID_OR_NAME', default=os.environ.get("NIXOPS_DEPLOYMENT", os.environ.get("CHARON_DEPLOYMENT", None)), help='UUID or symbolic name of the deployment') subparser.add_argument('--debug', action='store_true', help='enable debug output') diff --git a/tests/__init__.py b/tests/__init__.py index 6bdd94ec8..e77249d78 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -3,14 +3,14 @@ import sys import threading from os import path -import nixops.statefile +import nixops.state.file _multiprocess_shared_ = True db_file = '%s/test.nixops' % (path.dirname(__file__)) def setup(): - nixops.statefile.StateFile(db_file).close() + nixops.state.file.StateFile(db_file).close() def destroy(sf, uuid): depl = sf.open_deployment(uuid) @@ -27,7 +27,7 @@ def destroy(sf, uuid): depl.logger.log("deployment ‘{0}’ destroyed".format(uuid)) def teardown(): - sf = nixops.statefile.StateFile(db_file) + sf = nixops.state.file.StateFile(db_file) uuids = sf.query_deployments() threads = [] for uuid in uuids: diff --git a/tests/functional/__init__.py b/tests/functional/__init__.py index 7152f23e8..76cee27fd 100644 --- a/tests/functional/__init__.py +++ b/tests/functional/__init__.py @@ -1,6 +1,6 @@ import os from os import path -import nixops.statefile +import nixops.state.file from tests import db_file @@ -8,7 +8,7 @@ class DatabaseUsingTest(object): _multiprocess_can_split_ = True def setup(self): - self.sf = nixops.statefile.StateFile(db_file) + self.sf = nixops.state.file.StateFile(db_file) def teardown(self): self.sf.close() diff --git a/tests/functional/generic_deployment_test.py b/tests/functional/generic_deployment_test.py index 92736370a..26085fb82 100644 --- a/tests/functional/generic_deployment_test.py +++ b/tests/functional/generic_deployment_test.py @@ -1,6 +1,5 @@ import os import subprocess -import nixops.statefile from nose import SkipTest From 6da031f332ea229903be11bec085788e8fb7f196 Mon Sep 17 00:00:00 2001 From: Maarten Hoogendoorn Date: Mon, 13 Mar 2017 20:19:01 +0100 Subject: [PATCH 002/123] Introduce different state state schemes. Needed to faclitate multiple backends (like sqlite3/other remote database) --- nixops/state/__init__.py | 23 +++++++++++++++++++++++ scripts/nixops | 32 ++++++++++++++++---------------- 2 files changed, 39 insertions(+), 16 deletions(-) diff --git a/nixops/state/__init__.py b/nixops/state/__init__.py index e69de29bb..b1cc8d1f8 100644 --- a/nixops/state/__init__.py +++ b/nixops/state/__init__.py @@ -0,0 +1,23 @@ +import urlparse +import sys +import file + +class WrongStateSchemeException(Exception): + pass + +def open(url): + url = urlparse.urlparse(url) + scheme = url.scheme + + if scheme == "": + scheme = "file" + + def raise_(ex): + raise ex + + switcher = { + "file": lambda(url): file.StateFile(url.path), + } + + function = switcher.get(scheme, lambda(url): raise_(WrongStateSchemeException("Unknown state scheme!"))) + return function(url) diff --git a/scripts/nixops b/scripts/nixops index 2e561601a..195f29a80 100755 --- a/scripts/nixops +++ b/scripts/nixops @@ -48,16 +48,16 @@ def sort_deployments(depls): # $NIXOPS_DEPLOYMENT. def one_or_all(): if args.all: - sf = nixops.state.file.StateFile(args.state_file) - return sf.get_all_deployments() + state = nixops.state.open(args.state_url) + return state.get_all_deployments() else: return [open_deployment()] def op_list_deployments(): - sf = nixops.state.file.StateFile(args.state_file) + state = nixops.state.open(args.state_url) tbl = create_table([("UUID", 'l'), ("Name", 'l'), ("Description", 'l'), ("# Machines", 'r'), ("Type", 'c')]) - for depl in sort_deployments(sf.get_all_deployments()): + for depl in sort_deployments(state.get_all_deployments()): tbl.add_row( [depl.uuid, depl.name or "(none)", depl.description, len(depl.machines), @@ -67,8 +67,8 @@ def op_list_deployments(): def open_deployment(): - sf = nixops.state.file.StateFile(args.state_file) - depl = sf.open_deployment(uuid=args.deployment) + state = nixops.state.open(args.state_url) + depl = state.open_deployment(uuid=args.deployment) depl.extra_nix_path = sum(args.nix_path or [], []) for (n, v) in args.nix_options or []: depl.extra_nix_flags.extend(["--option", n, v]) @@ -101,8 +101,8 @@ def modify_deployment(depl): def op_create(): - sf = nixops.state.file.StateFile(args.state_file) - depl = sf.create_deployment() + state = nixops.state.open(args.state_url) + depl = state.create_deployment() sys.stderr.write("created deployment ‘{0}’\n".format(depl.uuid)) modify_deployment(depl) if args.name or args.deployment: set_name(depl, args.name or args.deployment) @@ -189,10 +189,10 @@ def op_info(): ]) if args.all: - sf = nixops.state.file.StateFile(args.state_file) + state = nixops.state.open(args.state_url) if not args.plain: tbl = create_table([('Deployment', 'l')] + table_headers) - for depl in sort_deployments(sf.get_all_deployments()): + for depl in sort_deployments(state.get_all_deployments()): do_eval(depl) print_deployment(depl) if not args.plain: print tbl @@ -491,16 +491,16 @@ def op_export(): def op_import(): - sf = nixops.state.file.StateFile(args.state_file) - existing = set(sf.query_deployments()) + state = nixops.state.open(args.state_url) + existing = set(state.query_deployments()) dump = json.loads(sys.stdin.read()) for uuid, attrs in dump.iteritems(): if uuid in existing: raise Exception("state file already contains a deployment with UUID ‘{0}’".format(uuid)) - with sf._db: - depl = sf.create_deployment(uuid=uuid) + with state._db: + depl = state.create_deployment(uuid=uuid) depl.import_(attrs) sys.stderr.write("added deployment ‘{0}’\n".format(uuid)) @@ -694,8 +694,8 @@ subparsers = parser.add_subparsers(help='sub-command help') def add_subparser(name, help): subparser = subparsers.add_parser(name, help=help) - subparser.add_argument('--state', '-s', dest='state_file', metavar='FILE', - default=nixops.state.file.get_default_state_file(), help='path to state file') + subparser.add_argument('--state', '-s', dest='state_url', metavar='FILE', + default=os.environ.get("NIXOPS_STATE_URL", nixops.state.file.get_default_state_file()), help='URL that points to the state provider.') subparser.add_argument('--deployment', '-d', dest='deployment', metavar='UUID_OR_NAME', default=os.environ.get("NIXOPS_DEPLOYMENT", os.environ.get("CHARON_DEPLOYMENT", None)), help='UUID or symbolic name of the deployment') subparser.add_argument('--debug', action='store_true', help='enable debug output') From 82696503819dab13b8ab3c0b86b92a4075b1b1f8 Mon Sep 17 00:00:00 2001 From: Maarten Hoogendoorn Date: Mon, 13 Mar 2017 20:40:16 +0100 Subject: [PATCH 003/123] Rename statefile->state in deployment.py --- nixops/deployment.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/nixops/deployment.py b/nixops/deployment.py index 4bd325073..1c7091942 100644 --- a/nixops/deployment.py +++ b/nixops/deployment.py @@ -49,9 +49,9 @@ class Deployment(object): configs_path = nixops.util.attr_property("configsPath", None) rollback_enabled = nixops.util.attr_property("rollbackEnabled", False) - def __init__(self, statefile, uuid, log_file=sys.stderr): - self._statefile = statefile - self._db = statefile._db + def __init__(self, state, uuid, log_file=sys.stderr): + self._state = state + self._db = state._db self.uuid = uuid self._last_log_prefix = None @@ -121,7 +121,7 @@ def get_machine(self, name): def _set_attrs(self, attrs): - """Update deployment attributes in the state file.""" + """Update deployment attributes in the state.""" with self._db: c = self._db.cursor() for n, v in attrs.iteritems(): @@ -133,18 +133,18 @@ def _set_attrs(self, attrs): def _set_attr(self, name, value): - """Update one deployment attribute in the state file.""" + """Update one deployment attribute in the state.""" self._set_attrs({name: value}) def _del_attr(self, name): - """Delete a deployment attribute from the state file.""" + """Delete a deployment attribute from the state.""" with self._db: self._db.execute("delete from DeploymentAttrs where deployment = ? and name = ?", (self.uuid, name)) def _get_attr(self, name, default=nixops.util.undefined): - """Get a deployment attribute from the state file.""" + """Get a deployment attribute from the state.""" with self._db: c = self._db.cursor() c.execute("select value from DeploymentAttrs where deployment = ? and name = ?", (self.uuid, name)) @@ -189,7 +189,7 @@ def import_(self, attrs): def clone(self): with self._db: - new = self._statefile.create_deployment() + new = self._state.create_deployment() self._db.execute("insert into DeploymentAttrs (deployment, name, value) " + "select ?, name, value from DeploymentAttrs where deployment = ?", (new.uuid, self.uuid)) From c73773f28e42a89bbe3c93caf5d03bd570f1d502 Mon Sep 17 00:00:00 2001 From: Maarten Hoogendoorn Date: Mon, 13 Mar 2017 20:40:35 +0100 Subject: [PATCH 004/123] Move file based deployment lock to file state implementation. --- nixops/deployment.py | 24 +----------------------- nixops/state/file.py | 25 +++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 23 deletions(-) diff --git a/nixops/deployment.py b/nixops/deployment.py index 1c7091942..887b207aa 100644 --- a/nixops/deployment.py +++ b/nixops/deployment.py @@ -21,7 +21,6 @@ import getpass import traceback import glob -import fcntl import itertools import platform from nixops.util import ansi_success @@ -198,28 +197,7 @@ def clone(self): def _get_deployment_lock(self): - if self._lock_file_path is None: - lock_dir = os.environ.get("HOME", "") + "/.nixops/locks" - if not os.path.exists(lock_dir): os.makedirs(lock_dir, 0700) - self._lock_file_path = lock_dir + "/" + self.uuid - class DeploymentLock(object): - def __init__(self, depl): - self._lock_file_path = depl._lock_file_path - self._logger = depl.logger - self._lock_file = None - def __enter__(self): - self._lock_file = open(self._lock_file_path, "w") - fcntl.fcntl(self._lock_file, fcntl.F_SETFD, fcntl.FD_CLOEXEC) - try: - fcntl.flock(self._lock_file, fcntl.LOCK_EX | fcntl.LOCK_NB) - except IOError: - self._logger.log( - "waiting for exclusive deployment lock..." - ) - fcntl.flock(self._lock_file, fcntl.LOCK_EX) - def __exit__(self, exception_type, exception_value, exception_traceback): - self._lock_file.close() - return DeploymentLock(self) + return self._state.get_deployment_lock(self) def delete_resource(self, m): diff --git a/nixops/state/file.py b/nixops/state/file.py index 0e8c2c5f3..aa3650d0d 100644 --- a/nixops/state/file.py +++ b/nixops/state/file.py @@ -6,6 +6,7 @@ from pysqlite2 import dbapi2 as sqlite3 import sys import threading +import fcntl class Connection(sqlite3.Connection): @@ -161,6 +162,30 @@ def create_deployment(self, uuid=None): self._db.execute("insert into Deployments(uuid) values (?)", (uuid,)) return nixops.deployment.Deployment(self, uuid, sys.stderr) + def get_deployment_lock(self, deployment): + lock_dir = os.environ.get("HOME", "") + "/.nixops/locks" + if not os.path.exists(lock_dir): os.makedirs(lock_dir, 0700) + lock_file_path = lock_dir + "/" + deployment.uuid + class DeploymentLock(object): + def __init__(self, logger, path): + self._lock_file_path = path + self._logger = logger + self._lock_file = None + def __enter__(self): + self._lock_file = open(self._lock_file_path, "w") + fcntl.fcntl(self._lock_file, fcntl.F_SETFD, fcntl.FD_CLOEXEC) + try: + fcntl.flock(self._lock_file, fcntl.LOCK_EX | fcntl.LOCK_NB) + except IOError: + self._logger.log( + "waiting for exclusive deployment lock..." + ) + fcntl.flock(self._lock_file, fcntl.LOCK_EX) + def __exit__(self, exception_type, exception_value, exception_traceback): + if self._lock_file: + self._lock_file.close() + return DeploymentLock(deployment.logger, lock_file_path) + def _table_exists(self, c, table): c.execute("select 1 from sqlite_master where name = ? and type='table'", (table,)); return c.fetchone() != None From 8a4e34a14817160e1156d9c22d8b6ef74f1a1c0f Mon Sep 17 00:00:00 2001 From: Maarten Hoogendoorn Date: Mon, 13 Mar 2017 22:32:50 +0100 Subject: [PATCH 005/123] Fix indenting --- nixops/state/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nixops/state/__init__.py b/nixops/state/__init__.py index b1cc8d1f8..ed3fcdff1 100644 --- a/nixops/state/__init__.py +++ b/nixops/state/__init__.py @@ -16,7 +16,8 @@ def raise_(ex): raise ex switcher = { - "file": lambda(url): file.StateFile(url.path), + "file": lambda(url): file.StateFile(url.path), + "etcd": lambda(url): raise_(WrongStateSchemeException("coming soon!")), } function = switcher.get(scheme, lambda(url): raise_(WrongStateSchemeException("Unknown state scheme!"))) From bb056ea9c552197a2975859911124fe1e86c20b7 Mon Sep 17 00:00:00 2001 From: Maarten Hoogendoorn Date: Mon, 13 Mar 2017 22:51:50 +0100 Subject: [PATCH 006/123] Bite the bullet; remove _db references from business objects + rename it in state file implementation to __db --- nixops/deployment.py | 1 - nixops/state/file.py | 13 +++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/nixops/deployment.py b/nixops/deployment.py index 887b207aa..b8c79d273 100644 --- a/nixops/deployment.py +++ b/nixops/deployment.py @@ -50,7 +50,6 @@ class Deployment(object): def __init__(self, state, uuid, log_file=sys.stderr): self._state = state - self._db = state._db self.uuid = uuid self._last_log_prefix = None diff --git a/nixops/state/file.py b/nixops/state/file.py index aa3650d0d..416d9c727 100644 --- a/nixops/state/file.py +++ b/nixops/state/file.py @@ -101,14 +101,15 @@ def __init__(self, db_file): else: raise Exception("this NixOps version is too old to deal with schema version {0}".format(version)) - self._db = db + self.__db = db def close(self): - self._db.close() + self.__db.close() + def query_deployments(self): """Return the UUIDs of all deployments in the database.""" - c = self._db.cursor() + c = self.__db.cursor() c.execute("select uuid from Deployments") res = c.fetchall() return [x[0] for x in res] @@ -125,7 +126,7 @@ def get_all_deployments(self): return res def _find_deployment(self, uuid=None): - c = self._db.cursor() + c = self.__db.cursor() if not uuid: c.execute("select uuid from Deployments") else: @@ -158,8 +159,8 @@ def create_deployment(self, uuid=None): if not uuid: import uuid uuid = str(uuid.uuid1()) - with self._db: - self._db.execute("insert into Deployments(uuid) values (?)", (uuid,)) + with self.__db: + self.__db.execute("insert into Deployments(uuid) values (?)", (uuid,)) return nixops.deployment.Deployment(self, uuid, sys.stderr) def get_deployment_lock(self, deployment): From 6cc171a34e45dcda7eed4e3e24ac84b69c50252b Mon Sep 17 00:00:00 2001 From: Maarten Hoogendoorn Date: Mon, 13 Mar 2017 23:12:21 +0100 Subject: [PATCH 007/123] move get_resources_for --- nixops/deployment.py | 9 +-------- nixops/state/file.py | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/nixops/deployment.py b/nixops/deployment.py index b8c79d273..d815f1704 100644 --- a/nixops/deployment.py +++ b/nixops/deployment.py @@ -69,15 +69,8 @@ def __init__(self, state, uuid, log_file=sys.stderr): if not os.path.exists(self.expr_path): self.expr_path = os.path.dirname(__file__) + "/../nix" - self.resources = {} - with self._db: - c = self._db.cursor() - c.execute("select id, name, type from Resources where deployment = ?", (self.uuid,)) - for (id, name, type) in c.fetchall(): - r = _create_state(self, type, name, id) - self.resources[name] = r + self.resources = self._state.get_resources_for(self.uuid) self.logger.update_log_prefixes() - self.definitions = None diff --git a/nixops/state/file.py b/nixops/state/file.py index 416d9c727..fcff45ce4 100644 --- a/nixops/state/file.py +++ b/nixops/state/file.py @@ -106,6 +106,8 @@ def __init__(self, db_file): def close(self): self.__db.close() + ############################################################################################### + ## Deployment def query_deployments(self): """Return the UUIDs of all deployments in the database.""" @@ -163,6 +165,18 @@ def create_deployment(self, uuid=None): self.__db.execute("insert into Deployments(uuid) values (?)", (uuid,)) return nixops.deployment.Deployment(self, uuid, sys.stderr) + + def get_resources_for(self, deployment_uuid): + """Get all the resources for a certain deployment""" + resources = {} + with self._db: + c = self._db.cursor() + c.execute("select id, name, type from Resources where deployment = ?", (self.uuid,)) + for (id, name, type) in c.fetchall(): + r = _create_state(self, type, name, id) + resources[name] = r + return resources + def get_deployment_lock(self, deployment): lock_dir = os.environ.get("HOME", "") + "/.nixops/locks" if not os.path.exists(lock_dir): os.makedirs(lock_dir, 0700) From 813a231c3131cc00bcf4c4440fadfdaa9fcdd99e Mon Sep 17 00:00:00 2001 From: Maarten Hoogendoorn Date: Mon, 13 Mar 2017 23:16:12 +0100 Subject: [PATCH 008/123] move set_deployment_attrs --- nixops/deployment.py | 10 +--------- nixops/state/file.py | 11 +++++++++++ 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/nixops/deployment.py b/nixops/deployment.py index d815f1704..a5cb3a5cd 100644 --- a/nixops/deployment.py +++ b/nixops/deployment.py @@ -110,17 +110,9 @@ def get_machine(self, name): raise Exception("resource ‘{0}’ is not a machine".format(name)) return res - def _set_attrs(self, attrs): """Update deployment attributes in the state.""" - with self._db: - c = self._db.cursor() - for n, v in attrs.iteritems(): - if v == None: - c.execute("delete from DeploymentAttrs where deployment = ? and name = ?", (self.uuid, n)) - else: - c.execute("insert or replace into DeploymentAttrs(deployment, name, value) values (?, ?, ?)", - (self.uuid, n, v)) + self.state.set_deployment_attrs(self.uuid, attrs) def _set_attr(self, name, value): diff --git a/nixops/state/file.py b/nixops/state/file.py index fcff45ce4..d1d533f97 100644 --- a/nixops/state/file.py +++ b/nixops/state/file.py @@ -177,6 +177,17 @@ def get_resources_for(self, deployment_uuid): resources[name] = r return resources + def set_deployment_attrs(self, deployment_uuid, attrs): + """Update deployment attributes in the state.""" + with self._db: + c = self._db.cursor() + for n, v in attrs.iteritems(): + if v == None: + c.execute("delete from DeploymentAttrs where deployment = ? and name = ?", (deployment_uuid, n)) + else: + c.execute("insert or replace into DeploymentAttrs(deployment, name, value) values (?, ?, ?)", + (deployment_uuid, n, v)) + def get_deployment_lock(self, deployment): lock_dir = os.environ.get("HOME", "") + "/.nixops/locks" if not os.path.exists(lock_dir): os.makedirs(lock_dir, 0700) From 7b5893472cf235057dc40c8a20ac210158b6de41 Mon Sep 17 00:00:00 2001 From: Maarten Hoogendoorn Date: Mon, 13 Mar 2017 23:18:42 +0100 Subject: [PATCH 009/123] move del_deployment_attr --- nixops/deployment.py | 3 +-- nixops/state/file.py | 4 ++++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/nixops/deployment.py b/nixops/deployment.py index a5cb3a5cd..bccf5067a 100644 --- a/nixops/deployment.py +++ b/nixops/deployment.py @@ -122,8 +122,7 @@ def _set_attr(self, name, value): def _del_attr(self, name): """Delete a deployment attribute from the state.""" - with self._db: - self._db.execute("delete from DeploymentAttrs where deployment = ? and name = ?", (self.uuid, name)) + self.state.del_deployment_attr(self.uuid, name) def _get_attr(self, name, default=nixops.util.undefined): diff --git a/nixops/state/file.py b/nixops/state/file.py index d1d533f97..bd5ea7f82 100644 --- a/nixops/state/file.py +++ b/nixops/state/file.py @@ -188,6 +188,10 @@ def set_deployment_attrs(self, deployment_uuid, attrs): c.execute("insert or replace into DeploymentAttrs(deployment, name, value) values (?, ?, ?)", (deployment_uuid, n, v)) + def del_deployment_attr(self, deployment_uuid, attr_name): + with self._db: + self._db.execute("delete from DeploymentAttrs where deployment = ? and name = ?", (deployment_uuid, attr_name)) + def get_deployment_lock(self, deployment): lock_dir = os.environ.get("HOME", "") + "/.nixops/locks" if not os.path.exists(lock_dir): os.makedirs(lock_dir, 0700) From 7ed3fd3cc4b38ddb9133275b0ad3796f1ee034a2 Mon Sep 17 00:00:00 2001 From: Maarten Hoogendoorn Date: Mon, 13 Mar 2017 23:20:36 +0100 Subject: [PATCH 010/123] move get_deployment_attr --- nixops/deployment.py | 10 +++------- nixops/state/file.py | 10 ++++++++++ 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/nixops/deployment.py b/nixops/deployment.py index bccf5067a..a9cbce1a1 100644 --- a/nixops/deployment.py +++ b/nixops/deployment.py @@ -125,15 +125,11 @@ def _del_attr(self, name): self.state.del_deployment_attr(self.uuid, name) + #TODO(moretea): The default param does not appear to be used at all? + # Removed it when moving the body to nixops/state/file.py. def _get_attr(self, name, default=nixops.util.undefined): """Get a deployment attribute from the state.""" - with self._db: - c = self._db.cursor() - c.execute("select value from DeploymentAttrs where deployment = ? and name = ?", (self.uuid, name)) - row = c.fetchone() - if row != None: return row[0] - return nixops.util.undefined - + return self.state.get_deployment_attr(self.uuid, name) def _create_resource(self, name, type): c = self._db.cursor() diff --git a/nixops/state/file.py b/nixops/state/file.py index bd5ea7f82..60fa855a7 100644 --- a/nixops/state/file.py +++ b/nixops/state/file.py @@ -192,6 +192,16 @@ def del_deployment_attr(self, deployment_uuid, attr_name): with self._db: self._db.execute("delete from DeploymentAttrs where deployment = ? and name = ?", (deployment_uuid, attr_name)) + + def get_deployment_attr(self, deployment_uuid, name): + """Get a deployment attribute from the state.""" + with self._db: + c = self._db.cursor() + c.execute("select value from DeploymentAttrs where deployment = ? and name = ?", (deployment_uuid, name)) + row = c.fetchone() + if row != None: return row[0] + return nixops.util.undefined + def get_deployment_lock(self, deployment): lock_dir = os.environ.get("HOME", "") + "/.nixops/locks" if not os.path.exists(lock_dir): os.makedirs(lock_dir, 0700) From 0db686d24ebee8d86155d723d10189ba6aba4773 Mon Sep 17 00:00:00 2001 From: Maarten Hoogendoorn Date: Mon, 13 Mar 2017 23:35:29 +0100 Subject: [PATCH 011/123] move create_resource --- nixops/deployment.py | 20 +------------------- nixops/state/file.py | 27 +++++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 19 deletions(-) diff --git a/nixops/deployment.py b/nixops/deployment.py index a9cbce1a1..2d7368637 100644 --- a/nixops/deployment.py +++ b/nixops/deployment.py @@ -132,16 +132,7 @@ def _get_attr(self, name, default=nixops.util.undefined): return self.state.get_deployment_attr(self.uuid, name) def _create_resource(self, name, type): - c = self._db.cursor() - c.execute("select 1 from Resources where deployment = ? and name = ?", (self.uuid, name)) - if len(c.fetchall()) != 0: - raise Exception("resource already exists in database!") - c.execute("insert into Resources(deployment, name, type) values (?, ?, ?)", - (self.uuid, name, type)) - id = c.lastrowid - r = _create_state(self, type, name, id) - self.resources[name] = r - return r + return self.state.create_resource(self.uuid, name, type) def export(self): @@ -1144,15 +1135,6 @@ def _create_definition(xml, config, type_name): raise nixops.deployment.UnknownBackend("unknown resource type ‘{0}’".format(type_name)) -def _create_state(depl, type, name, id): - """Create a resource state object of the desired type.""" - - for cls in _subclasses(nixops.resources.ResourceState): - if type == cls.get_type(): - return cls(depl, name, id) - - raise nixops.deployment.UnknownBackend("unknown resource type ‘{0}’".format(type)) - # Automatically load all resource types. def _load_modules_from(dir): diff --git a/nixops/state/file.py b/nixops/state/file.py index 60fa855a7..6c78c8c94 100644 --- a/nixops/state/file.py +++ b/nixops/state/file.py @@ -226,6 +226,33 @@ def __exit__(self, exception_type, exception_value, exception_traceback): self._lock_file.close() return DeploymentLock(deployment.logger, lock_file_path) + ############################################################################################### + ## Resources + + def create_resource(self, deployment_uuid, name, type): + c = self._db.cursor() + c.execute("select 1 from Resources where deployment = ? and name = ?", (deployment_uuid, name)) + if len(c.fetchall()) != 0: + raise Exception("resource already exists in database!") + c.execute("insert into Resources(deployment, name, type) values (?, ?, ?)", + (deployment_uuid, name, type)) + id = c.lastrowid + r = _create_state(self, type, name, id) + self.resources[name] = r + return r + + + ### STATE + def _create_state(depl, type, name, id): + """Create a resource state object of the desired type.""" + + for cls in _subclasses(nixops.resources.ResourceState): + if type == cls.get_type(): + return cls(depl, name, id) + + raise nixops.deployment.UnknownBackend("unknown resource type ‘{0}’".format(type)) + + def _table_exists(self, c, table): c.execute("select 1 from sqlite_master where name = ? and type='table'", (table,)); return c.fetchone() != None From 97e0714a95b6653ee12e819cd9b6ce5369463c73 Mon Sep 17 00:00:00 2001 From: Maarten Hoogendoorn Date: Mon, 13 Mar 2017 23:38:51 +0100 Subject: [PATCH 012/123] Move most logic in export() to get_all_deployment_attrs in statefile. --- nixops/deployment.py | 10 +++------- nixops/state/file.py | 8 ++++++++ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/nixops/deployment.py b/nixops/deployment.py index 2d7368637..7a5c6ebfc 100644 --- a/nixops/deployment.py +++ b/nixops/deployment.py @@ -136,13 +136,9 @@ def _create_resource(self, name, type): def export(self): - with self._db: - c = self._db.cursor() - c.execute("select name, value from DeploymentAttrs where deployment = ?", (self.uuid,)) - rows = c.fetchall() - res = {row[0]: row[1] for row in rows} - res['resources'] = {r.name: r.export() for r in self.resources.itervalues()} - return res + res = self.state.get_all_deployment_attrs(self.uuid) + res['resources'] = {r.name: r.export() for r in self.resources.itervalues()} + return res def import_(self, attrs): diff --git a/nixops/state/file.py b/nixops/state/file.py index 6c78c8c94..22628db31 100644 --- a/nixops/state/file.py +++ b/nixops/state/file.py @@ -202,6 +202,14 @@ def get_deployment_attr(self, deployment_uuid, name): if row != None: return row[0] return nixops.util.undefined + def get_all_deployment_attrs(self, deployment_uuid): + with self._db: + c = self._db.cursor() + c.execute("select name, value from DeploymentAttrs where deployment = ?", (deployment_uuid)) + rows = c.fetchall() + res = {row[0]: row[1] for row in rows} + return res + def get_deployment_lock(self, deployment): lock_dir = os.environ.get("HOME", "") + "/.nixops/locks" if not os.path.exists(lock_dir): os.makedirs(lock_dir, 0700) From 584d6584e71f6c40b903441478b88278ad65a5c8 Mon Sep 17 00:00:00 2001 From: Maarten Hoogendoorn Date: Mon, 13 Mar 2017 23:39:40 +0100 Subject: [PATCH 013/123] Make import use an atomic transaction thingy. For now it's just the underlying DB, but it resembles something that can roll back the current transaction. --- nixops/deployment.py | 2 +- nixops/state/file.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/nixops/deployment.py b/nixops/deployment.py index 7a5c6ebfc..4fbdbbef4 100644 --- a/nixops/deployment.py +++ b/nixops/deployment.py @@ -142,7 +142,7 @@ def export(self): def import_(self, attrs): - with self._db: + with self.state.atomic: for k, v in attrs.iteritems(): if k == 'resources': continue self._set_attr(k, v) diff --git a/nixops/state/file.py b/nixops/state/file.py index 22628db31..609c5e065 100644 --- a/nixops/state/file.py +++ b/nixops/state/file.py @@ -103,6 +103,9 @@ def __init__(self, db_file): self.__db = db + # TODO; implement some other special wrapper that ONLY does the transaction stuff. + self.atomic = db + def close(self): self.__db.close() From 97bda1a790536d501033803465805f585e225a28 Mon Sep 17 00:00:00 2001 From: Maarten Hoogendoorn Date: Mon, 13 Mar 2017 23:47:25 +0100 Subject: [PATCH 014/123] move clone_deployment --- nixops/deployment.py | 8 +------- nixops/state/file.py | 10 ++++++++++ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/nixops/deployment.py b/nixops/deployment.py index 4fbdbbef4..b48a983c8 100644 --- a/nixops/deployment.py +++ b/nixops/deployment.py @@ -153,13 +153,7 @@ def import_(self, attrs): def clone(self): - with self._db: - new = self._state.create_deployment() - self._db.execute("insert into DeploymentAttrs (deployment, name, value) " + - "select ?, name, value from DeploymentAttrs where deployment = ?", - (new.uuid, self.uuid)) - new.configs_path = None - return new + return self.state.clone_deployment(self.uuid) def _get_deployment_lock(self): diff --git a/nixops/state/file.py b/nixops/state/file.py index 609c5e065..0a994ef7f 100644 --- a/nixops/state/file.py +++ b/nixops/state/file.py @@ -168,6 +168,16 @@ def create_deployment(self, uuid=None): self.__db.execute("insert into Deployments(uuid) values (?)", (uuid,)) return nixops.deployment.Deployment(self, uuid, sys.stderr) + def clone_deployment(self, deployment_uuid): + with self._db: + new = self.create_deployment() + self._db.execute("insert into DeploymentAttrs (deployment, name, value) " + + "select ?, name, value from DeploymentAttrs where deployment = ?", + (new.uuid, deployment_uuid)) + new.configs_path = None + return new + + def get_resources_for(self, deployment_uuid): """Get all the resources for a certain deployment""" From 77e9b932f4993cd7df3b8450cd54ef7e24118322 Mon Sep 17 00:00:00 2001 From: Maarten Hoogendoorn Date: Mon, 13 Mar 2017 23:50:39 +0100 Subject: [PATCH 015/123] move part of delete_resource --- nixops/deployment.py | 3 +-- nixops/state/file.py | 4 ++++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/nixops/deployment.py b/nixops/deployment.py index b48a983c8..956f5898e 100644 --- a/nixops/deployment.py +++ b/nixops/deployment.py @@ -162,8 +162,7 @@ def _get_deployment_lock(self): def delete_resource(self, m): del self.resources[m.name] - with self._db: - self._db.execute("delete from Resources where deployment = ? and id = ?", (self.uuid, m.id)) + self._state.delete_resource(self.uuid, m.id) def delete(self, force=False): diff --git a/nixops/state/file.py b/nixops/state/file.py index 0a994ef7f..e0cd0f75c 100644 --- a/nixops/state/file.py +++ b/nixops/state/file.py @@ -262,6 +262,10 @@ def create_resource(self, deployment_uuid, name, type): self.resources[name] = r return r + def delete_resource(self, deployment_uuid, res_id): + with self._db: + self._db.execute("delete from Resources where deployment = ? and id = ?", (deployment_uuid, res_id)) + ### STATE def _create_state(depl, type, name, id): From 4d33f93412be280df449e0792461afeed6959d18 Mon Sep 17 00:00:00 2001 From: Maarten Hoogendoorn Date: Mon, 13 Mar 2017 23:51:40 +0100 Subject: [PATCH 016/123] missed a few _'s --- nixops/deployment.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/nixops/deployment.py b/nixops/deployment.py index 956f5898e..2b209d6d4 100644 --- a/nixops/deployment.py +++ b/nixops/deployment.py @@ -112,7 +112,7 @@ def get_machine(self, name): def _set_attrs(self, attrs): """Update deployment attributes in the state.""" - self.state.set_deployment_attrs(self.uuid, attrs) + self._state.set_deployment_attrs(self.uuid, attrs) def _set_attr(self, name, value): @@ -122,27 +122,27 @@ def _set_attr(self, name, value): def _del_attr(self, name): """Delete a deployment attribute from the state.""" - self.state.del_deployment_attr(self.uuid, name) + self._state.del_deployment_attr(self.uuid, name) #TODO(moretea): The default param does not appear to be used at all? # Removed it when moving the body to nixops/state/file.py. def _get_attr(self, name, default=nixops.util.undefined): """Get a deployment attribute from the state.""" - return self.state.get_deployment_attr(self.uuid, name) + return self._state.get_deployment_attr(self.uuid, name) def _create_resource(self, name, type): - return self.state.create_resource(self.uuid, name, type) + return self._state.create_resource(self.uuid, name, type) def export(self): - res = self.state.get_all_deployment_attrs(self.uuid) + res = self._state.get_all_deployment_attrs(self.uuid) res['resources'] = {r.name: r.export() for r in self.resources.itervalues()} return res def import_(self, attrs): - with self.state.atomic: + with self._state.atomic: for k, v in attrs.iteritems(): if k == 'resources': continue self._set_attr(k, v) @@ -153,7 +153,7 @@ def import_(self, attrs): def clone(self): - return self.state.clone_deployment(self.uuid) + return self._state.clone_deployment(self.uuid) def _get_deployment_lock(self): From 5649e39e9e392ebe2cdcccccfd4a35b730e5ea39 Mon Sep 17 00:00:00 2001 From: Maarten Hoogendoorn Date: Tue, 14 Mar 2017 00:05:26 +0100 Subject: [PATCH 017/123] move part of delete to _delete_deployment --- nixops/deployment.py | 4 ++-- nixops/state/file.py | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/nixops/deployment.py b/nixops/deployment.py index 2b209d6d4..b14ab3f65 100644 --- a/nixops/deployment.py +++ b/nixops/deployment.py @@ -167,7 +167,7 @@ def delete_resource(self, m): def delete(self, force=False): """Delete this deployment from the state file.""" - with self._db: + with self._state.atomic: if not force and len(self.resources) > 0: raise Exception("cannot delete this deployment because it still has resources") @@ -178,7 +178,7 @@ def delete(self, force=False): if os.path.islink(p): os.remove(p) # Delete the deployment from the database. - self._db.execute("delete from Deployments where uuid = ?", (self.uuid,)) + self._state._delete_deployment(self.uuid) def _nix_path_flags(self): diff --git a/nixops/state/file.py b/nixops/state/file.py index e0cd0f75c..a98c9730b 100644 --- a/nixops/state/file.py +++ b/nixops/state/file.py @@ -168,6 +168,10 @@ def create_deployment(self, uuid=None): self.__db.execute("insert into Deployments(uuid) values (?)", (uuid,)) return nixops.deployment.Deployment(self, uuid, sys.stderr) + def _delete_deployment(self, deployment_uuid): + """NOTE: This is UNSAFE, it's guarded in nixops/deployment.py. Do not call this function except from there!""" + self._db.execute("delete from Deployments where uuid = ?", (self.uuid,)) + def clone_deployment(self, deployment_uuid): with self._db: new = self.create_deployment() From 40759017ef7cda168df6ac7c947740956a2e4e90 Mon Sep 17 00:00:00 2001 From: Maarten Hoogendoorn Date: Tue, 14 Mar 2017 00:06:57 +0100 Subject: [PATCH 018/123] use state atomic --- nixops/deployment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nixops/deployment.py b/nixops/deployment.py index b14ab3f65..3d3dabcf0 100644 --- a/nixops/deployment.py +++ b/nixops/deployment.py @@ -772,7 +772,7 @@ def evaluate_active(self, include=[], exclude=[], kill_obsolete=False): self.evaluate() # Create state objects for all defined resources. - with self._db: + with self.atomic: for m in self.definitions.itervalues(): if m.name not in self.resources: self._create_resource(m.name, m.get_type()) From 5f39278f61362d4de6a3bd7f6f5439c0655879a4 Mon Sep 17 00:00:00 2001 From: Maarten Hoogendoorn Date: Tue, 14 Mar 2017 00:07:27 +0100 Subject: [PATCH 019/123] Move part of logic of rename to _rename_resource --- nixops/deployment.py | 4 +--- nixops/state/file.py | 5 +++++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/nixops/deployment.py b/nixops/deployment.py index 3d3dabcf0..c9b80d106 100644 --- a/nixops/deployment.py +++ b/nixops/deployment.py @@ -1077,9 +1077,7 @@ def rename(self, name, new_name): m = self.resources.pop(name) self.resources[new_name] = m - - with self._db: - self._db.execute("update Resources set name = ? where deployment = ? and id = ?", (new_name, self.uuid, m.id)) + self._state._rename_resource(self.uuid, name, new_name) def send_keys(self, include=[], exclude=[]): diff --git a/nixops/state/file.py b/nixops/state/file.py index a98c9730b..90f6e3304 100644 --- a/nixops/state/file.py +++ b/nixops/state/file.py @@ -270,6 +270,11 @@ def delete_resource(self, deployment_uuid, res_id): with self._db: self._db.execute("delete from Resources where deployment = ? and id = ?", (deployment_uuid, res_id)) + def _rename_resource(self, deployment_uuid, current_name, new_name): + """NOTE: Invariants are checked in nixops/deployment.py#rename""" + with self._db: + self._db.execute("update Resources set name = ? where deployment = ? and id = ?", (new_name, self.uuid, m.id)) + ### STATE def _create_state(depl, type, name, id): From 6605128ff4abac68dd191d7c90b41f8e1597f55a Mon Sep 17 00:00:00 2001 From: Maarten Hoogendoorn Date: Tue, 14 Mar 2017 00:12:08 +0100 Subject: [PATCH 020/123] oh yeah, I did rename _db to __db! --- nixops/state/file.py | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/nixops/state/file.py b/nixops/state/file.py index 90f6e3304..634808359 100644 --- a/nixops/state/file.py +++ b/nixops/state/file.py @@ -170,12 +170,12 @@ def create_deployment(self, uuid=None): def _delete_deployment(self, deployment_uuid): """NOTE: This is UNSAFE, it's guarded in nixops/deployment.py. Do not call this function except from there!""" - self._db.execute("delete from Deployments where uuid = ?", (self.uuid,)) + self.__db.execute("delete from Deployments where uuid = ?", (self.uuid,)) def clone_deployment(self, deployment_uuid): - with self._db: + with self.__db: new = self.create_deployment() - self._db.execute("insert into DeploymentAttrs (deployment, name, value) " + + self.__db.execute("insert into DeploymentAttrs (deployment, name, value) " + "select ?, name, value from DeploymentAttrs where deployment = ?", (new.uuid, deployment_uuid)) new.configs_path = None @@ -186,8 +186,8 @@ def clone_deployment(self, deployment_uuid): def get_resources_for(self, deployment_uuid): """Get all the resources for a certain deployment""" resources = {} - with self._db: - c = self._db.cursor() + with self.__db: + c = self.__db.cursor() c.execute("select id, name, type from Resources where deployment = ?", (self.uuid,)) for (id, name, type) in c.fetchall(): r = _create_state(self, type, name, id) @@ -196,8 +196,8 @@ def get_resources_for(self, deployment_uuid): def set_deployment_attrs(self, deployment_uuid, attrs): """Update deployment attributes in the state.""" - with self._db: - c = self._db.cursor() + with self.__db: + c = self.__db.cursor() for n, v in attrs.iteritems(): if v == None: c.execute("delete from DeploymentAttrs where deployment = ? and name = ?", (deployment_uuid, n)) @@ -206,22 +206,22 @@ def set_deployment_attrs(self, deployment_uuid, attrs): (deployment_uuid, n, v)) def del_deployment_attr(self, deployment_uuid, attr_name): - with self._db: - self._db.execute("delete from DeploymentAttrs where deployment = ? and name = ?", (deployment_uuid, attr_name)) + with self.__db: + self.__db.execute("delete from DeploymentAttrs where deployment = ? and name = ?", (deployment_uuid, attr_name)) def get_deployment_attr(self, deployment_uuid, name): """Get a deployment attribute from the state.""" - with self._db: - c = self._db.cursor() + with self.__db: + c = self.__db.cursor() c.execute("select value from DeploymentAttrs where deployment = ? and name = ?", (deployment_uuid, name)) row = c.fetchone() if row != None: return row[0] return nixops.util.undefined def get_all_deployment_attrs(self, deployment_uuid): - with self._db: - c = self._db.cursor() + with self.__db: + c = self.__db.cursor() c.execute("select name, value from DeploymentAttrs where deployment = ?", (deployment_uuid)) rows = c.fetchall() res = {row[0]: row[1] for row in rows} @@ -255,7 +255,7 @@ def __exit__(self, exception_type, exception_value, exception_traceback): ## Resources def create_resource(self, deployment_uuid, name, type): - c = self._db.cursor() + c = self.__db.cursor() c.execute("select 1 from Resources where deployment = ? and name = ?", (deployment_uuid, name)) if len(c.fetchall()) != 0: raise Exception("resource already exists in database!") @@ -267,13 +267,13 @@ def create_resource(self, deployment_uuid, name, type): return r def delete_resource(self, deployment_uuid, res_id): - with self._db: - self._db.execute("delete from Resources where deployment = ? and id = ?", (deployment_uuid, res_id)) + with self.__db: + self.__db.execute("delete from Resources where deployment = ? and id = ?", (deployment_uuid, res_id)) def _rename_resource(self, deployment_uuid, current_name, new_name): """NOTE: Invariants are checked in nixops/deployment.py#rename""" - with self._db: - self._db.execute("update Resources set name = ? where deployment = ? and id = ?", (new_name, self.uuid, m.id)) + with self.__db: + self.__db.execute("update Resources set name = ? where deployment = ? and id = ?", (new_name, self.uuid, m.id)) ### STATE From 5881c0688f47d5ac57394f09efb0daff09c38e0c Mon Sep 17 00:00:00 2001 From: Maarten Hoogendoorn Date: Tue, 14 Mar 2017 00:14:08 +0100 Subject: [PATCH 021/123] Use atomic instead of _db --- scripts/nixops | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/nixops b/scripts/nixops index 195f29a80..47cd7eba7 100755 --- a/scripts/nixops +++ b/scripts/nixops @@ -499,7 +499,7 @@ def op_import(): for uuid, attrs in dump.iteritems(): if uuid in existing: raise Exception("state file already contains a deployment with UUID ‘{0}’".format(uuid)) - with state._db: + with state.atomic: depl = state.create_deployment(uuid=uuid) depl.import_(attrs) sys.stderr.write("added deployment ‘{0}’\n".format(uuid)) From de9d70027594afd58333ff6295b00da8b23d93e9 Mon Sep 17 00:00:00 2001 From: Maarten Hoogendoorn Date: Tue, 14 Mar 2017 00:14:56 +0100 Subject: [PATCH 022/123] Use atomic instead of _db --- nixops/backends/virtualbox.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nixops/backends/virtualbox.py b/nixops/backends/virtualbox.py index 493c3dedb..6996aa27d 100644 --- a/nixops/backends/virtualbox.py +++ b/nixops/backends/virtualbox.py @@ -195,7 +195,7 @@ def create(self, defn, check, allow_reboot, allow_recreate): # Generate a public/private host key. if not self.public_host_key: (private, public) = nixops.util.create_key_pair() - with self.depl._db: + with self.depl.atomic: self.public_host_key = public self.private_host_key = private @@ -204,7 +204,7 @@ def create(self, defn, check, allow_reboot, allow_recreate): # Backwards compatibility. if self.disk: - with self.depl._db: + with self.depl.atomic: self._update_disk("disk1", {"created": True, "path": self.disk, "attached": self.disk_attached, "port": 0}) From ff23c1c536d75b42a8d2dd8180a973de525d06f9 Mon Sep 17 00:00:00 2001 From: Maarten Hoogendoorn Date: Tue, 14 Mar 2017 00:18:28 +0100 Subject: [PATCH 023/123] move set_resource_attrs --- nixops/resources/__init__.py | 10 ++-------- nixops/state/file.py | 11 +++++++++++ 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/nixops/resources/__init__.py b/nixops/resources/__init__.py index 359f61278..bdb0aa9cd 100644 --- a/nixops/resources/__init__.py +++ b/nixops/resources/__init__.py @@ -64,14 +64,8 @@ def __init__(self, depl, name, id): def _set_attrs(self, attrs): """Update machine attributes in the state file.""" - with self.depl._db: - c = self.depl._db.cursor() - for n, v in attrs.iteritems(): - if v == None: - c.execute("delete from ResourceAttrs where machine = ? and name = ?", (self.id, n)) - else: - c.execute("insert or replace into ResourceAttrs(machine, name, value) values (?, ?, ?)", - (self.id, n, v)) + self.depl._state.set_resource_attrs(self.id, attrs) + def _set_attr(self, name, value): """Update one machine attribute in the state file.""" diff --git a/nixops/state/file.py b/nixops/state/file.py index 634808359..97bc30ce5 100644 --- a/nixops/state/file.py +++ b/nixops/state/file.py @@ -276,6 +276,17 @@ def _rename_resource(self, deployment_uuid, current_name, new_name): self.__db.execute("update Resources set name = ? where deployment = ? and id = ?", (new_name, self.uuid, m.id)) + def set_resource_attrs(self, resource_id, attrs): + with self.depl._db: + c = self.depl._db.cursor() + for n, v in attrs.iteritems(): + if v == None: + c.execute("delete from ResourceAttrs where machine = ? and name = ?", (resource_id, n)) + else: + c.execute("insert or replace into ResourceAttrs(machine, name, value) values (?, ?, ?)", + (resource_id, n, v)) + + ### STATE def _create_state(depl, type, name, id): """Create a resource state object of the desired type.""" From eb2a24a148233671d3c2071856a0ee085a792d61 Mon Sep 17 00:00:00 2001 From: Maarten Hoogendoorn Date: Tue, 14 Mar 2017 00:23:47 +0100 Subject: [PATCH 024/123] move del_resource_attr --- nixops/resources/__init__.py | 3 +-- nixops/state/file.py | 4 ++++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/nixops/resources/__init__.py b/nixops/resources/__init__.py index bdb0aa9cd..83c90330e 100644 --- a/nixops/resources/__init__.py +++ b/nixops/resources/__init__.py @@ -73,8 +73,7 @@ def _set_attr(self, name, value): def _del_attr(self, name): """Delete a machine attribute from the state file.""" - with self.depl._db: - self.depl._db.execute("delete from ResourceAttrs where machine = ? and name = ?", (self.id, name)) + self.depl._state.del_resource_attr(name) def _get_attr(self, name, default=nixops.util.undefined): """Get a machine attribute from the state file.""" diff --git a/nixops/state/file.py b/nixops/state/file.py index 97bc30ce5..61571bb39 100644 --- a/nixops/state/file.py +++ b/nixops/state/file.py @@ -286,6 +286,10 @@ def set_resource_attrs(self, resource_id, attrs): c.execute("insert or replace into ResourceAttrs(machine, name, value) values (?, ?, ?)", (resource_id, n, v)) + def del_resource_attr(self, resource_id, name): + with self.depl._db: + self.depl._db.execute("delete from ResourceAttrs where machine = ? and name = ?", (resource_id, name)) + ### STATE def _create_state(depl, type, name, id): From 47b3a436bf29f1e482481ef1e72cc259d6d796a4 Mon Sep 17 00:00:00 2001 From: Maarten Hoogendoorn Date: Tue, 14 Mar 2017 00:24:21 +0100 Subject: [PATCH 025/123] move get_resource_attr --- nixops/resources/__init__.py | 9 +++------ nixops/state/file.py | 8 ++++++++ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/nixops/resources/__init__.py b/nixops/resources/__init__.py index 83c90330e..f1b9373a4 100644 --- a/nixops/resources/__init__.py +++ b/nixops/resources/__init__.py @@ -75,14 +75,11 @@ def _del_attr(self, name): """Delete a machine attribute from the state file.""" self.depl._state.del_resource_attr(name) + #TODO(moretea): again, the default option appears to be defunct. + # Have removed it in state/file.py. def _get_attr(self, name, default=nixops.util.undefined): """Get a machine attribute from the state file.""" - with self.depl._db: - c = self.depl._db.cursor() - c.execute("select value from ResourceAttrs where machine = ? and name = ?", (self.id, name)) - row = c.fetchone() - if row != None: return row[0] - return nixops.util.undefined + return self.depl._state.get_resource_attr(self.id, name) def export(self): """Export the resource to move between databases""" diff --git a/nixops/state/file.py b/nixops/state/file.py index 61571bb39..3c132dcff 100644 --- a/nixops/state/file.py +++ b/nixops/state/file.py @@ -290,6 +290,14 @@ def del_resource_attr(self, resource_id, name): with self.depl._db: self.depl._db.execute("delete from ResourceAttrs where machine = ? and name = ?", (resource_id, name)) + def get_resource_attr(self, resource_id, name): + """Get a machine attribute from the state file.""" + with self.depl._db: + c = self.depl._db.cursor() + c.execute("select value from ResourceAttrs where machine = ? and name = ?", (resource_id, name)) + row = c.fetchone() + if row != None: return row[0] + return nixops.util.undefined ### STATE def _create_state(depl, type, name, id): From 6835823590ec0f7f5eff706d5e43f820c0e725bf Mon Sep 17 00:00:00 2001 From: Maarten Hoogendoorn Date: Tue, 14 Mar 2017 00:24:52 +0100 Subject: [PATCH 026/123] move part of export to get_all_resource_attrs --- nixops/resources/__init__.py | 10 +++------- nixops/state/file.py | 8 ++++++++ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/nixops/resources/__init__.py b/nixops/resources/__init__.py index f1b9373a4..1cfaac059 100644 --- a/nixops/resources/__init__.py +++ b/nixops/resources/__init__.py @@ -83,13 +83,9 @@ def _get_attr(self, name, default=nixops.util.undefined): def export(self): """Export the resource to move between databases""" - with self.depl._db: - c = self.depl._db.cursor() - c.execute("select name, value from ResourceAttrs where machine = ?", (self.id,)) - rows = c.fetchall() - res = {row[0]: row[1] for row in rows} - res['type'] = self.get_type() - return res + res = self.depl._state.get_all_resource_attrs(self.id) + res['type'] = self.get_type() + return res def import_(self, attrs): """Import the resource from another database""" diff --git a/nixops/state/file.py b/nixops/state/file.py index 3c132dcff..2823e8276 100644 --- a/nixops/state/file.py +++ b/nixops/state/file.py @@ -299,6 +299,14 @@ def get_resource_attr(self, resource_id, name): if row != None: return row[0] return nixops.util.undefined + def get_all_resource_attrs(self, resource_id): + with self.depl._db: + c = self.depl._db.cursor() + c.execute("select name, value from ResourceAttrs where machine = ?", (resource_id,)) + rows = c.fetchall() + res = {row[0]: row[1] for row in rows} + return res + ### STATE def _create_state(depl, type, name, id): """Create a resource state object of the desired type.""" From 3739051c7c2e328e9bde059df17fecd10fc4dbb7 Mon Sep 17 00:00:00 2001 From: Maarten Hoogendoorn Date: Tue, 14 Mar 2017 00:25:38 +0100 Subject: [PATCH 027/123] Use atomic instead of _db --- nixops/resources/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nixops/resources/__init__.py b/nixops/resources/__init__.py index 1cfaac059..51c2334eb 100644 --- a/nixops/resources/__init__.py +++ b/nixops/resources/__init__.py @@ -89,7 +89,7 @@ def export(self): def import_(self, attrs): """Import the resource from another database""" - with self.depl._db: + with self.depl.atomic: for k, v in attrs.iteritems(): if k == 'type': continue self._set_attr(k, v) From 66dee1ddcfecbcfb57be41eec8ed5dff9ab6e574 Mon Sep 17 00:00:00 2001 From: Maarten Hoogendoorn Date: Tue, 14 Mar 2017 00:26:41 +0100 Subject: [PATCH 028/123] oh yeah, I did rename _db to __db! --- nixops/state/file.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/nixops/state/file.py b/nixops/state/file.py index 2823e8276..08673c3ce 100644 --- a/nixops/state/file.py +++ b/nixops/state/file.py @@ -277,8 +277,8 @@ def _rename_resource(self, deployment_uuid, current_name, new_name): def set_resource_attrs(self, resource_id, attrs): - with self.depl._db: - c = self.depl._db.cursor() + with self.depl.__db: + c = self.depl.__db.cursor() for n, v in attrs.iteritems(): if v == None: c.execute("delete from ResourceAttrs where machine = ? and name = ?", (resource_id, n)) @@ -287,21 +287,21 @@ def set_resource_attrs(self, resource_id, attrs): (resource_id, n, v)) def del_resource_attr(self, resource_id, name): - with self.depl._db: - self.depl._db.execute("delete from ResourceAttrs where machine = ? and name = ?", (resource_id, name)) + with self.depl.__db: + self.depl.__db.execute("delete from ResourceAttrs where machine = ? and name = ?", (resource_id, name)) def get_resource_attr(self, resource_id, name): """Get a machine attribute from the state file.""" - with self.depl._db: - c = self.depl._db.cursor() + with self.depl.__db: + c = self.depl.__db.cursor() c.execute("select value from ResourceAttrs where machine = ? and name = ?", (resource_id, name)) row = c.fetchone() if row != None: return row[0] return nixops.util.undefined def get_all_resource_attrs(self, resource_id): - with self.depl._db: - c = self.depl._db.cursor() + with self.depl.__db: + c = self.depl.__db.cursor() c.execute("select name, value from ResourceAttrs where machine = ?", (resource_id,)) rows = c.fetchall() res = {row[0]: row[1] for row in rows} From 6254ccb2ce1ac1adc361708c6c3a3ca2669f8af6 Mon Sep 17 00:00:00 2001 From: Maarten Hoogendoorn Date: Tue, 14 Mar 2017 00:55:48 +0100 Subject: [PATCH 029/123] simple typo's --- nixops/backends/virtualbox.py | 2 +- nixops/deployment.py | 2 +- nixops/state/file.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/nixops/backends/virtualbox.py b/nixops/backends/virtualbox.py index 6996aa27d..8a1912109 100644 --- a/nixops/backends/virtualbox.py +++ b/nixops/backends/virtualbox.py @@ -195,7 +195,7 @@ def create(self, defn, check, allow_reboot, allow_recreate): # Generate a public/private host key. if not self.public_host_key: (private, public) = nixops.util.create_key_pair() - with self.depl.atomic: + with self.depl._state.atomic: self.public_host_key = public self.private_host_key = private diff --git a/nixops/deployment.py b/nixops/deployment.py index c9b80d106..57a792413 100644 --- a/nixops/deployment.py +++ b/nixops/deployment.py @@ -772,7 +772,7 @@ def evaluate_active(self, include=[], exclude=[], kill_obsolete=False): self.evaluate() # Create state objects for all defined resources. - with self.atomic: + with self._state.atomic: for m in self.definitions.itervalues(): if m.name not in self.resources: self._create_resource(m.name, m.get_type()) diff --git a/nixops/state/file.py b/nixops/state/file.py index 08673c3ce..b5bc84b14 100644 --- a/nixops/state/file.py +++ b/nixops/state/file.py @@ -277,8 +277,8 @@ def _rename_resource(self, deployment_uuid, current_name, new_name): def set_resource_attrs(self, resource_id, attrs): - with self.depl.__db: - c = self.depl.__db.cursor() + with self.__db: + c = self.__db.cursor() for n, v in attrs.iteritems(): if v == None: c.execute("delete from ResourceAttrs where machine = ? and name = ?", (resource_id, n)) From 2d1a822dcdf8aaf8fcfc0e0b17436ac58c797da7 Mon Sep 17 00:00:00 2001 From: Maarten Hoogendoorn Date: Tue, 14 Mar 2017 00:56:25 +0100 Subject: [PATCH 030/123] Need to pass in object; too much coupling to refactor right now --- nixops/deployment.py | 2 +- nixops/state/file.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/nixops/deployment.py b/nixops/deployment.py index 57a792413..c4cbbeb88 100644 --- a/nixops/deployment.py +++ b/nixops/deployment.py @@ -69,7 +69,7 @@ def __init__(self, state, uuid, log_file=sys.stderr): if not os.path.exists(self.expr_path): self.expr_path = os.path.dirname(__file__) + "/../nix" - self.resources = self._state.get_resources_for(self.uuid) + self.resources = self._state.get_resources_for(self) self.logger.update_log_prefixes() self.definitions = None diff --git a/nixops/state/file.py b/nixops/state/file.py index b5bc84b14..080e5faed 100644 --- a/nixops/state/file.py +++ b/nixops/state/file.py @@ -183,14 +183,14 @@ def clone_deployment(self, deployment_uuid): - def get_resources_for(self, deployment_uuid): + def get_resources_for(self, deployment): """Get all the resources for a certain deployment""" resources = {} with self.__db: c = self.__db.cursor() - c.execute("select id, name, type from Resources where deployment = ?", (self.uuid,)) + c.execute("select id, name, type from Resources where deployment = ?", (deployment.uuid,)) for (id, name, type) in c.fetchall(): - r = _create_state(self, type, name, id) + r = self._create_state(deployment, type, name, id) resources[name] = r return resources From 20c5e89651867934aa0e65b9070843f3e3ae5c19 Mon Sep 17 00:00:00 2001 From: Maarten Hoogendoorn Date: Tue, 14 Mar 2017 00:57:32 +0100 Subject: [PATCH 031/123] missed assigning to local dict --- nixops/deployment.py | 4 +++- nixops/state/file.py | 9 ++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/nixops/deployment.py b/nixops/deployment.py index c4cbbeb88..938e55750 100644 --- a/nixops/deployment.py +++ b/nixops/deployment.py @@ -132,7 +132,9 @@ def _get_attr(self, name, default=nixops.util.undefined): return self._state.get_deployment_attr(self.uuid, name) def _create_resource(self, name, type): - return self._state.create_resource(self.uuid, name, type) + r = self._state.create_resource(self, name, type) + self.resources[name] = r + return r def export(self): diff --git a/nixops/state/file.py b/nixops/state/file.py index 080e5faed..a054bc3e9 100644 --- a/nixops/state/file.py +++ b/nixops/state/file.py @@ -254,16 +254,15 @@ def __exit__(self, exception_type, exception_value, exception_traceback): ############################################################################################### ## Resources - def create_resource(self, deployment_uuid, name, type): + def create_resource(self, deployment, name, type): c = self.__db.cursor() - c.execute("select 1 from Resources where deployment = ? and name = ?", (deployment_uuid, name)) + c.execute("select 1 from Resources where deployment = ? and name = ?", (deployment.uuid, name)) if len(c.fetchall()) != 0: raise Exception("resource already exists in database!") c.execute("insert into Resources(deployment, name, type) values (?, ?, ?)", - (deployment_uuid, name, type)) + (deployment.uuid, name, type)) id = c.lastrowid - r = _create_state(self, type, name, id) - self.resources[name] = r + r = self._create_state(deployment, type, name, id) return r def delete_resource(self, deployment_uuid, res_id): From 6c37bbdd4b921e811115361f153d43c0a4654adc Mon Sep 17 00:00:00 2001 From: Maarten Hoogendoorn Date: Tue, 14 Mar 2017 00:57:50 +0100 Subject: [PATCH 032/123] typo --- nixops/resources/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nixops/resources/__init__.py b/nixops/resources/__init__.py index 51c2334eb..a1c3b9928 100644 --- a/nixops/resources/__init__.py +++ b/nixops/resources/__init__.py @@ -73,7 +73,7 @@ def _set_attr(self, name, value): def _del_attr(self, name): """Delete a machine attribute from the state file.""" - self.depl._state.del_resource_attr(name) + self.depl._state.del_resource_attr(self.id, name) #TODO(moretea): again, the default option appears to be defunct. # Have removed it in state/file.py. From 4baebf98c891efd5996fc3824241e526a10e91a0 Mon Sep 17 00:00:00 2001 From: Maarten Hoogendoorn Date: Tue, 14 Mar 2017 00:58:05 +0100 Subject: [PATCH 033/123] missing subclass function. Just copied for now --- nixops/state/file.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nixops/state/file.py b/nixops/state/file.py index a054bc3e9..62b32b886 100644 --- a/nixops/state/file.py +++ b/nixops/state/file.py @@ -8,6 +8,9 @@ import threading import fcntl +def _subclasses(cls): + sub = cls.__subclasses__() + return [cls] if not sub else [g for s in sub for g in _subclasses(s)] class Connection(sqlite3.Connection): From 44652be424ca2574ffe37938fc56a41e14a8f8a1 Mon Sep 17 00:00:00 2001 From: Maarten Hoogendoorn Date: Tue, 14 Mar 2017 00:58:23 +0100 Subject: [PATCH 034/123] typo --- nixops/state/file.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nixops/state/file.py b/nixops/state/file.py index 62b32b886..15f3eea0e 100644 --- a/nixops/state/file.py +++ b/nixops/state/file.py @@ -173,7 +173,7 @@ def create_deployment(self, uuid=None): def _delete_deployment(self, deployment_uuid): """NOTE: This is UNSAFE, it's guarded in nixops/deployment.py. Do not call this function except from there!""" - self.__db.execute("delete from Deployments where uuid = ?", (self.uuid,)) + self.__db.execute("delete from Deployments where uuid = ?", (deployment_uuid,)) def clone_deployment(self, deployment_uuid): with self.__db: From e2388740b451792c1f22013a21de0f9f10b8c1cb Mon Sep 17 00:00:00 2001 From: Maarten Hoogendoorn Date: Tue, 14 Mar 2017 00:58:52 +0100 Subject: [PATCH 035/123] locally reachable already --- nixops/state/file.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/nixops/state/file.py b/nixops/state/file.py index 15f3eea0e..b4911fbe7 100644 --- a/nixops/state/file.py +++ b/nixops/state/file.py @@ -289,21 +289,21 @@ def set_resource_attrs(self, resource_id, attrs): (resource_id, n, v)) def del_resource_attr(self, resource_id, name): - with self.depl.__db: - self.depl.__db.execute("delete from ResourceAttrs where machine = ? and name = ?", (resource_id, name)) + with self.__db: + self.__db.execute("delete from ResourceAttrs where machine = ? and name = ?", (resource_id, name)) def get_resource_attr(self, resource_id, name): """Get a machine attribute from the state file.""" - with self.depl.__db: - c = self.depl.__db.cursor() + with self.__db: + c = self.__db.cursor() c.execute("select value from ResourceAttrs where machine = ? and name = ?", (resource_id, name)) row = c.fetchone() if row != None: return row[0] return nixops.util.undefined def get_all_resource_attrs(self, resource_id): - with self.depl.__db: - c = self.depl.__db.cursor() + with self.__db: + c = self.__db.cursor() c.execute("select name, value from ResourceAttrs where machine = ?", (resource_id,)) rows = c.fetchall() res = {row[0]: row[1] for row in rows} From 56c39b7ee14606b44ffa9df063b1433b20e1e3ef Mon Sep 17 00:00:00 2001 From: Maarten Hoogendoorn Date: Tue, 14 Mar 2017 00:59:15 +0100 Subject: [PATCH 036/123] missed the self parameter --- nixops/state/file.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nixops/state/file.py b/nixops/state/file.py index b4911fbe7..22bc1bc47 100644 --- a/nixops/state/file.py +++ b/nixops/state/file.py @@ -310,7 +310,7 @@ def get_all_resource_attrs(self, resource_id): return res ### STATE - def _create_state(depl, type, name, id): + def _create_state(self, depl, type, name, id): """Create a resource state object of the desired type.""" for cls in _subclasses(nixops.resources.ResourceState): From 83b5ca1249df8983d20a93c2d26502c361a4c4e8 Mon Sep 17 00:00:00 2001 From: Maarten Hoogendoorn Date: Tue, 14 Mar 2017 01:24:55 +0100 Subject: [PATCH 037/123] use _state.atomic instead of _db --- nixops/backends/ec2.py | 18 +++++++++--------- nixops/backends/hetzner.py | 2 +- nixops/resources/cloudwatch_log_group.py | 6 +++--- nixops/resources/cloudwatch_log_stream.py | 6 +++--- nixops/resources/datadog-monitor.py | 2 +- nixops/resources/datadog-screenboard.py | 2 +- nixops/resources/datadog-timeboard.py | 2 +- nixops/resources/ebs_volume.py | 2 +- nixops/resources/ec2_keypair.py | 4 ++-- nixops/resources/ec2_placement_group.py | 6 +++--- nixops/resources/ec2_rds_dbinstance.py | 8 ++++---- nixops/resources/ec2_security_group.py | 6 +++--- nixops/resources/elastic_file_system.py | 4 ++-- .../elastic_file_system_mount_target.py | 4 ++-- nixops/resources/elastic_ip.py | 2 +- nixops/resources/iam_role.py | 4 ++-- nixops/resources/s3_bucket.py | 2 +- nixops/resources/sns_topic.py | 4 ++-- nixops/resources/sqs_queue.py | 4 ++-- nixops/resources/ssh_keypair.py | 2 +- 20 files changed, 45 insertions(+), 45 deletions(-) diff --git a/nixops/backends/ec2.py b/nixops/backends/ec2.py index c41959878..5919be5b5 100644 --- a/nixops/backends/ec2.py +++ b/nixops/backends/ec2.py @@ -124,7 +124,7 @@ def __init__(self, depl, name, id): def _reset_state(self): """Discard all state pertaining to an instance.""" - with self.depl._db: + with self.depl._state.atomic: self.state = MachineState.MISSING self.associate_public_ip_address = None self.use_private_ip_address = None @@ -343,7 +343,7 @@ def _instance_ip_ready(ins): self.log_end("{0} / {1}".format(instance.ip_address, instance.private_ip_address)) - with self.depl._db: + with self.depl._state.atomic: self.private_ipv4 = instance.private_ip_address self.public_ipv4 = instance.ip_address self.public_dns_name = instance.public_dns_name @@ -585,7 +585,7 @@ def _assign_elastic_ip(self, elastic_ipv4, check): nixops.known_hosts.update(self.public_ipv4, elastic_ipv4, self.public_host_key) - with self.depl._db: + with self.depl._state.atomic: self.elastic_ipv4 = elastic_ipv4 self.public_ipv4 = elastic_ipv4 self.ssh_pinged = False @@ -598,7 +598,7 @@ def _assign_elastic_ip(self, elastic_ipv4, check): else: self.log("address ‘{0}’ was not associated with instance ‘{1}’".format(self.elastic_ipv4, self.vm_id)) - with self.depl._db: + with self.depl._state.atomic: self.elastic_ipv4 = None self.public_ipv4 = None self.ssh_pinged = False @@ -662,7 +662,7 @@ def create_instance(self, defn, zone, devmap, user_data, ebs_optimized): lambda: self._conn.request_spot_instances(price=defn.spot_instance_price/100.0, **common_args) )[0] - with self.depl._db: + with self.depl._state.atomic: self.spot_instance_price = defn.spot_instance_price self.spot_instance_request_id = request.id @@ -693,7 +693,7 @@ def create_instance(self, defn, zone, devmap, user_data, ebs_optimized): # the instance ID, we'll get the same instance ID on the # next run. if not self.client_token: - with self.depl._db: + with self.depl._state.atomic: self.client_token = nixops.util.generate_random_string(length=48) # = 64 ASCII chars self.state = self.STARTING @@ -892,7 +892,7 @@ def create(self, defn, check, allow_reboot, allow_recreate): # Generate a public/private host key. if not self.public_host_key: (private, public) = nixops.util.create_key_pair(type=defn.host_key_type()) - with self.depl._db: + with self.depl._state.atomic: self.public_host_key = public self.private_host_key = private @@ -902,7 +902,7 @@ def create(self, defn, check, allow_reboot, allow_recreate): instance = self.create_instance(defn, zone, devmap, user_data, ebs_optimized) - with self.depl._db: + with self.depl._state.atomic: self.vm_id = instance.id self.ami = defn.ami self.instance_type = defn.instance_type @@ -970,7 +970,7 @@ def create(self, defn, check, allow_reboot, allow_recreate): elastic_ipv4 = res.public_ipv4 self._assign_elastic_ip(elastic_ipv4, check) - with self.depl._db: + with self.depl._state.atomic: self.use_private_ip_address = defn.use_private_ip_address self.associate_public_ip_address = defn.associate_public_ip_address diff --git a/nixops/backends/hetzner.py b/nixops/backends/hetzner.py index 320c1accd..69343d09e 100644 --- a/nixops/backends/hetzner.py +++ b/nixops/backends/hetzner.py @@ -584,7 +584,7 @@ def create(self, defn, check, allow_reboot, allow_recreate): "‘{0}’... ".format(self.name)) # Create a new Admin account exclusively for this machine. server = self._get_server_from_main_robot(self.main_ipv4, defn) - with self.depl._db: + with self.depl._state.atomic: (self.robot_admin_user, self.robot_admin_pass) = server.admin.create() self.log_end("done. ({0})".format(self.robot_admin_user)) diff --git a/nixops/resources/cloudwatch_log_group.py b/nixops/resources/cloudwatch_log_group.py index e53827284..353a9716b 100644 --- a/nixops/resources/cloudwatch_log_group.py +++ b/nixops/resources/cloudwatch_log_group.py @@ -72,7 +72,7 @@ def _destroy(self): self._conn.delete_log_group(self.log_group_name) except boto.logs.exceptions.ResourceNotFoundException, e: self.log("the log group ‘{0}’ was already deleted".format(self.log_group_name)) - with self.depl._db: + with self.depl._state.atomic: self.state = self.MISSING self.log_group_name = None self.region = None @@ -114,7 +114,7 @@ def create(self, defn, check, allow_reboot, allow_recreate): self.log("setting the retention in days of '{0}' to '{1}'".format(defn.config['name'], defn.config['retentionInDays'])) self._conn.set_retention(log_group_name=defn.config['name'],retention_in_days=defn.config['retentionInDays']) - with self.depl._db: + with self.depl._state.atomic: self.state = self.UP self.log_group_name = defn.config['name'] self.region = defn.config['region'] @@ -123,4 +123,4 @@ def create(self, defn, check, allow_reboot, allow_recreate): def destroy(self, wipe=False): self._destroy() - return True \ No newline at end of file + return True diff --git a/nixops/resources/cloudwatch_log_stream.py b/nixops/resources/cloudwatch_log_stream.py index c53c71dd8..5e0934411 100644 --- a/nixops/resources/cloudwatch_log_stream.py +++ b/nixops/resources/cloudwatch_log_stream.py @@ -72,7 +72,7 @@ def _destroy(self): self._conn.delete_log_stream(log_group_name=self.log_group_name,log_stream_name=self.log_stream_name) except boto.logs.exceptions.ResourceNotFoundException, e: self.log("the log group ‘{0}’ or log stream ‘{1}’ was already deleted".format(self.log_group_name,self.log_stream_name)) - with self.depl._db: + with self.depl._state.atomic: self.state = self.MISSING self.log_group_name = None self.log_stream_name = None @@ -121,7 +121,7 @@ def create(self, defn, check, allow_reboot, allow_recreate): exist, arn = self.lookup_cloudwatch_log_stream(log_group_name=defn.config['logGroupName'], log_stream_name=defn.config['name']) - with self.depl._db: + with self.depl._state.atomic: self.state = self.UP self.log_stream_name = defn.config['name'] self.log_group_name = defn.config['logGroupName'] @@ -130,4 +130,4 @@ def create(self, defn, check, allow_reboot, allow_recreate): def destroy(self, wipe=False): self._destroy() - return True \ No newline at end of file + return True diff --git a/nixops/resources/datadog-monitor.py b/nixops/resources/datadog-monitor.py index 11756e6f5..64c5c39ff 100644 --- a/nixops/resources/datadog-monitor.py +++ b/nixops/resources/datadog-monitor.py @@ -106,7 +106,7 @@ def create(self, defn, check, allow_reboot, allow_recreate): if 'errors' in response: raise Exception(str(response['errors'])) - with self.depl._db: + with self.depl._state.atomic: self.state = self.UP self.api_key = defn.config['apiKey'] self.app_key = defn.config['appKey'] diff --git a/nixops/resources/datadog-screenboard.py b/nixops/resources/datadog-screenboard.py index bcf51c982..102d2bd03 100644 --- a/nixops/resources/datadog-screenboard.py +++ b/nixops/resources/datadog-screenboard.py @@ -104,7 +104,7 @@ def create(self, defn, check, allow_reboot, allow_recreate): if 'errors' in response: raise Exception(str(response['errors'])) - with self.depl._db: + with self.depl._state.atomic: self.state = self.UP self.api_key = defn.config['apiKey'] self.app_key = defn.config['appKey'] diff --git a/nixops/resources/datadog-timeboard.py b/nixops/resources/datadog-timeboard.py index b0079624e..6da4e5240 100644 --- a/nixops/resources/datadog-timeboard.py +++ b/nixops/resources/datadog-timeboard.py @@ -111,7 +111,7 @@ def create(self, defn, check, allow_reboot, allow_recreate): else: url = response['url'] - with self.depl._db: + with self.depl._state.atomic: self.state = self.UP self.api_key = defn.config['apiKey'] self.app_key = defn.config['appKey'] diff --git a/nixops/resources/ebs_volume.py b/nixops/resources/ebs_volume.py index ec1044e16..a23c4b63d 100644 --- a/nixops/resources/ebs_volume.py +++ b/nixops/resources/ebs_volume.py @@ -122,7 +122,7 @@ def create(self, defn, check, allow_reboot, allow_recreate): # volume we just created. Doesn't seem to be anything we # can do about this. - with self.depl._db: + with self.depl._state.atomic: self.state = self.STARTING self.region = defn.region self.zone = defn.zone diff --git a/nixops/resources/ec2_keypair.py b/nixops/resources/ec2_keypair.py index 2e5d1be01..e7dc5fc7b 100644 --- a/nixops/resources/ec2_keypair.py +++ b/nixops/resources/ec2_keypair.py @@ -78,7 +78,7 @@ def create(self, defn, check, allow_reboot, allow_recreate): # Generate the key pair locally. if not self.public_key: (private, public) = nixops.util.create_key_pair(type="rsa") # EC2 only supports RSA keys. - with self.depl._db: + with self.depl._state.atomic: self.public_key = public self.private_key = private @@ -102,7 +102,7 @@ def create(self, defn, check, allow_reboot, allow_recreate): self.log("uploading EC2 key pair ‘{0}’...".format(defn.keypair_name)) self._conn.import_key_pair(defn.keypair_name, self.public_key) - with self.depl._db: + with self.depl._state.atomic: self.state = self.UP self.keypair_name = defn.keypair_name diff --git a/nixops/resources/ec2_placement_group.py b/nixops/resources/ec2_placement_group.py index 511e52370..5bc3dbd3a 100644 --- a/nixops/resources/ec2_placement_group.py +++ b/nixops/resources/ec2_placement_group.py @@ -67,11 +67,11 @@ def _connect(self): def create(self, defn, check, allow_reboot, allow_recreate): # Name or region change means a completely new security group if self.placement_group_name and (defn.placement_group_name != self.placement_group_name or defn.region != self.region): - with self.depl._db: + with self.depl._state.atomic: self.state = self.UNKNOWN self.old_placement_groups = self.old_placement_groups + [{'name': self.placement_group_name, 'region': self.region}] - with self.depl._db: + with self.depl._state.atomic: self.region = defn.region self.access_key_id = defn.access_key_id or nixops.ec2_utils.get_access_key_id() self.placement_group_name = defn.placement_group_name @@ -79,7 +79,7 @@ def create(self, defn, check, allow_reboot, allow_recreate): grp = None if check: - with self.depl._db: + with self.depl._state.atomic: self._connect() try: diff --git a/nixops/resources/ec2_rds_dbinstance.py b/nixops/resources/ec2_rds_dbinstance.py index d314f006d..0c814d338 100644 --- a/nixops/resources/ec2_rds_dbinstance.py +++ b/nixops/resources/ec2_rds_dbinstance.py @@ -148,7 +148,7 @@ def _wait_for_dbinstance(self, dbinstance, state='available'): time.sleep(6) def _copy_dbinstance_attrs(self, dbinstance): - with self.depl._db: + with self.depl._state.atomic: self.rds_dbinstance_id = dbinstance.id self.rds_dbinstance_allocated_storage = int(dbinstance.allocated_storage) self.rds_dbinstance_instance_class = dbinstance.instance_class @@ -170,7 +170,7 @@ def _compare_instance_id(self, instance_id): return unicode(self.rds_dbinstance_id).lower() == unicode(instance_id).lower() def create(self, defn, check, allow_reboot, allow_recreate): - with self.depl._db: + with self.depl._state.atomic: self.access_key_id = defn.access_key_id or nixops.ec2_utils.get_access_key_id() if not self.access_key_id: raise Exception("please set ‘accessKeyId’, $EC2_ACCESS_KEY or $AWS_ACCESS_KEY_ID") @@ -192,7 +192,7 @@ def create(self, defn, check, allow_reboot, allow_recreate): dbinstance = self._try_fetch_dbinstance(self.rds_dbinstance_id) - with self.depl._db: + with self.depl._state.atomic: if check or self.state == self.MISSING or self.state == self.UNKNOWN: if dbinstance and (self.state == self.MISSING or self.state == self.UNKNOWN): if dbinstance.status == 'deleting': @@ -233,7 +233,7 @@ def create(self, defn, check, allow_reboot, allow_recreate): self._copy_dbinstance_attrs(dbinstance) self.state = self.UP - with self.depl._db: + with self.depl._state.atomic: if self.state == self.UP and self._diff_defn(defn): if dbinstance is None: raise Exception("state is UP but database instance does not exist. re-run with --check option to synchronize states") diff --git a/nixops/resources/ec2_security_group.py b/nixops/resources/ec2_security_group.py index 832ed8c2a..d8b2ec94c 100644 --- a/nixops/resources/ec2_security_group.py +++ b/nixops/resources/ec2_security_group.py @@ -96,11 +96,11 @@ def _connect(self): def create(self, defn, check, allow_reboot, allow_recreate): # Name or region change means a completely new security group if self.security_group_name and (defn.security_group_name != self.security_group_name or defn.region != self.region): - with self.depl._db: + with self.depl._state.atomic: self.state = self.UNKNOWN self.old_security_groups = self.old_security_groups + [{'name': self.security_group_name, 'region': self.region}] - with self.depl._db: + with self.depl._state.atomic: self.region = defn.region self.access_key_id = defn.access_key_id or nixops.ec2_utils.get_access_key_id() self.security_group_name = defn.security_group_name @@ -109,7 +109,7 @@ def create(self, defn, check, allow_reboot, allow_recreate): grp = None if check: - with self.depl._db: + with self.depl._state.atomic: self._connect() try: diff --git a/nixops/resources/elastic_file_system.py b/nixops/resources/elastic_file_system.py index d4b23b4e3..ee80f227a 100644 --- a/nixops/resources/elastic_file_system.py +++ b/nixops/resources/elastic_file_system.py @@ -85,7 +85,7 @@ def create(self, defn, check, allow_reboot, allow_recreate): if len(fss) == 1: fs = fss[0] if fs["LifeCycleState"] == "available": - with self.depl._db: + with self.depl._state.atomic: self.state = self.UP self.fs_id = fs["FileSystemId"] self.region = defn.config["region"] @@ -146,7 +146,7 @@ def destroy(self, wipe=False): break time.sleep(1) - with self.depl._db: + with self.depl._state.atomic: self.state = self.MISSING self.fs_id = None self.region = None diff --git a/nixops/resources/elastic_file_system_mount_target.py b/nixops/resources/elastic_file_system_mount_target.py index 74033a5c7..81d87a5f1 100644 --- a/nixops/resources/elastic_file_system_mount_target.py +++ b/nixops/resources/elastic_file_system_mount_target.py @@ -41,7 +41,7 @@ def get_type(cls): return "elastic-file-system-mount-target" def _reset_state(self): - with self.depl._db: + with self.depl._state.atomic: self.state = self.MISSING self.access_key_id = None self.region = None @@ -94,7 +94,7 @@ def create(self, defn, check, allow_reboot, allow_recreate): args["IpAddress"] = defn.config["ipAddress"] res = client.create_mount_target(FileSystemId=fs_id, SubnetId=defn.config["subnet"], **args) - with self.depl._db: + with self.depl._state.atomic: self.state = self.STARTING self.fsmt_id = res["MountTargetId"] self.fs_id = fs_id diff --git a/nixops/resources/elastic_ip.py b/nixops/resources/elastic_ip.py index 6528532fa..f907ca299 100644 --- a/nixops/resources/elastic_ip.py +++ b/nixops/resources/elastic_ip.py @@ -84,7 +84,7 @@ def create(self, defn, check, allow_reboot, allow_recreate): # address we just created. Doesn't seem to be anything we # can do about this. - with self.depl._db: + with self.depl._state.atomic: self.state = self.UP self.region = defn.region self.public_ipv4 = address.public_ip diff --git a/nixops/resources/iam_role.py b/nixops/resources/iam_role.py index 8f055d7ee..b0ae6a5f1 100644 --- a/nixops/resources/iam_role.py +++ b/nixops/resources/iam_role.py @@ -105,7 +105,7 @@ def _destroy(self): self.log("could not find instance profile") - with self.depl._db: + with self.depl._state.atomic: self.state = self.MISSING self.role_name = None self.access_key_id = None @@ -167,7 +167,7 @@ def create(self, defn, check, allow_reboot, allow_recreate): if defn.assume_role_policy != "": self._conn.update_assume_role_policy(defn.role_name, defn.assume_role_policy) - with self.depl._db: + with self.depl._state.atomic: self.state = self.UP self.role_name = defn.role_name self.policy = defn.policy diff --git a/nixops/resources/s3_bucket.py b/nixops/resources/s3_bucket.py index 986ec645f..fd993850b 100644 --- a/nixops/resources/s3_bucket.py +++ b/nixops/resources/s3_bucket.py @@ -101,7 +101,7 @@ def create(self, defn, check, allow_reboot, allow_recreate): # [http://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketDELETEpolicy.html] if e.status != 204: raise # (204 : Bucket didn't have any policy to delete) - with self.depl._db: + with self.depl._state.atomic: self.state = self.UP self.bucket_name = defn.bucket_name self.region = defn.region diff --git a/nixops/resources/sns_topic.py b/nixops/resources/sns_topic.py index e454b7821..e10cdad33 100644 --- a/nixops/resources/sns_topic.py +++ b/nixops/resources/sns_topic.py @@ -74,7 +74,7 @@ def _destroy(self): self.connect() self.log("destroying SNS topic ‘{0}’...".format(self.topic_name)) self._conn.delete_topic(self.arn) - with self.depl._db: + with self.depl._state.atomic: self.state = self.MISSING self.topic_name = None self.region = None @@ -136,7 +136,7 @@ def create(self, defn, check, allow_reboot, allow_recreate): if subscriber_arn != "PendingConfirmation": self._conn.unsubscribe(subscription=subscriber_arn) - with self.depl._db: + with self.depl._state.atomic: self.state = self.UP self.topic_name = defn.config['name'] self.display_name = defn.config['displayName'] diff --git a/nixops/resources/sqs_queue.py b/nixops/resources/sqs_queue.py index 4ad53ba8d..06596924c 100644 --- a/nixops/resources/sqs_queue.py +++ b/nixops/resources/sqs_queue.py @@ -84,7 +84,7 @@ def _destroy(self): if q: self.log("destroying SQS queue ‘{0}’...".format(self.queue_name)) self._conn.delete_queue(q) - with self.depl._db: + with self.depl._state.atomic: self.state = self.MISSING self.queue_name = None self.queue_base_name = None @@ -122,7 +122,7 @@ def create(self, defn, check, allow_reboot, allow_recreate): self.log("creating SQS queue ‘{0}’...".format(defn.queue_name)) q = nixops.ec2_utils.retry(lambda: self._conn.create_queue(defn.queue_name, defn.visibility_timeout), error_codes = ['AWS.SimpleQueueService.QueueDeletedRecently']) - with self.depl._db: + with self.depl._state.atomic: self.state = self.UP self.queue_name = defn.queue_name self.url = q.url diff --git a/nixops/resources/ssh_keypair.py b/nixops/resources/ssh_keypair.py index 63fe78164..612df9261 100644 --- a/nixops/resources/ssh_keypair.py +++ b/nixops/resources/ssh_keypair.py @@ -46,7 +46,7 @@ def create(self, defn, check, allow_reboot, allow_recreate): # Generate the key pair locally. if not self.public_key: (private, public) = nixops.util.create_key_pair(type="ed25519") - with self.depl._db: + with self.depl._state.atomic: self.public_key = public self.private_key = private self.state = state = nixops.resources.ResourceState.UP From abf3d6ebbbe8cdd49a1c5a399e707043bd7471b7 Mon Sep 17 00:00:00 2001 From: Maarten Hoogendoorn Date: Tue, 14 Mar 2017 12:35:14 +0100 Subject: [PATCH 038/123] schema: file -> sqlite3 --- nixops/state/__init__.py | 7 +++---- nixops/state/{file.py => sqlite3_file.py} | 0 2 files changed, 3 insertions(+), 4 deletions(-) rename nixops/state/{file.py => sqlite3_file.py} (100%) diff --git a/nixops/state/__init__.py b/nixops/state/__init__.py index ed3fcdff1..3e72e6d6e 100644 --- a/nixops/state/__init__.py +++ b/nixops/state/__init__.py @@ -1,6 +1,6 @@ import urlparse import sys -import file +import sqlite3_file class WrongStateSchemeException(Exception): pass @@ -10,14 +10,13 @@ def open(url): scheme = url.scheme if scheme == "": - scheme = "file" + scheme = "sqlite3" def raise_(ex): raise ex switcher = { - "file": lambda(url): file.StateFile(url.path), - "etcd": lambda(url): raise_(WrongStateSchemeException("coming soon!")), + "sqlite3": lambda(url): sqlite3_file.StateFile(url.path), } function = switcher.get(scheme, lambda(url): raise_(WrongStateSchemeException("Unknown state scheme!"))) diff --git a/nixops/state/file.py b/nixops/state/sqlite3_file.py similarity index 100% rename from nixops/state/file.py rename to nixops/state/sqlite3_file.py From 0d999cf1559381af076be11ad656832207a13f22 Mon Sep 17 00:00:00 2001 From: Maarten Hoogendoorn Date: Tue, 14 Mar 2017 16:03:59 +0100 Subject: [PATCH 039/123] Fix _rename_resource, needs resource_id --- nixops/deployment.py | 2 +- nixops/state/sqlite3_file.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/nixops/deployment.py b/nixops/deployment.py index 938e55750..bf3f3bf87 100644 --- a/nixops/deployment.py +++ b/nixops/deployment.py @@ -1079,7 +1079,7 @@ def rename(self, name, new_name): m = self.resources.pop(name) self.resources[new_name] = m - self._state._rename_resource(self.uuid, name, new_name) + self._state._rename_resource(self.uuid, m.id, new_name) def send_keys(self, include=[], exclude=[]): diff --git a/nixops/state/sqlite3_file.py b/nixops/state/sqlite3_file.py index 22bc1bc47..b053b5ff7 100644 --- a/nixops/state/sqlite3_file.py +++ b/nixops/state/sqlite3_file.py @@ -272,10 +272,10 @@ def delete_resource(self, deployment_uuid, res_id): with self.__db: self.__db.execute("delete from Resources where deployment = ? and id = ?", (deployment_uuid, res_id)) - def _rename_resource(self, deployment_uuid, current_name, new_name): + def _rename_resource(self, deployment_uuid, resource_id, new_name): """NOTE: Invariants are checked in nixops/deployment.py#rename""" with self.__db: - self.__db.execute("update Resources set name = ? where deployment = ? and id = ?", (new_name, self.uuid, m.id)) + self.__db.execute("update Resources set name = ? where deployment = ? and id = ?", (new_name, deployment_uuid, resource_id)) def set_resource_attrs(self, resource_id, attrs): From a08de6265aab0704c887d80abacf2ec809211f98 Mon Sep 17 00:00:00 2001 From: Maarten Hoogendoorn Date: Tue, 14 Mar 2017 16:04:57 +0100 Subject: [PATCH 040/123] WIP: initial json backend. NOTE: does not work yet! --- nixops/state/__init__.py | 2 + nixops/state/json_file.py | 365 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 367 insertions(+) create mode 100644 nixops/state/json_file.py diff --git a/nixops/state/__init__.py b/nixops/state/__init__.py index 3e72e6d6e..214c19009 100644 --- a/nixops/state/__init__.py +++ b/nixops/state/__init__.py @@ -1,6 +1,7 @@ import urlparse import sys import sqlite3_file +import json_file class WrongStateSchemeException(Exception): pass @@ -17,6 +18,7 @@ def raise_(ex): switcher = { "sqlite3": lambda(url): sqlite3_file.StateFile(url.path), + "json": lambda(url): json_file.JsonFile(url.path), } function = switcher.get(scheme, lambda(url): raise_(WrongStateSchemeException("Unknown state scheme!"))) diff --git a/nixops/state/json_file.py b/nixops/state/json_file.py new file mode 100644 index 000000000..19147ad61 --- /dev/null +++ b/nixops/state/json_file.py @@ -0,0 +1,365 @@ +# -*- coding: utf-8 -*- + +import nixops.deployment +import os +import os.path +import sys +import threading +import fcntl +import re +import json +import copy + +from uuid import uuid1 as gen_uuid + +def _subclasses(cls): + sub = cls.__subclasses__() + return [cls] if not sub else [g for s in sub for g in _subclasses(s)] + +class TransactionalJsonFile: + """ + Transactional access to a JSON file, with support + of nested transactions. + + This is made possible by keeping track of the transaction nest level. + If a transaction is started, the current JSON file is flocked() and read into memory. + All modifications to the document are kept in memory, until the last nested context is + exited again. + + Then, the in memory dict written to a temporary file, which is moved in place of + the original file to prevent partial writes. + """ + + # Implementation notes: + # if self.nesting > 0, then no write will propagate. + def __init__(self, db_file): + + lock_file_path = re.sub("\.json$", ".lock", db_file) + self._lock_file = open(lock_file_path, "w") + fcntl.fcntl(self._lock_file, fcntl.F_SETFD, fcntl.FD_CLOEXEC) # to not keep the lock in child processes + + self._db_file = db_file + self.nesting = 0 + self.lock = threading.RLock() + + ## Make sure that a JSON database file is in place. + with self: + pass + + def read(self): + if self.nesting == 0: + with open(self._db_file,"r") as f: + return json.load(f) + else: + assert self.nesting > 0 + return self._current_state + + # Implement Python's context management protocol so that "with db" + # automatically commits or rolls back. + def __enter__(self): + self.lock.acquire() + if self.nesting == 0: + fcntl.flock(self._lock_file, fcntl.LOCK_EX) + self._ensure_db_exists() + self.must_rollback = False + json = self.read() + self._backup_state = copy.deepcopy(json) + self._current_state = copy.deepcopy(json) + self.nesting = self.nesting + 1 + + def __exit__(self, exception_type, exception_value, exception_traceback): + if exception_type != None: self.must_rollback = True + self.nesting = self.nesting - 1 + assert self.nesting >= 0 + if self.nesting == 0: + if self.must_rollback: + self._rollback() + else: + self._commit() + fcntl.flock(self._lock_file, fcntl.LOCK_UN) + self.lock.release() + + def _rollback(self): + self._backup_state = None + self._current_state = None + pass + + def set(self, state): + self._current_state = state + + def _commit(self): + assert self.nesting == 0 + + # TODO: write to temp file, then mv + with open(self._db_file, "w") as f: + json.dump(self._current_state, f,indent=2) + + self._backup_state = None + self._current_state = None + + def _ensure_db_exists(self): + db_exists = os.path.exists(self._db_file) + if not db_exists: + initial_db = { + "schemaVersion": 0, + "deployments": {} + } + + with open(self._db_file, "w", 0o600) as f: + json.dump(initial_db, f) + f.close() + + def schema_version(self): + version = self.read()["schemaVersion"] + if version is None: + raise "illegal datafile" #TODO: proper exception + else: + return version + +class JsonFile(object): + """NixOps state file.""" + + def __init__(self, json_file): + self.file_path = json_file + + if os.path.splitext(json_file)[1] not in ['.json']: + raise Exception("state file ‘{0}’ should have extension ‘.json’".format(json_file)) + + self.db = TransactionalJsonFile(json_file) + + # Check that we're not using a to new DB schema version. + with self.db: + version = self.db.schema_version() + if version > 0: + raise Exception("this NixOps version is too old to deal with JSON schema version {0}".format(version)) + + # The "db" will provide atomicity as well. + self.atomic = self.db + + ############################################################################################### + ## Deployment + + def query_deployments(self): + """Return the UUIDs of all deployments in the database.""" + + return self.db.read()["deployments"].keys() + + def get_all_deployments(self): + """Return Deployment objects for every deployment in the database.""" + uuids = self.query_deployments() + res = [] + for uuid in uuids: + try: + res.append(self.open_deployment(uuid=uuid)) + except nixops.deployment.UnknownBackend as e: + sys.stderr.write("skipping deployment ‘{0}’: {1}\n".format(uuid, str(e))) + return res + + def _find_deployment(self, uuid=None): + all_deployments = self.db.read()["deployments"] + found = [] + if not uuid: + found = all_deployments + if not found: + found = filter(lambda(id): id == uuid, all_deployments) + if not found: + found = filter(lambda(id): all_deployments[id]["attributes"].get("name") == uuid, all_deployments) + + if not found: + found = filter(lambda(id): id.startswith(uuid), all_deployments) + + if not found: + return None + + if len(found) > 1: + if uuid: + raise Exception("state file contains multiple deployments with the same name, so you should specify one using its UUID") + else: + raise Exception("state file contains multiple deployments, so you should specify which one to use using ‘-d’, or set the environment variable NIXOPS_DEPLOYMENT") + return nixops.deployment.Deployment(self, found[0], sys.stderr) + + def open_deployment(self, uuid=None): + """Open an existing deployment.""" + deployment = self._find_deployment(uuid=uuid) + if deployment: return deployment + raise Exception("could not find specified deployment in state file ‘{0}’".format(self.db_file)) + + def create_deployment(self, uuid=None): + """Create a new deployment.""" + if not uuid: + import uuid + uuid = str(uuid.uuid1()) + with self.db: + state = self.db.read() + state["deployments"][uuid] = { "attributes": {}, "resources": {} } + self.db.set(state) + return nixops.deployment.Deployment(self, uuid, sys.stderr) + + def _delete_deployment(self, deployment_uuid): + """NOTE: This is UNSAFE, it's guarded in nixops/deployment.py. Do not call this function except from there!""" + self.__db.execute("delete from Deployments where uuid = ?", (deployment_uuid,)) + with self.db: + state = self.db.read() + state["deployments"].pop(deployment_uuid, None) + self.db.set(state) + + def clone_deployment(self, deployment_uuid): + with self.db: + if not uuid: + import uuid + new_uuid = str(uuid.uuid1()) + state = self.db.read() + + cloned_attributes = copy.deepcopy(state["deployments"][deployment_uuid]["attributes"]) + state["deployments"][new_uuid] = { + "attributes": cloned_attributes, + "resources": {} + } + + self.db.set(state) + + return self._find_deployment(new_uuid) + + def get_resources_for(self, deployment): + """Get all the resources for a certain deployment""" + resources = {} + with self.db: + state = self.db.read() + state_resources = state["deployments"][deployment.uuid]["resources"] + for res_id, res in state_resources.items(): + r = self._create_state(deployment, res["type"], res["name"], res_id) + resources[name] = r + self.db.set(state) + return resources + + def set_deployment_attrs(self, deployment_uuid, attrs): + """Update deployment attributes in the state.""" + with self.db: + state = self.db.read() + for n, v in attrs.iteritems(): + if v == None: + state["deployments"][deployment_uuid]["attributes"].pop(n,None) + else: + state["deployments"][deployment_uuid]["attributes"][n] = v + self.db.set(state) + + def del_deployment_attr(self, deployment_uuid, attr_name): + with self.db: + state = self.db.read() + state["deployments"][deployment_uuid]["attributes"].pop(attr_name,None) + self.db.set(state) + + def get_deployment_attr(self, deployment_uuid, name): + """Get a deployment attribute from the state.""" + with self.db: + state = self.db.read() + result = state["deployments"][deployment_uuid]["attributes"].get(name) + if result: + return result + else: + return nixops.util.undefined + + def get_all_deployment_attrs(self, deployment_uuid): + with self.db: + state = self.db.read() + return copy.deepcopy(state["deployments"][deployment]["attributes"]) + + def get_deployment_lock(self, deployment): + lock_dir = os.environ.get("HOME", "") + "/.nixops/locks" + if not os.path.exists(lock_dir): os.makedirs(lock_dir, 0700) + lock_file_path = lock_dir + "/" + deployment.uuid + class DeploymentLock(object): + def __init__(self, logger, path): + self._lock_file_path = path + self._logger = logger + self._lock_file = None + def __enter__(self): + self._lock_file = open(self._lock_file_path, "w") + fcntl.fcntl(self._lock_file, fcntl.F_SETFD, fcntl.FD_CLOEXEC) + try: + fcntl.flock(self._lock_file, fcntl.LOCK_EX | fcntl.LOCK_NB) + except IOError: + self._logger.log( + "waiting for exclusive deployment lock..." + ) + fcntl.flock(self._lock_file, fcntl.LOCK_EX) + def __exit__(self, exception_type, exception_value, exception_traceback): + if self._lock_file: + self._lock_file.close() + return DeploymentLock(deployment.logger, lock_file_path) + + ############################################################################################### + ## Resources + + def create_resource(self, deployment, name, type): + with self.db: + state = self.db.read() + if name in state["deployments"][deployment.uuid]["resources"]: + raise Exception("resource already exists in database!") + id = gen_uuid() + state["deployments"][deployment.uuid]["resources"][id] = { + "name": name, + "type" : type, + "attributes" : {} + } + self.db.set(state) + r = self._create_state(deployment, type, name, id) + return r + + def delete_resource(self, deployment_uuid, res_id): + with self.db: + state = self.db.read() + state["deployments"][deployment.uuid]["resources"].pop(res_id) + self.db.set(state) + + def _rename_resource(self, deployment_uuid, resource_id, new_name): + """NOTE: Invariants are checked in nixops/deployment.py#rename""" + with self.db: + state = self.db.read() + state["deployments"][deployment.uuid]["resources"][resource_id]["name"] = new_name + self.db.set(state) + + def set_resource_attrs(self, resource_id, attrs): + with self.db: + state = self.db.read() + resource_attrs = state["deployments"][deployment.uuid]["resources"][resource_id]["attributes"] + for n, v in attrs.iteritems(): + if v == None: + resource_attrs.pop(n, None) + else: + resource_attrs[n] = v + state["deployments"][deployment.uuid]["resources"][resource_id]["attributes"] = resource_attrs + self.db.set(state) + + def del_resource_attr(self, resource_id, name): + with self.db: + state = self.db.read() + resource_attrs = state["deployments"][deployment.uuid]["resources"][resource_id]["attributes"] + resource_attrs.pop(name, None) + state["deployments"][deployment.uuid]["resources"][resource_id]["attributes"] = resource_attrs + self.db.set(state) + + def get_resource_attr(self, resource_id, name): + """Get a machine attribute from the state file.""" + with self.db: + state = self.db.read() + resource_attrs = state["deployments"][deployment.uuid]["resources"][resource_id]["attributes"] + res = resource_attrs.get(name) + if res != None: return res + return nixops.util.undefined + + def get_all_resource_attrs(self, resource_id): + with self.db: + state = self.db.read() + resource_attrs = state["deployments"][deployment.uuid]["resources"][resource_id]["attributes"] + return copy.deepcopy(resource_attrs) + + ### STATE + def _create_state(self, depl, type, name, id): + """Create a resource state object of the desired type.""" + + for cls in _subclasses(nixops.resources.ResourceState): + if type == cls.get_type(): + return cls(depl, name, id) + + raise nixops.deployment.UnknownBackend("unknown resource type ‘{0}’".format(type)) From 6baf57f1c043bad489e5c0d8eed2504af2d679ab Mon Sep 17 00:00:00 2001 From: Maarten Hoogendoorn Date: Tue, 14 Mar 2017 16:25:56 +0100 Subject: [PATCH 041/123] Include the deployment_uuid in the API for modifying resource attributes --- nixops/resources/__init__.py | 8 ++++---- nixops/state/sqlite3_file.py | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/nixops/resources/__init__.py b/nixops/resources/__init__.py index a1c3b9928..3e6c7b5ba 100644 --- a/nixops/resources/__init__.py +++ b/nixops/resources/__init__.py @@ -64,7 +64,7 @@ def __init__(self, depl, name, id): def _set_attrs(self, attrs): """Update machine attributes in the state file.""" - self.depl._state.set_resource_attrs(self.id, attrs) + self.depl._state.set_resource_attrs(self.depl.uuid, self.id, attrs) def _set_attr(self, name, value): @@ -73,17 +73,17 @@ def _set_attr(self, name, value): def _del_attr(self, name): """Delete a machine attribute from the state file.""" - self.depl._state.del_resource_attr(self.id, name) + self.depl._state.del_resource_attr(self.depl.uuid, self.id, name) #TODO(moretea): again, the default option appears to be defunct. # Have removed it in state/file.py. def _get_attr(self, name, default=nixops.util.undefined): """Get a machine attribute from the state file.""" - return self.depl._state.get_resource_attr(self.id, name) + return self.depl._state.get_resource_attr(self.depl.uuid, self.id, name) def export(self): """Export the resource to move between databases""" - res = self.depl._state.get_all_resource_attrs(self.id) + res = self.depl._state.get_all_resource_attrs(self.depl.uuid, self.id) res['type'] = self.get_type() return res diff --git a/nixops/state/sqlite3_file.py b/nixops/state/sqlite3_file.py index b053b5ff7..167e27d82 100644 --- a/nixops/state/sqlite3_file.py +++ b/nixops/state/sqlite3_file.py @@ -278,7 +278,7 @@ def _rename_resource(self, deployment_uuid, resource_id, new_name): self.__db.execute("update Resources set name = ? where deployment = ? and id = ?", (new_name, deployment_uuid, resource_id)) - def set_resource_attrs(self, resource_id, attrs): + def set_resource_attrs(self, _deployment_uuid, resource_id, attrs): with self.__db: c = self.__db.cursor() for n, v in attrs.iteritems(): @@ -288,11 +288,11 @@ def set_resource_attrs(self, resource_id, attrs): c.execute("insert or replace into ResourceAttrs(machine, name, value) values (?, ?, ?)", (resource_id, n, v)) - def del_resource_attr(self, resource_id, name): + def del_resource_attr(self, _deployment_uuid, resource_id, name): with self.__db: self.__db.execute("delete from ResourceAttrs where machine = ? and name = ?", (resource_id, name)) - def get_resource_attr(self, resource_id, name): + def get_resource_attr(self, _deployment_uuid, resource_id, name): """Get a machine attribute from the state file.""" with self.__db: c = self.__db.cursor() @@ -301,7 +301,7 @@ def get_resource_attr(self, resource_id, name): if row != None: return row[0] return nixops.util.undefined - def get_all_resource_attrs(self, resource_id): + def get_all_resource_attrs(self, deployment_uuid, resource_id): with self.__db: c = self.__db.cursor() c.execute("select name, value from ResourceAttrs where machine = ?", (resource_id,)) From 2dfa148474160d6d3ddea3af410de24105c494bc Mon Sep 17 00:00:00 2001 From: Maarten Hoogendoorn Date: Tue, 14 Mar 2017 16:26:21 +0100 Subject: [PATCH 042/123] Small bug fixes --- nixops/state/json_file.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nixops/state/json_file.py b/nixops/state/json_file.py index 19147ad61..72cc51a12 100644 --- a/nixops/state/json_file.py +++ b/nixops/state/json_file.py @@ -228,7 +228,7 @@ def get_resources_for(self, deployment): state_resources = state["deployments"][deployment.uuid]["resources"] for res_id, res in state_resources.items(): r = self._create_state(deployment, res["type"], res["name"], res_id) - resources[name] = r + resources[res["name"]] = r self.db.set(state) return resources @@ -296,7 +296,7 @@ def create_resource(self, deployment, name, type): state = self.db.read() if name in state["deployments"][deployment.uuid]["resources"]: raise Exception("resource already exists in database!") - id = gen_uuid() + id = str(gen_uuid()) state["deployments"][deployment.uuid]["resources"][id] = { "name": name, "type" : type, From 6733fa7c45a81ed7d9aa30122b6e9ff97932168c Mon Sep 17 00:00:00 2001 From: Maarten Hoogendoorn Date: Tue, 14 Mar 2017 16:26:38 +0100 Subject: [PATCH 043/123] Utilize deployment_uid --- nixops/state/json_file.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/nixops/state/json_file.py b/nixops/state/json_file.py index 72cc51a12..eb9254b15 100644 --- a/nixops/state/json_file.py +++ b/nixops/state/json_file.py @@ -309,49 +309,49 @@ def create_resource(self, deployment, name, type): def delete_resource(self, deployment_uuid, res_id): with self.db: state = self.db.read() - state["deployments"][deployment.uuid]["resources"].pop(res_id) + state["deployments"][deployment_uuid]["resources"].pop(res_id) self.db.set(state) def _rename_resource(self, deployment_uuid, resource_id, new_name): """NOTE: Invariants are checked in nixops/deployment.py#rename""" with self.db: state = self.db.read() - state["deployments"][deployment.uuid]["resources"][resource_id]["name"] = new_name + state["deployments"][deployment_uuid]["resources"][resource_id]["name"] = new_name self.db.set(state) - def set_resource_attrs(self, resource_id, attrs): + def set_resource_attrs(self, deployment_uuid, resource_id, attrs): with self.db: state = self.db.read() - resource_attrs = state["deployments"][deployment.uuid]["resources"][resource_id]["attributes"] + resource_attrs = state["deployments"][deployment_uuid]["resources"][resource_id]["attributes"] for n, v in attrs.iteritems(): if v == None: resource_attrs.pop(n, None) else: resource_attrs[n] = v - state["deployments"][deployment.uuid]["resources"][resource_id]["attributes"] = resource_attrs + state["deployments"][deployment_uuid]["resources"][resource_id]["attributes"] = resource_attrs self.db.set(state) - def del_resource_attr(self, resource_id, name): + def del_resource_attr(self, deployment_uuid, resource_id, name): with self.db: state = self.db.read() - resource_attrs = state["deployments"][deployment.uuid]["resources"][resource_id]["attributes"] + resource_attrs = state["deployments"][deployment_uuid]["resources"][resource_id]["attributes"] resource_attrs.pop(name, None) - state["deployments"][deployment.uuid]["resources"][resource_id]["attributes"] = resource_attrs + state["deployments"][deployment_uuid]["resources"][resource_id]["attributes"] = resource_attrs self.db.set(state) - def get_resource_attr(self, resource_id, name): + def get_resource_attr(self, deployment_uuid, resource_id, name): """Get a machine attribute from the state file.""" with self.db: state = self.db.read() - resource_attrs = state["deployments"][deployment.uuid]["resources"][resource_id]["attributes"] + resource_attrs = state["deployments"][deployment_uuid]["resources"][resource_id]["attributes"] res = resource_attrs.get(name) if res != None: return res return nixops.util.undefined - def get_all_resource_attrs(self, resource_id): + def get_all_resource_attrs(self, deployment_uuid, resource_id): with self.db: state = self.db.read() - resource_attrs = state["deployments"][deployment.uuid]["resources"][resource_id]["attributes"] + resource_attrs = state["deployments"][deployment_uuid]["resources"][resource_id]["attributes"] return copy.deepcopy(resource_attrs) ### STATE From 3827e69193f9c2fcfa0d2122899d1a1c7c78d6ff Mon Sep 17 00:00:00 2001 From: Maarten Hoogendoorn Date: Tue, 14 Mar 2017 16:45:08 +0100 Subject: [PATCH 044/123] Also recognize json's true as a valid bool value --- nixops/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nixops/util.py b/nixops/util.py index d017b5323..493a87108 100644 --- a/nixops/util.py +++ b/nixops/util.py @@ -218,7 +218,7 @@ def get(self): if s == None: return None elif type is str: return s elif type is int: return int(s) - elif type is bool: return True if s == "1" else False + elif type is bool: return True if s == True or s == "1" else False elif type is 'json': return json.loads(s) else: assert False def set(self, x): From 056ab60d1f8de4e50090748a5985a1542db1f5ad Mon Sep 17 00:00:00 2001 From: Maarten Hoogendoorn Date: Wed, 29 Mar 2017 13:10:39 +0200 Subject: [PATCH 045/123] Undo split between state.atomic and state.db --- nixops/backends/ec2.py | 18 ++--- nixops/backends/hetzner.py | 2 +- nixops/backends/virtualbox.py | 4 +- nixops/deployment.py | 6 +- nixops/resources/__init__.py | 2 +- nixops/resources/cloudwatch_log_group.py | 4 +- nixops/resources/cloudwatch_log_stream.py | 4 +- nixops/resources/datadog-monitor.py | 2 +- nixops/resources/datadog-screenboard.py | 2 +- nixops/resources/datadog-timeboard.py | 2 +- nixops/resources/ebs_volume.py | 2 +- nixops/resources/ec2_keypair.py | 4 +- nixops/resources/ec2_placement_group.py | 6 +- nixops/resources/ec2_rds_dbinstance.py | 8 +-- nixops/resources/ec2_security_group.py | 6 +- nixops/resources/elastic_file_system.py | 4 +- .../elastic_file_system_mount_target.py | 4 +- nixops/resources/elastic_ip.py | 2 +- nixops/resources/iam_role.py | 4 +- nixops/resources/s3_bucket.py | 2 +- nixops/resources/sns_topic.py | 4 +- nixops/resources/sqs_queue.py | 4 +- nixops/resources/ssh_keypair.py | 2 +- nixops/state/json_file.py | 3 - nixops/state/sqlite3_file.py | 66 +++++++++---------- scripts/nixops | 2 +- 26 files changed, 82 insertions(+), 87 deletions(-) diff --git a/nixops/backends/ec2.py b/nixops/backends/ec2.py index 5919be5b5..de0e43753 100644 --- a/nixops/backends/ec2.py +++ b/nixops/backends/ec2.py @@ -124,7 +124,7 @@ def __init__(self, depl, name, id): def _reset_state(self): """Discard all state pertaining to an instance.""" - with self.depl._state.atomic: + with self.depl._state.db: self.state = MachineState.MISSING self.associate_public_ip_address = None self.use_private_ip_address = None @@ -343,7 +343,7 @@ def _instance_ip_ready(ins): self.log_end("{0} / {1}".format(instance.ip_address, instance.private_ip_address)) - with self.depl._state.atomic: + with self.depl._state.db: self.private_ipv4 = instance.private_ip_address self.public_ipv4 = instance.ip_address self.public_dns_name = instance.public_dns_name @@ -585,7 +585,7 @@ def _assign_elastic_ip(self, elastic_ipv4, check): nixops.known_hosts.update(self.public_ipv4, elastic_ipv4, self.public_host_key) - with self.depl._state.atomic: + with self.depl._state.db: self.elastic_ipv4 = elastic_ipv4 self.public_ipv4 = elastic_ipv4 self.ssh_pinged = False @@ -598,7 +598,7 @@ def _assign_elastic_ip(self, elastic_ipv4, check): else: self.log("address ‘{0}’ was not associated with instance ‘{1}’".format(self.elastic_ipv4, self.vm_id)) - with self.depl._state.atomic: + with self.depl._state.db: self.elastic_ipv4 = None self.public_ipv4 = None self.ssh_pinged = False @@ -662,7 +662,7 @@ def create_instance(self, defn, zone, devmap, user_data, ebs_optimized): lambda: self._conn.request_spot_instances(price=defn.spot_instance_price/100.0, **common_args) )[0] - with self.depl._state.atomic: + with self.depl._state.db: self.spot_instance_price = defn.spot_instance_price self.spot_instance_request_id = request.id @@ -693,7 +693,7 @@ def create_instance(self, defn, zone, devmap, user_data, ebs_optimized): # the instance ID, we'll get the same instance ID on the # next run. if not self.client_token: - with self.depl._state.atomic: + with self.depl._state.db: self.client_token = nixops.util.generate_random_string(length=48) # = 64 ASCII chars self.state = self.STARTING @@ -892,7 +892,7 @@ def create(self, defn, check, allow_reboot, allow_recreate): # Generate a public/private host key. if not self.public_host_key: (private, public) = nixops.util.create_key_pair(type=defn.host_key_type()) - with self.depl._state.atomic: + with self.depl._state.db: self.public_host_key = public self.private_host_key = private @@ -902,7 +902,7 @@ def create(self, defn, check, allow_reboot, allow_recreate): instance = self.create_instance(defn, zone, devmap, user_data, ebs_optimized) - with self.depl._state.atomic: + with self.depl._state.db: self.vm_id = instance.id self.ami = defn.ami self.instance_type = defn.instance_type @@ -970,7 +970,7 @@ def create(self, defn, check, allow_reboot, allow_recreate): elastic_ipv4 = res.public_ipv4 self._assign_elastic_ip(elastic_ipv4, check) - with self.depl._state.atomic: + with self.depl._state.db: self.use_private_ip_address = defn.use_private_ip_address self.associate_public_ip_address = defn.associate_public_ip_address diff --git a/nixops/backends/hetzner.py b/nixops/backends/hetzner.py index 69343d09e..f96cc6c19 100644 --- a/nixops/backends/hetzner.py +++ b/nixops/backends/hetzner.py @@ -584,7 +584,7 @@ def create(self, defn, check, allow_reboot, allow_recreate): "‘{0}’... ".format(self.name)) # Create a new Admin account exclusively for this machine. server = self._get_server_from_main_robot(self.main_ipv4, defn) - with self.depl._state.atomic: + with self.depl._state.db: (self.robot_admin_user, self.robot_admin_pass) = server.admin.create() self.log_end("done. ({0})".format(self.robot_admin_user)) diff --git a/nixops/backends/virtualbox.py b/nixops/backends/virtualbox.py index 8a1912109..afc3aa9c9 100644 --- a/nixops/backends/virtualbox.py +++ b/nixops/backends/virtualbox.py @@ -195,7 +195,7 @@ def create(self, defn, check, allow_reboot, allow_recreate): # Generate a public/private host key. if not self.public_host_key: (private, public) = nixops.util.create_key_pair() - with self.depl._state.atomic: + with self.depl._state.db: self.public_host_key = public self.private_host_key = private @@ -204,7 +204,7 @@ def create(self, defn, check, allow_reboot, allow_recreate): # Backwards compatibility. if self.disk: - with self.depl.atomic: + with self.depl._state.db: self._update_disk("disk1", {"created": True, "path": self.disk, "attached": self.disk_attached, "port": 0}) diff --git a/nixops/deployment.py b/nixops/deployment.py index bf3f3bf87..0b043aedd 100644 --- a/nixops/deployment.py +++ b/nixops/deployment.py @@ -144,7 +144,7 @@ def export(self): def import_(self, attrs): - with self._state.atomic: + with self._state.db: for k, v in attrs.iteritems(): if k == 'resources': continue self._set_attr(k, v) @@ -169,7 +169,7 @@ def delete_resource(self, m): def delete(self, force=False): """Delete this deployment from the state file.""" - with self._state.atomic: + with self._state.db: if not force and len(self.resources) > 0: raise Exception("cannot delete this deployment because it still has resources") @@ -774,7 +774,7 @@ def evaluate_active(self, include=[], exclude=[], kill_obsolete=False): self.evaluate() # Create state objects for all defined resources. - with self._state.atomic: + with self._state.db: for m in self.definitions.itervalues(): if m.name not in self.resources: self._create_resource(m.name, m.get_type()) diff --git a/nixops/resources/__init__.py b/nixops/resources/__init__.py index 3e6c7b5ba..cc27a62e0 100644 --- a/nixops/resources/__init__.py +++ b/nixops/resources/__init__.py @@ -89,7 +89,7 @@ def export(self): def import_(self, attrs): """Import the resource from another database""" - with self.depl.atomic: + with self.depl._state.db: for k, v in attrs.iteritems(): if k == 'type': continue self._set_attr(k, v) diff --git a/nixops/resources/cloudwatch_log_group.py b/nixops/resources/cloudwatch_log_group.py index 353a9716b..943739e74 100644 --- a/nixops/resources/cloudwatch_log_group.py +++ b/nixops/resources/cloudwatch_log_group.py @@ -72,7 +72,7 @@ def _destroy(self): self._conn.delete_log_group(self.log_group_name) except boto.logs.exceptions.ResourceNotFoundException, e: self.log("the log group ‘{0}’ was already deleted".format(self.log_group_name)) - with self.depl._state.atomic: + with self.depl._state.db: self.state = self.MISSING self.log_group_name = None self.region = None @@ -114,7 +114,7 @@ def create(self, defn, check, allow_reboot, allow_recreate): self.log("setting the retention in days of '{0}' to '{1}'".format(defn.config['name'], defn.config['retentionInDays'])) self._conn.set_retention(log_group_name=defn.config['name'],retention_in_days=defn.config['retentionInDays']) - with self.depl._state.atomic: + with self.depl._state.db: self.state = self.UP self.log_group_name = defn.config['name'] self.region = defn.config['region'] diff --git a/nixops/resources/cloudwatch_log_stream.py b/nixops/resources/cloudwatch_log_stream.py index 5e0934411..4d6e770b6 100644 --- a/nixops/resources/cloudwatch_log_stream.py +++ b/nixops/resources/cloudwatch_log_stream.py @@ -72,7 +72,7 @@ def _destroy(self): self._conn.delete_log_stream(log_group_name=self.log_group_name,log_stream_name=self.log_stream_name) except boto.logs.exceptions.ResourceNotFoundException, e: self.log("the log group ‘{0}’ or log stream ‘{1}’ was already deleted".format(self.log_group_name,self.log_stream_name)) - with self.depl._state.atomic: + with self.depl._state.db: self.state = self.MISSING self.log_group_name = None self.log_stream_name = None @@ -121,7 +121,7 @@ def create(self, defn, check, allow_reboot, allow_recreate): exist, arn = self.lookup_cloudwatch_log_stream(log_group_name=defn.config['logGroupName'], log_stream_name=defn.config['name']) - with self.depl._state.atomic: + with self.depl._state.db: self.state = self.UP self.log_stream_name = defn.config['name'] self.log_group_name = defn.config['logGroupName'] diff --git a/nixops/resources/datadog-monitor.py b/nixops/resources/datadog-monitor.py index 64c5c39ff..77f009010 100644 --- a/nixops/resources/datadog-monitor.py +++ b/nixops/resources/datadog-monitor.py @@ -106,7 +106,7 @@ def create(self, defn, check, allow_reboot, allow_recreate): if 'errors' in response: raise Exception(str(response['errors'])) - with self.depl._state.atomic: + with self.depl._state.db: self.state = self.UP self.api_key = defn.config['apiKey'] self.app_key = defn.config['appKey'] diff --git a/nixops/resources/datadog-screenboard.py b/nixops/resources/datadog-screenboard.py index 102d2bd03..7c7bf0891 100644 --- a/nixops/resources/datadog-screenboard.py +++ b/nixops/resources/datadog-screenboard.py @@ -104,7 +104,7 @@ def create(self, defn, check, allow_reboot, allow_recreate): if 'errors' in response: raise Exception(str(response['errors'])) - with self.depl._state.atomic: + with self.depl._state.db: self.state = self.UP self.api_key = defn.config['apiKey'] self.app_key = defn.config['appKey'] diff --git a/nixops/resources/datadog-timeboard.py b/nixops/resources/datadog-timeboard.py index 6da4e5240..2a893ae02 100644 --- a/nixops/resources/datadog-timeboard.py +++ b/nixops/resources/datadog-timeboard.py @@ -111,7 +111,7 @@ def create(self, defn, check, allow_reboot, allow_recreate): else: url = response['url'] - with self.depl._state.atomic: + with self.depl._state.db: self.state = self.UP self.api_key = defn.config['apiKey'] self.app_key = defn.config['appKey'] diff --git a/nixops/resources/ebs_volume.py b/nixops/resources/ebs_volume.py index a23c4b63d..0485057ec 100644 --- a/nixops/resources/ebs_volume.py +++ b/nixops/resources/ebs_volume.py @@ -122,7 +122,7 @@ def create(self, defn, check, allow_reboot, allow_recreate): # volume we just created. Doesn't seem to be anything we # can do about this. - with self.depl._state.atomic: + with self.depl._state.db: self.state = self.STARTING self.region = defn.region self.zone = defn.zone diff --git a/nixops/resources/ec2_keypair.py b/nixops/resources/ec2_keypair.py index e7dc5fc7b..4e82864e4 100644 --- a/nixops/resources/ec2_keypair.py +++ b/nixops/resources/ec2_keypair.py @@ -78,7 +78,7 @@ def create(self, defn, check, allow_reboot, allow_recreate): # Generate the key pair locally. if not self.public_key: (private, public) = nixops.util.create_key_pair(type="rsa") # EC2 only supports RSA keys. - with self.depl._state.atomic: + with self.depl._state.db: self.public_key = public self.private_key = private @@ -102,7 +102,7 @@ def create(self, defn, check, allow_reboot, allow_recreate): self.log("uploading EC2 key pair ‘{0}’...".format(defn.keypair_name)) self._conn.import_key_pair(defn.keypair_name, self.public_key) - with self.depl._state.atomic: + with self.depl._state.db: self.state = self.UP self.keypair_name = defn.keypair_name diff --git a/nixops/resources/ec2_placement_group.py b/nixops/resources/ec2_placement_group.py index 5bc3dbd3a..871f2cea6 100644 --- a/nixops/resources/ec2_placement_group.py +++ b/nixops/resources/ec2_placement_group.py @@ -67,11 +67,11 @@ def _connect(self): def create(self, defn, check, allow_reboot, allow_recreate): # Name or region change means a completely new security group if self.placement_group_name and (defn.placement_group_name != self.placement_group_name or defn.region != self.region): - with self.depl._state.atomic: + with self.depl._state.db: self.state = self.UNKNOWN self.old_placement_groups = self.old_placement_groups + [{'name': self.placement_group_name, 'region': self.region}] - with self.depl._state.atomic: + with self.depl._state.db: self.region = defn.region self.access_key_id = defn.access_key_id or nixops.ec2_utils.get_access_key_id() self.placement_group_name = defn.placement_group_name @@ -79,7 +79,7 @@ def create(self, defn, check, allow_reboot, allow_recreate): grp = None if check: - with self.depl._state.atomic: + with self.depl._state.db: self._connect() try: diff --git a/nixops/resources/ec2_rds_dbinstance.py b/nixops/resources/ec2_rds_dbinstance.py index 0c814d338..9921ab2db 100644 --- a/nixops/resources/ec2_rds_dbinstance.py +++ b/nixops/resources/ec2_rds_dbinstance.py @@ -148,7 +148,7 @@ def _wait_for_dbinstance(self, dbinstance, state='available'): time.sleep(6) def _copy_dbinstance_attrs(self, dbinstance): - with self.depl._state.atomic: + with self.depl._state.db: self.rds_dbinstance_id = dbinstance.id self.rds_dbinstance_allocated_storage = int(dbinstance.allocated_storage) self.rds_dbinstance_instance_class = dbinstance.instance_class @@ -170,7 +170,7 @@ def _compare_instance_id(self, instance_id): return unicode(self.rds_dbinstance_id).lower() == unicode(instance_id).lower() def create(self, defn, check, allow_reboot, allow_recreate): - with self.depl._state.atomic: + with self.depl._state.db: self.access_key_id = defn.access_key_id or nixops.ec2_utils.get_access_key_id() if not self.access_key_id: raise Exception("please set ‘accessKeyId’, $EC2_ACCESS_KEY or $AWS_ACCESS_KEY_ID") @@ -192,7 +192,7 @@ def create(self, defn, check, allow_reboot, allow_recreate): dbinstance = self._try_fetch_dbinstance(self.rds_dbinstance_id) - with self.depl._state.atomic: + with self.depl._state.db: if check or self.state == self.MISSING or self.state == self.UNKNOWN: if dbinstance and (self.state == self.MISSING or self.state == self.UNKNOWN): if dbinstance.status == 'deleting': @@ -233,7 +233,7 @@ def create(self, defn, check, allow_reboot, allow_recreate): self._copy_dbinstance_attrs(dbinstance) self.state = self.UP - with self.depl._state.atomic: + with self.depl._state.db: if self.state == self.UP and self._diff_defn(defn): if dbinstance is None: raise Exception("state is UP but database instance does not exist. re-run with --check option to synchronize states") diff --git a/nixops/resources/ec2_security_group.py b/nixops/resources/ec2_security_group.py index d8b2ec94c..4e37a8a2b 100644 --- a/nixops/resources/ec2_security_group.py +++ b/nixops/resources/ec2_security_group.py @@ -96,11 +96,11 @@ def _connect(self): def create(self, defn, check, allow_reboot, allow_recreate): # Name or region change means a completely new security group if self.security_group_name and (defn.security_group_name != self.security_group_name or defn.region != self.region): - with self.depl._state.atomic: + with self.depl._state.db: self.state = self.UNKNOWN self.old_security_groups = self.old_security_groups + [{'name': self.security_group_name, 'region': self.region}] - with self.depl._state.atomic: + with self.depl._state.db: self.region = defn.region self.access_key_id = defn.access_key_id or nixops.ec2_utils.get_access_key_id() self.security_group_name = defn.security_group_name @@ -109,7 +109,7 @@ def create(self, defn, check, allow_reboot, allow_recreate): grp = None if check: - with self.depl._state.atomic: + with self.depl._state.db: self._connect() try: diff --git a/nixops/resources/elastic_file_system.py b/nixops/resources/elastic_file_system.py index ee80f227a..64560a376 100644 --- a/nixops/resources/elastic_file_system.py +++ b/nixops/resources/elastic_file_system.py @@ -85,7 +85,7 @@ def create(self, defn, check, allow_reboot, allow_recreate): if len(fss) == 1: fs = fss[0] if fs["LifeCycleState"] == "available": - with self.depl._state.atomic: + with self.depl._state.db: self.state = self.UP self.fs_id = fs["FileSystemId"] self.region = defn.config["region"] @@ -146,7 +146,7 @@ def destroy(self, wipe=False): break time.sleep(1) - with self.depl._state.atomic: + with self.depl._state.db: self.state = self.MISSING self.fs_id = None self.region = None diff --git a/nixops/resources/elastic_file_system_mount_target.py b/nixops/resources/elastic_file_system_mount_target.py index 81d87a5f1..c6b318268 100644 --- a/nixops/resources/elastic_file_system_mount_target.py +++ b/nixops/resources/elastic_file_system_mount_target.py @@ -41,7 +41,7 @@ def get_type(cls): return "elastic-file-system-mount-target" def _reset_state(self): - with self.depl._state.atomic: + with self.depl._state.db: self.state = self.MISSING self.access_key_id = None self.region = None @@ -94,7 +94,7 @@ def create(self, defn, check, allow_reboot, allow_recreate): args["IpAddress"] = defn.config["ipAddress"] res = client.create_mount_target(FileSystemId=fs_id, SubnetId=defn.config["subnet"], **args) - with self.depl._state.atomic: + with self.depl._state.db: self.state = self.STARTING self.fsmt_id = res["MountTargetId"] self.fs_id = fs_id diff --git a/nixops/resources/elastic_ip.py b/nixops/resources/elastic_ip.py index f907ca299..8a2ee2723 100644 --- a/nixops/resources/elastic_ip.py +++ b/nixops/resources/elastic_ip.py @@ -84,7 +84,7 @@ def create(self, defn, check, allow_reboot, allow_recreate): # address we just created. Doesn't seem to be anything we # can do about this. - with self.depl._state.atomic: + with self.depl._state.db: self.state = self.UP self.region = defn.region self.public_ipv4 = address.public_ip diff --git a/nixops/resources/iam_role.py b/nixops/resources/iam_role.py index b0ae6a5f1..c8c6f9dac 100644 --- a/nixops/resources/iam_role.py +++ b/nixops/resources/iam_role.py @@ -105,7 +105,7 @@ def _destroy(self): self.log("could not find instance profile") - with self.depl._state.atomic: + with self.depl._state.db: self.state = self.MISSING self.role_name = None self.access_key_id = None @@ -167,7 +167,7 @@ def create(self, defn, check, allow_reboot, allow_recreate): if defn.assume_role_policy != "": self._conn.update_assume_role_policy(defn.role_name, defn.assume_role_policy) - with self.depl._state.atomic: + with self.depl._state.db: self.state = self.UP self.role_name = defn.role_name self.policy = defn.policy diff --git a/nixops/resources/s3_bucket.py b/nixops/resources/s3_bucket.py index fd993850b..142f6209e 100644 --- a/nixops/resources/s3_bucket.py +++ b/nixops/resources/s3_bucket.py @@ -101,7 +101,7 @@ def create(self, defn, check, allow_reboot, allow_recreate): # [http://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketDELETEpolicy.html] if e.status != 204: raise # (204 : Bucket didn't have any policy to delete) - with self.depl._state.atomic: + with self.depl._state.db: self.state = self.UP self.bucket_name = defn.bucket_name self.region = defn.region diff --git a/nixops/resources/sns_topic.py b/nixops/resources/sns_topic.py index e10cdad33..fddc05666 100644 --- a/nixops/resources/sns_topic.py +++ b/nixops/resources/sns_topic.py @@ -74,7 +74,7 @@ def _destroy(self): self.connect() self.log("destroying SNS topic ‘{0}’...".format(self.topic_name)) self._conn.delete_topic(self.arn) - with self.depl._state.atomic: + with self.depl._state.db: self.state = self.MISSING self.topic_name = None self.region = None @@ -136,7 +136,7 @@ def create(self, defn, check, allow_reboot, allow_recreate): if subscriber_arn != "PendingConfirmation": self._conn.unsubscribe(subscription=subscriber_arn) - with self.depl._state.atomic: + with self.depl._state.db: self.state = self.UP self.topic_name = defn.config['name'] self.display_name = defn.config['displayName'] diff --git a/nixops/resources/sqs_queue.py b/nixops/resources/sqs_queue.py index 06596924c..207fb794c 100644 --- a/nixops/resources/sqs_queue.py +++ b/nixops/resources/sqs_queue.py @@ -84,7 +84,7 @@ def _destroy(self): if q: self.log("destroying SQS queue ‘{0}’...".format(self.queue_name)) self._conn.delete_queue(q) - with self.depl._state.atomic: + with self.depl._state.db: self.state = self.MISSING self.queue_name = None self.queue_base_name = None @@ -122,7 +122,7 @@ def create(self, defn, check, allow_reboot, allow_recreate): self.log("creating SQS queue ‘{0}’...".format(defn.queue_name)) q = nixops.ec2_utils.retry(lambda: self._conn.create_queue(defn.queue_name, defn.visibility_timeout), error_codes = ['AWS.SimpleQueueService.QueueDeletedRecently']) - with self.depl._state.atomic: + with self.depl._state.db: self.state = self.UP self.queue_name = defn.queue_name self.url = q.url diff --git a/nixops/resources/ssh_keypair.py b/nixops/resources/ssh_keypair.py index 612df9261..faf15907b 100644 --- a/nixops/resources/ssh_keypair.py +++ b/nixops/resources/ssh_keypair.py @@ -46,7 +46,7 @@ def create(self, defn, check, allow_reboot, allow_recreate): # Generate the key pair locally. if not self.public_key: (private, public) = nixops.util.create_key_pair(type="ed25519") - with self.depl._state.atomic: + with self.depl._state.db: self.public_key = public self.private_key = private self.state = state = nixops.resources.ResourceState.UP diff --git a/nixops/state/json_file.py b/nixops/state/json_file.py index eb9254b15..873b27ab5 100644 --- a/nixops/state/json_file.py +++ b/nixops/state/json_file.py @@ -133,9 +133,6 @@ def __init__(self, json_file): if version > 0: raise Exception("this NixOps version is too old to deal with JSON schema version {0}".format(version)) - # The "db" will provide atomicity as well. - self.atomic = self.db - ############################################################################################### ## Deployment diff --git a/nixops/state/sqlite3_file.py b/nixops/state/sqlite3_file.py index 167e27d82..592874f9a 100644 --- a/nixops/state/sqlite3_file.py +++ b/nixops/state/sqlite3_file.py @@ -104,20 +104,18 @@ def __init__(self, db_file): else: raise Exception("this NixOps version is too old to deal with schema version {0}".format(version)) - self.__db = db + self.db = db - # TODO; implement some other special wrapper that ONLY does the transaction stuff. - self.atomic = db def close(self): - self.__db.close() + self.db.close() ############################################################################################### ## Deployment def query_deployments(self): """Return the UUIDs of all deployments in the database.""" - c = self.__db.cursor() + c = self.db.cursor() c.execute("select uuid from Deployments") res = c.fetchall() return [x[0] for x in res] @@ -134,7 +132,7 @@ def get_all_deployments(self): return res def _find_deployment(self, uuid=None): - c = self.__db.cursor() + c = self.db.cursor() if not uuid: c.execute("select uuid from Deployments") else: @@ -167,18 +165,18 @@ def create_deployment(self, uuid=None): if not uuid: import uuid uuid = str(uuid.uuid1()) - with self.__db: - self.__db.execute("insert into Deployments(uuid) values (?)", (uuid,)) + with self.db: + self.db.execute("insert into Deployments(uuid) values (?)", (uuid,)) return nixops.deployment.Deployment(self, uuid, sys.stderr) def _delete_deployment(self, deployment_uuid): """NOTE: This is UNSAFE, it's guarded in nixops/deployment.py. Do not call this function except from there!""" - self.__db.execute("delete from Deployments where uuid = ?", (deployment_uuid,)) + self.db.execute("delete from Deployments where uuid = ?", (deployment_uuid,)) def clone_deployment(self, deployment_uuid): - with self.__db: + with self.db: new = self.create_deployment() - self.__db.execute("insert into DeploymentAttrs (deployment, name, value) " + + self.db.execute("insert into DeploymentAttrs (deployment, name, value) " + "select ?, name, value from DeploymentAttrs where deployment = ?", (new.uuid, deployment_uuid)) new.configs_path = None @@ -189,8 +187,8 @@ def clone_deployment(self, deployment_uuid): def get_resources_for(self, deployment): """Get all the resources for a certain deployment""" resources = {} - with self.__db: - c = self.__db.cursor() + with self.db: + c = self.db.cursor() c.execute("select id, name, type from Resources where deployment = ?", (deployment.uuid,)) for (id, name, type) in c.fetchall(): r = self._create_state(deployment, type, name, id) @@ -199,8 +197,8 @@ def get_resources_for(self, deployment): def set_deployment_attrs(self, deployment_uuid, attrs): """Update deployment attributes in the state.""" - with self.__db: - c = self.__db.cursor() + with self.db: + c = self.db.cursor() for n, v in attrs.iteritems(): if v == None: c.execute("delete from DeploymentAttrs where deployment = ? and name = ?", (deployment_uuid, n)) @@ -209,22 +207,22 @@ def set_deployment_attrs(self, deployment_uuid, attrs): (deployment_uuid, n, v)) def del_deployment_attr(self, deployment_uuid, attr_name): - with self.__db: - self.__db.execute("delete from DeploymentAttrs where deployment = ? and name = ?", (deployment_uuid, attr_name)) + with self.db: + self.db.execute("delete from DeploymentAttrs where deployment = ? and name = ?", (deployment_uuid, attr_name)) def get_deployment_attr(self, deployment_uuid, name): """Get a deployment attribute from the state.""" - with self.__db: - c = self.__db.cursor() + with self.db: + c = self.db.cursor() c.execute("select value from DeploymentAttrs where deployment = ? and name = ?", (deployment_uuid, name)) row = c.fetchone() if row != None: return row[0] return nixops.util.undefined def get_all_deployment_attrs(self, deployment_uuid): - with self.__db: - c = self.__db.cursor() + with self.db: + c = self.db.cursor() c.execute("select name, value from DeploymentAttrs where deployment = ?", (deployment_uuid)) rows = c.fetchall() res = {row[0]: row[1] for row in rows} @@ -258,7 +256,7 @@ def __exit__(self, exception_type, exception_value, exception_traceback): ## Resources def create_resource(self, deployment, name, type): - c = self.__db.cursor() + c = self.db.cursor() c.execute("select 1 from Resources where deployment = ? and name = ?", (deployment.uuid, name)) if len(c.fetchall()) != 0: raise Exception("resource already exists in database!") @@ -269,18 +267,18 @@ def create_resource(self, deployment, name, type): return r def delete_resource(self, deployment_uuid, res_id): - with self.__db: - self.__db.execute("delete from Resources where deployment = ? and id = ?", (deployment_uuid, res_id)) + with self.db: + self.db.execute("delete from Resources where deployment = ? and id = ?", (deployment_uuid, res_id)) def _rename_resource(self, deployment_uuid, resource_id, new_name): """NOTE: Invariants are checked in nixops/deployment.py#rename""" - with self.__db: - self.__db.execute("update Resources set name = ? where deployment = ? and id = ?", (new_name, deployment_uuid, resource_id)) + with self.db: + self.db.execute("update Resources set name = ? where deployment = ? and id = ?", (new_name, deployment_uuid, resource_id)) def set_resource_attrs(self, _deployment_uuid, resource_id, attrs): - with self.__db: - c = self.__db.cursor() + with self.db: + c = self.db.cursor() for n, v in attrs.iteritems(): if v == None: c.execute("delete from ResourceAttrs where machine = ? and name = ?", (resource_id, n)) @@ -289,21 +287,21 @@ def set_resource_attrs(self, _deployment_uuid, resource_id, attrs): (resource_id, n, v)) def del_resource_attr(self, _deployment_uuid, resource_id, name): - with self.__db: - self.__db.execute("delete from ResourceAttrs where machine = ? and name = ?", (resource_id, name)) + with self.db: + self.db.execute("delete from ResourceAttrs where machine = ? and name = ?", (resource_id, name)) def get_resource_attr(self, _deployment_uuid, resource_id, name): """Get a machine attribute from the state file.""" - with self.__db: - c = self.__db.cursor() + with self.db: + c = self.db.cursor() c.execute("select value from ResourceAttrs where machine = ? and name = ?", (resource_id, name)) row = c.fetchone() if row != None: return row[0] return nixops.util.undefined def get_all_resource_attrs(self, deployment_uuid, resource_id): - with self.__db: - c = self.__db.cursor() + with self.db: + c = self.db.cursor() c.execute("select name, value from ResourceAttrs where machine = ?", (resource_id,)) rows = c.fetchall() res = {row[0]: row[1] for row in rows} diff --git a/scripts/nixops b/scripts/nixops index 47cd7eba7..c912376b5 100755 --- a/scripts/nixops +++ b/scripts/nixops @@ -499,7 +499,7 @@ def op_import(): for uuid, attrs in dump.iteritems(): if uuid in existing: raise Exception("state file already contains a deployment with UUID ‘{0}’".format(uuid)) - with state.atomic: + with state.db: depl = state.create_deployment(uuid=uuid) depl.import_(attrs) sys.stderr.write("added deployment ‘{0}’\n".format(uuid)) From 38aa18129f2ba4f6941a09e2ad2d571747efd98d Mon Sep 17 00:00:00 2001 From: Maarten Hoogendoorn Date: Wed, 29 Mar 2017 13:28:54 +0200 Subject: [PATCH 046/123] Typo --- doc/manual/hacking.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/hacking.xml b/doc/manual/hacking.xml index 6f070ae87..ba28857af 100644 --- a/doc/manual/hacking.xml +++ b/doc/manual/hacking.xml @@ -59,7 +59,7 @@ filter on one or more tags. To run e.g. only the virtualbox tests, run: -$ python 2 tests.py tests.functional -A vbox +$ python2 tests.py tests.functional -A vbox From 4aaf084d0459d4ad61f7cc5455beefa72de9902b Mon Sep 17 00:00:00 2001 From: Maarten Hoogendoorn Date: Wed, 29 Mar 2017 13:35:50 +0200 Subject: [PATCH 047/123] Rename file -> sqlite3_file in tests --- tests/__init__.py | 6 +++--- tests/functional/__init__.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/__init__.py b/tests/__init__.py index e77249d78..b586d2885 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -3,14 +3,14 @@ import sys import threading from os import path -import nixops.state.file +import nixops.state.sqlite3_file _multiprocess_shared_ = True db_file = '%s/test.nixops' % (path.dirname(__file__)) def setup(): - nixops.state.file.StateFile(db_file).close() + nixops.state.sqlite3_file.StateFile(db_file).close() def destroy(sf, uuid): depl = sf.open_deployment(uuid) @@ -27,7 +27,7 @@ def destroy(sf, uuid): depl.logger.log("deployment ‘{0}’ destroyed".format(uuid)) def teardown(): - sf = nixops.state.file.StateFile(db_file) + sf = nixops.state.sqlite3_file.StateFile(db_file) uuids = sf.query_deployments() threads = [] for uuid in uuids: diff --git a/tests/functional/__init__.py b/tests/functional/__init__.py index 76cee27fd..e721b2132 100644 --- a/tests/functional/__init__.py +++ b/tests/functional/__init__.py @@ -1,6 +1,6 @@ import os from os import path -import nixops.state.file +import nixops.state.sqlite3_file from tests import db_file @@ -8,7 +8,7 @@ class DatabaseUsingTest(object): _multiprocess_can_split_ = True def setup(self): - self.sf = nixops.state.file.StateFile(db_file) + self.sf = nixops.state.sqlite3_file.StateFile(db_file) def teardown(self): self.sf.close() From 6950f37f3ef65ab55ba99a5a35bfd9ae20919e8d Mon Sep 17 00:00:00 2001 From: Maarten Hoogendoorn Date: Wed, 29 Mar 2017 13:54:13 +0200 Subject: [PATCH 048/123] Rename file -> sqlite3_file in scripts/nixops --- scripts/nixops | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/nixops b/scripts/nixops index c912376b5..b9b83b99a 100755 --- a/scripts/nixops +++ b/scripts/nixops @@ -5,7 +5,7 @@ from nixops import deployment from nixops.nix_expr import py2nix from nixops.parallel import MultipleExceptions, run_tasks -import nixops.state.file +import nixops.state.sqlite3_file import prettytable import argparse import os From 6576e167871ff807b5d17a286b76a3990fe1f2b8 Mon Sep 17 00:00:00 2001 From: Maarten Hoogendoorn Date: Wed, 29 Mar 2017 14:04:26 +0200 Subject: [PATCH 049/123] Rename file -> sqlite3_file in scripts/nixops --- scripts/nixops | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/nixops b/scripts/nixops index b9b83b99a..dc69f0073 100755 --- a/scripts/nixops +++ b/scripts/nixops @@ -695,7 +695,7 @@ subparsers = parser.add_subparsers(help='sub-command help') def add_subparser(name, help): subparser = subparsers.add_parser(name, help=help) subparser.add_argument('--state', '-s', dest='state_url', metavar='FILE', - default=os.environ.get("NIXOPS_STATE_URL", nixops.state.file.get_default_state_file()), help='URL that points to the state provider.') + default=os.environ.get("NIXOPS_STATE_URL", nixops.state.sqlite3_file.get_default_state_file()), help='URL that points to the state provider.') subparser.add_argument('--deployment', '-d', dest='deployment', metavar='UUID_OR_NAME', default=os.environ.get("NIXOPS_DEPLOYMENT", os.environ.get("CHARON_DEPLOYMENT", None)), help='UUID or symbolic name of the deployment') subparser.add_argument('--debug', action='store_true', help='enable debug output') From 4ff5a248a2210efcd05350ad97c917e0f5222dec Mon Sep 17 00:00:00 2001 From: Matthew O'Gorman Date: Sun, 4 Feb 2018 18:04:59 -0500 Subject: [PATCH 050/123] fixed probable typo from fork --- nixops/backends/hetzner.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nixops/backends/hetzner.py b/nixops/backends/hetzner.py index d1a50ca20..4b2661646 100644 --- a/nixops/backends/hetzner.py +++ b/nixops/backends/hetzner.py @@ -594,7 +594,7 @@ def create(self, defn, check, allow_reboot, allow_recreate): self.log_start("creating an exclusive robot admin sub-account " "for ‘{0}’... ".format(self.name)) server = self._get_server_from_main_robot(self.main_ipv4, defn) - with self.depl._state._db: + with self.depl._state.db: (self.robot_admin_user, self.robot_admin_pass) = server.admin.create() self.log_end("done. ({0})".format(self.robot_admin_user)) @@ -609,7 +609,7 @@ def create(self, defn, check, allow_reboot, allow_recreate): ) if robot_user != self.robot_admin_user or \ robot_pass != self.robot_admin_pass: - with self.depl._state._db: + with self.depl._state.db: (self.robot_admin_user, self.robot_admin_pass) = (robot_user, robot_pass) From f7ef59ffed588cd370b1aca0486bc7840c972e16 Mon Sep 17 00:00:00 2001 From: Matthew O'Gorman Date: Sun, 4 Feb 2018 22:35:09 -0500 Subject: [PATCH 051/123] change all instances to use state wrapper --- nixops/resources/aws_vpn_connection.py | 4 ++-- nixops/resources/aws_vpn_connection_route.py | 4 ++-- nixops/resources/aws_vpn_gateway.py | 4 ++-- nixops/resources/ec2_common.py | 2 +- nixops/resources/ec2_rds_dbsecurity_group.py | 8 ++++---- nixops/resources/elastic_ip.py | 2 +- nixops/resources/route53_health_check.py | 6 +++--- nixops/resources/route53_hosted_zone.py | 6 +++--- nixops/resources/route53_recordset.py | 4 ++-- nixops/resources/vpc.py | 12 ++++++------ nixops/resources/vpc_customer_gateway.py | 6 +++--- nixops/resources/vpc_dhcp_options.py | 6 +++--- nixops/resources/vpc_egress_only_internet_gateway.py | 4 ++-- nixops/resources/vpc_endpoint.py | 6 +++--- nixops/resources/vpc_internet_gateway.py | 4 ++-- nixops/resources/vpc_nat_gateway.py | 6 +++--- nixops/resources/vpc_network_acl.py | 8 ++++---- nixops/resources/vpc_network_interface.py | 6 +++--- nixops/resources/vpc_network_interface_attachment.py | 8 ++++---- nixops/resources/vpc_route.py | 4 ++-- nixops/resources/vpc_route_table.py | 6 +++--- nixops/resources/vpc_route_table_association.py | 4 ++-- nixops/resources/vpc_subnet.py | 10 +++++----- nixops/state.py | 2 +- 24 files changed, 66 insertions(+), 66 deletions(-) diff --git a/nixops/resources/aws_vpn_connection.py b/nixops/resources/aws_vpn_connection.py index b2a3237a2..080b3be71 100644 --- a/nixops/resources/aws_vpn_connection.py +++ b/nixops/resources/aws_vpn_connection.py @@ -89,7 +89,7 @@ def realize_create_vpn_conn(self, allow_recreate): ) vpn_conn_id = vpn_connection['VpnConnection']['VpnConnectionId'] - with self.depl._db: + with self.depl._state.db: self.state = self.UP self._state['vpnConnectionId'] = vpn_conn_id self._state['vpnGatewayId'] = vpn_gateway_id @@ -114,7 +114,7 @@ def _destroy(self): else: raise e - with self.depl._db: + with self.depl._state.db: self.state = self.MISSING self._state['vpnConnectionId'] = None self._state['vpnGatewayId'] = None diff --git a/nixops/resources/aws_vpn_connection_route.py b/nixops/resources/aws_vpn_connection_route.py index 93ba3b29d..2da1631b0 100644 --- a/nixops/resources/aws_vpn_connection_route.py +++ b/nixops/resources/aws_vpn_connection_route.py @@ -80,7 +80,7 @@ def realize_create_vpn_route(self, allow_recreate): DestinationCidrBlock=config['destinationCidrBlock'], VpnConnectionId=vpn_conn_id) - with self.depl._db: + with self.depl._state.db: self.state = self.UP self._state['vpnConnectionId'] = vpn_conn_id self._state['destinationCidrBlock'] = config['destinationCidrBlock'] @@ -98,7 +98,7 @@ def _destroy(self): else: raise e - with self.depl._db: + with self.depl._state.db: self.state = self.MISSING self._state['vpnConnectionId'] = None self._state['destinationCidrBlock'] = None diff --git a/nixops/resources/aws_vpn_gateway.py b/nixops/resources/aws_vpn_gateway.py index 57c168987..d97087878 100644 --- a/nixops/resources/aws_vpn_gateway.py +++ b/nixops/resources/aws_vpn_gateway.py @@ -85,7 +85,7 @@ def realize_create_vpn_gtw(self, allow_recreate): VpnGatewayId=vpn_gtw_id) #TODO wait for the attchement state - with self.depl._db: + with self.depl._state.db: self.state = self.UP self._state['vpnGatewayId'] = vpn_gtw_id self._state['vpcId'] = vpc_id @@ -122,7 +122,7 @@ def _destroy(self): else: raise e - with self.depl._db: + with self.depl._state.db: self.state = self.MISSING self._state['region'] = None self._state['vpnGatewayId'] = None diff --git a/nixops/resources/ec2_common.py b/nixops/resources/ec2_common.py index d58f53a08..e3d1eacd5 100644 --- a/nixops/resources/ec2_common.py +++ b/nixops/resources/ec2_common.py @@ -19,7 +19,7 @@ def _retry(self, fun, **kwargs): def get_common_tags(self): tags = {'CharonNetworkUUID': self.depl.uuid, 'CharonMachineName': self.name, - 'CharonStateFile': "{0}@{1}:{2}".format(getpass.getuser(), socket.gethostname(), self.depl._db.db_file)} + 'CharonStateFile': "{0}@{1}:{2}".format(getpass.getuser(), socket.gethostname(), self.depl._state.db.db_file)} if self.depl.name: tags['CharonNetworkName'] = self.depl.name return tags diff --git a/nixops/resources/ec2_rds_dbsecurity_group.py b/nixops/resources/ec2_rds_dbsecurity_group.py index 6f5b122f7..d73b4530f 100644 --- a/nixops/resources/ec2_rds_dbsecurity_group.py +++ b/nixops/resources/ec2_rds_dbsecurity_group.py @@ -86,7 +86,7 @@ def generate_rule(rule): for rule in response['DBSecurityGroups'][0].get('IPRanges', []): rules.append(generate_rule(rule)) - with self.depl._db: + with self.depl._state.db: self._state['rules'] = rules def realize_create_sg(self, allow_recreate): @@ -107,7 +107,7 @@ def realize_create_sg(self, allow_recreate): self.get_client("rds").create_db_security_group( DBSecurityGroupName=config['groupName'], DBSecurityGroupDescription=config['description']) - with self.depl._db: + with self.depl._state.db: self.state = self.UP self._state['groupName'] = config['groupName'] self._state['description'] = config['description'] @@ -128,7 +128,7 @@ def realize_rules_change(self, allow_recreate): kwargs = self.process_rule(rule) self.get_client("rds").authorize_db_security_group_ingress(**kwargs) - with self.depl._db: + with self.depl._state.db: self._state['rules'] = config['rules'] def process_rule(self, config): @@ -153,7 +153,7 @@ def _destroy(self): else: raise error - with self.depl._db: + with self.depl._state.db: self.state = self.MISSING self._state['groupName'] = None self._state['region'] = None diff --git a/nixops/resources/elastic_ip.py b/nixops/resources/elastic_ip.py index cca413b78..df3a45754 100644 --- a/nixops/resources/elastic_ip.py +++ b/nixops/resources/elastic_ip.py @@ -123,7 +123,7 @@ def destroy(self, wipe=False): else: self._client.release_address(PublicIp=eip['PublicIp']) - with self.depl._db: + with self.depl._state.db: self.state = self.MISSING self.public_ipv4 = None self.allocation_id = None diff --git a/nixops/resources/route53_health_check.py b/nixops/resources/route53_health_check.py index b6f7a1800..883628e25 100644 --- a/nixops/resources/route53_health_check.py +++ b/nixops/resources/route53_health_check.py @@ -152,7 +152,7 @@ def cannot_change(desc, sk, d): ref = str(uuid.uuid1()) self.log('creating health check') health_check = client.create_health_check(CallerReference=ref, HealthCheckConfig=cfg) - with self.depl._db: + with self.depl._state.db: self.state = self.UP self.health_check_id = health_check['HealthCheck']['Id'] else: @@ -165,7 +165,7 @@ def cannot_change(desc, sk, d): self.log('updating health check') client.update_health_check(**cfg) - with self.depl._db: + with self.depl._state.db: self.health_check_config = orig_cfg self.child_health_checks = defn.child_health_checks @@ -184,7 +184,7 @@ def destroy(self, wipe=False): pass raise - with self.depl._db: + with self.depl._state.db: self.state = self.MISSING return True diff --git a/nixops/resources/route53_hosted_zone.py b/nixops/resources/route53_hosted_zone.py index ffaf8126d..f071a557a 100644 --- a/nixops/resources/route53_hosted_zone.py +++ b/nixops/resources/route53_hosted_zone.py @@ -96,7 +96,7 @@ def create(self, defn, check, allow_reboot, allow_recreate): self.log('creating hosted zone for {}'.format(defn.zone_name)) hosted_zone = client.create_hosted_zone(**args) - with self.depl._db: + with self.depl._state.db: self.state = self.UP self.zone_id = hosted_zone['HostedZone']['Id'] self.private_zone = defn.private_zone @@ -119,7 +119,7 @@ def create(self, defn, check, allow_reboot, allow_recreate): for assoc in tbd: client.disassociate_vpc_from_hosted_zone(HostedZoneId=self.zone_id, VPC={ 'VPCId': assoc['vpcId'], 'VPCRegion': assoc['region'] }) - with self.depl._db: + with self.depl._state.db: self.associated_vpcs = defn.associated_vpcs return True @@ -137,7 +137,7 @@ def destroy(self, wipe=False): pass raise - with self.depl._db: + with self.depl._state.db: self.state = self.MISSING return True diff --git a/nixops/resources/route53_recordset.py b/nixops/resources/route53_recordset.py index e88c80310..e4ae44687 100644 --- a/nixops/resources/route53_recordset.py +++ b/nixops/resources/route53_recordset.py @@ -177,7 +177,7 @@ def resolve_machine_ip(v): ChangeBatch=self.make_batch('UPSERT', defn) )) - with self.depl._db: + with self.depl._state.db: self.state = self.UP self.zone_name = zone_name self.zone_id = zone_id @@ -249,7 +249,7 @@ def destroy(self, wipe=False): HostedZoneId=self.zone_id, ChangeBatch=self.make_batch('DELETE', self))) - with self.depl._db: + with self.depl._state.db: self.state = self.MISSING return True diff --git a/nixops/resources/vpc.py b/nixops/resources/vpc.py index 285883c26..28d1344aa 100755 --- a/nixops/resources/vpc.py +++ b/nixops/resources/vpc.py @@ -85,7 +85,7 @@ def _destroy(self): self.cleanup_state() def cleanup_state(self): - with self.depl._db: + with self.depl._state.db: self.state = self.MISSING self._state['vpcId'] = None self._state['region'] = None @@ -130,7 +130,7 @@ def wait_for_vpc_available(self, vpc_id): raise Exception("couldn't find vpc {}, please run a deploy with --check".format(self._state["vpcId"])) self.log_end(" done") - with self.depl._db: + with self.depl._state.db: self.state = self.UP def realize_create_vpc(self, allow_recreate): @@ -151,7 +151,7 @@ def realize_create_vpc(self, allow_recreate): InstanceTenancy=config['instanceTenancy']) self.vpc_id = vpc.get('Vpc').get('VpcId') - with self.depl._db: + with self.depl._state.db: self.state = self.STARTING self._state["vpcId"] = self.vpc_id self._state["region"] = config['region'] @@ -171,7 +171,7 @@ def realize_classic_link_change(self, allow_recreate): self.get_client().enable_vpc_classic_link(VpcId=self.vpc_id) elif config['enableClassicLink'] == False and self._state.get('enableClassicLink', None): self.get_client().disable_vpc_classic_link(VpcId=self.vpc_id) - with self.depl._db: + with self.depl._state.db: self._state["enableClassicLink"] = config['enableClassicLink'] def realize_dns_config(self, allow_recreate): @@ -184,7 +184,7 @@ def realize_dns_config(self, allow_recreate): EnableDnsHostnames={ 'Value': config['enableDnsHostnames'] }) - with self.depl._db: + with self.depl._state.db: self._state["enableDnsSupport"] = config['enableDnsSupport'] self._state["enableDnsHostnames"] = config['enableDnsHostnames'] @@ -229,7 +229,7 @@ def realize_associate_ipv6_cidr_block(self, allow_recreate): self.get_client().disassociate_vpc_cidr_block( AssociationId=self._state['associationId']) - with self.depl._db: + with self.depl._state.db: self._state["amazonProvidedIpv6CidrBlock"] = config['amazonProvidedIpv6CidrBlock'] if assign_cidr: self._state['associationId'] = association_id diff --git a/nixops/resources/vpc_customer_gateway.py b/nixops/resources/vpc_customer_gateway.py index 9cdeb8929..129d7d846 100644 --- a/nixops/resources/vpc_customer_gateway.py +++ b/nixops/resources/vpc_customer_gateway.py @@ -79,12 +79,12 @@ def realize_create_customer_gtw(self, allow_recreate): Type=config['type']) customer_gtw_id = response['CustomerGateway']['CustomerGatewayId'] - with self.depl._db: self.state = self.STARTING + with self.depl._state.db: self.state = self.STARTING waiter = self.get_client().get_waiter('customer_gateway_available') waiter.wait(CustomerGatewayIds=[customer_gtw_id]) - with self.depl._db: + with self.depl._state.db: self.state = self.UP self._state['region'] = config['region'] self._state['customerGatewayId'] = customer_gtw_id @@ -110,7 +110,7 @@ def _destroy(self): raise e #TODO wait for customer gtw to be deleted - with self.depl._db: + with self.depl._state.db: self.state = self.MISSING self._state['region'] = None self._state['customerGatewayId'] = None diff --git a/nixops/resources/vpc_dhcp_options.py b/nixops/resources/vpc_dhcp_options.py index ee67f7271..114692687 100644 --- a/nixops/resources/vpc_dhcp_options.py +++ b/nixops/resources/vpc_dhcp_options.py @@ -111,7 +111,7 @@ def create_dhcp_options(dhcp_config): return response.get('DhcpOptions').get('DhcpOptionsId') dhcp_options_id = create_dhcp_options(dhcp_config) - with self.depl._db: + with self.depl._state.db: self.state = self.STARTING self._state['vpcId'] = vpc_id self._state['dhcpOptionsId'] = dhcp_options_id @@ -122,7 +122,7 @@ def create_dhcp_options(dhcp_config): self._state['netbiosNodeType'] = config["netbiosNodeType"] self.get_client().associate_dhcp_options(DhcpOptionsId=dhcp_options_id, VpcId=vpc_id) - with self.depl._db: + with self.depl._state.db: self.state = self.UP def realize_update_tag(self, allow_recreate): @@ -145,7 +145,7 @@ def _destroy(self): else: raise e - with self.depl._db: + with self.depl._state.db: self.state = self.MISSING self._state['vpcId'] = None self._state['dhcpOptions'] = None diff --git a/nixops/resources/vpc_egress_only_internet_gateway.py b/nixops/resources/vpc_egress_only_internet_gateway.py index 81609be8f..d76791f60 100644 --- a/nixops/resources/vpc_egress_only_internet_gateway.py +++ b/nixops/resources/vpc_egress_only_internet_gateway.py @@ -82,7 +82,7 @@ def realize_create_gtw(self, allow_recreate): response = self.get_client().create_egress_only_internet_gateway(VpcId=vpc_id) igw_id = response['EgressOnlyInternetGateway']['EgressOnlyInternetGatewayId'] - with self.depl._db: + with self.depl._state.db: self.state = self.UP self._state['region'] = config['region'] self._state['vpcId'] = vpc_id @@ -93,7 +93,7 @@ def _destroy(self): self.log("deleting egress only internet gateway {0}".format(self._state['egressOnlyInternetGatewayId'])) self.get_client().delete_egress_only_internet_gateway(EgressOnlyInternetGatewayId=self._state['egressOnlyInternetGatewayId']) - with self.depl._db: + with self.depl._state.db: self.state = self.MISSING self._state['region'] = None self._state['vpcId'] = None diff --git a/nixops/resources/vpc_endpoint.py b/nixops/resources/vpc_endpoint.py index 1413feb3d..55671dbb6 100644 --- a/nixops/resources/vpc_endpoint.py +++ b/nixops/resources/vpc_endpoint.py @@ -89,7 +89,7 @@ def realize_create_endpoint(self, allow_recreate): VpcId=vpc_id) endpoint_id = response['VpcEndpoint']['VpcEndpointId'] - with self.depl._db: + with self.depl._state.db: self.state = self.UP self._state['endpointId'] = endpoint_id self._state['vpcId'] = vpc_id @@ -117,7 +117,7 @@ def realize_modify_endpoint(self, allow_recreate): self.get_client().modify_vpc_endpoint(**edp_input) - with self.depl._db: + with self.depl._state.db: self._state['policy'] = config['policy'] self._state['routeTableIds'] = new_rtbs @@ -132,7 +132,7 @@ def _destroy(self): else: raise e - with self.depl._db: + with self.depl._state.db: self.state = self.MISSING self._state['endpointId'] = None self._state['vpcId'] = None diff --git a/nixops/resources/vpc_internet_gateway.py b/nixops/resources/vpc_internet_gateway.py index 49cc7b140..4a3045b00 100644 --- a/nixops/resources/vpc_internet_gateway.py +++ b/nixops/resources/vpc_internet_gateway.py @@ -89,7 +89,7 @@ def realize_create_gtw(self, allow_recreate): self.log("attaching internet gateway {0} to vpc {1}".format(igw_id, vpc_id)) self.get_client().attach_internet_gateway(InternetGatewayId=igw_id, VpcId=vpc_id) - with self.depl._db: + with self.depl._state.db: self.state = self.UP self._state['region'] = config['region'] self._state['vpcId'] = vpc_id @@ -110,7 +110,7 @@ def _destroy(self): self.log("deleting internet gateway {0}".format(self._state['internetGatewayId'])) self.get_client().delete_internet_gateway(InternetGatewayId=self._state['internetGatewayId']) - with self.depl._db: + with self.depl._state.db: self.state = self.MISSING self._state['region'] = None self._state['vpcId'] = None diff --git a/nixops/resources/vpc_nat_gateway.py b/nixops/resources/vpc_nat_gateway.py index 6a8f7f91a..7295e4e1d 100644 --- a/nixops/resources/vpc_nat_gateway.py +++ b/nixops/resources/vpc_nat_gateway.py @@ -101,7 +101,7 @@ def realize_create_gtw(self, allow_recreate): SubnetId=subnet_id) gtw_id = response['NatGateway']['NatGatewayId'] - with self.depl._db: + with self.depl._state.db: self.state = self.UP self._state['subnetId'] = subnet_id self._state['allocationId'] = allocation_id @@ -137,7 +137,7 @@ def _destroy(self): self.log("deleting vpc NAT gateway {}".format(self._state['natGatewayId'])) try: self.get_client().delete_nat_gateway(NatGatewayId=self._state['natGatewayId']) - with self.depl._db: self.state = self.STOPPING + with self.depl._state.db: self.state = self.STOPPING except botocore.exceptions.ClientError as e: if e.response['Error']['Code'] == "InvalidNatGatewayID.NotFound" or e.response['Error']['Code'] == "NatGatewayNotFound": self.warn("nat gateway {} was already deleted".format(self._state['natGatewayId'])) @@ -147,7 +147,7 @@ def _destroy(self): if self.state == self.STOPPING: self.wait_for_nat_gtw_deletion() - with self.depl._db: + with self.depl._state.db: self.state = self.MISSING self._state['region'] = None self._state['subnetId'] = None diff --git a/nixops/resources/vpc_network_acl.py b/nixops/resources/vpc_network_acl.py index 026058940..6a5cf1b72 100644 --- a/nixops/resources/vpc_network_acl.py +++ b/nixops/resources/vpc_network_acl.py @@ -89,7 +89,7 @@ def realize_create_network_acl(self, allow_recreate): response = self.get_client().create_network_acl(VpcId=vpc_id) self.network_acl_id = response['NetworkAcl']['NetworkAclId'] - with self.depl._db: + with self.depl._state.db: self.state = self.UP self._state['vpcId'] = vpc_id self._state['networkAclId'] = self.network_acl_id @@ -112,7 +112,7 @@ def realize_entries_change(self, allow_recreate): for entry in to_create: rule = self.process_rule_entry(entry) self.get_client().create_network_acl_entry(**rule) - with self.depl._db: + with self.depl._state.db: self._state['entries'] = config['entries'] def realize_subnets_change(self, allow_recreate): @@ -147,7 +147,7 @@ def realize_subnets_change(self, allow_recreate): self.log("associating subnet {0} to network acl {1}".format(subnet, self.network_acl_id)) self.get_client().replace_network_acl_association(AssociationId=association_id, NetworkAclId=self.network_acl_id) - with self.depl._db: + with self.depl._state.db: self._state['subnetIds'] = new_subnets def get_default_network_acl(self, vpc_id): @@ -193,7 +193,7 @@ def _destroy(self): else: raise e - with self.depl._db: + with self.depl._state.db: self.state = self.MISSING self._state['networkAclId'] = None self._state['region'] = None diff --git a/nixops/resources/vpc_network_interface.py b/nixops/resources/vpc_network_interface.py index 8a36b513a..8797e4a9b 100644 --- a/nixops/resources/vpc_network_interface.py +++ b/nixops/resources/vpc_network_interface.py @@ -99,7 +99,7 @@ def split_ips(eni_ips): primary, secondary = split_ips(eni['PrivateIpAddresses']) - with self.depl._db: + with self.depl._state.db: self.state = self.UP self._state['subnetId'] = eni_input['SubnetId'] self._state['networkInterfaceId'] = eni['NetworkInterfaceId'] @@ -169,7 +169,7 @@ def realize_modify_eni_attrs(self, allow_recreate): SourceDestCheck={ 'Value':config['sourceDestCheck'] }) - with self.depl._db: + with self.depl._state.db: self._state['description'] = config['description'] self._state['securityGroups'] = groups self._state['sourceDestCheck'] = config['sourceDestCheck'] @@ -191,7 +191,7 @@ def _destroy(self): else: raise e - with self.depl._db: + with self.depl._state.db: self.state = self.MISSING self._state['networkInterfaceId'] = None self._state['primaryPrivateIpAddress'] = None diff --git a/nixops/resources/vpc_network_interface_attachment.py b/nixops/resources/vpc_network_interface_attachment.py index b72dd8cd7..cb22b924f 100644 --- a/nixops/resources/vpc_network_interface_attachment.py +++ b/nixops/resources/vpc_network_interface_attachment.py @@ -94,7 +94,7 @@ def wait_for_eni_attachment(self, eni_id): self.log_end(" done") - with self.depl._db: + with self.depl._state.db: self.state = self.UP def realize_create_eni_attachment(self, allow_recreate): @@ -123,7 +123,7 @@ def realize_create_eni_attachment(self, allow_recreate): InstanceId=vm_id, NetworkInterfaceId=eni_id) - with self.depl._db: + with self.depl._state.db: self.state = self.STARTING self._state['attachmentId'] = eni_attachment['AttachmentId'] self._state['instanceId'] = vm_id @@ -157,7 +157,7 @@ def _destroy(self): try: self.get_client().detach_network_interface(AttachmentId=self._state['attachmentId'], Force=True) - with self.depl._db: + with self.depl._state.db: self.state = self.STOPPING except botocore.exceptions.ClientError as e: @@ -168,7 +168,7 @@ def _destroy(self): if self.state == self.STOPPING: self.wait_for_eni_detachment() - with self.depl._db: + with self.depl._state.db: self.state = self.MISSING self._state['region'] = None self._state['attachmentId'] = None diff --git a/nixops/resources/vpc_route.py b/nixops/resources/vpc_route.py index a58c15fdd..2bb88a014 100644 --- a/nixops/resources/vpc_route.py +++ b/nixops/resources/vpc_route.py @@ -120,7 +120,7 @@ def retrieve_defn(option): self.log("creating route {0} => {1} in route table {2}".format(retrieve_defn(target), config[destination], rtb_id)) self.get_client().create_route(**route) - with self.depl._db: + with self.depl._state.db: self.state = self.UP self._state[target] = route[self.upper(target)] self._state[destination] = config[destination] @@ -144,7 +144,7 @@ def _destroy(self): else: raise error - with self.depl._db: + with self.depl._state.db: self.state = self.MISSING self._state['routeTableId'] = None self._state[destination] = None diff --git a/nixops/resources/vpc_route_table.py b/nixops/resources/vpc_route_table.py index 60dbbbe07..a0fd0c123 100644 --- a/nixops/resources/vpc_route_table.py +++ b/nixops/resources/vpc_route_table.py @@ -91,7 +91,7 @@ def realize_create_route_table(self, allow_recreate): self.log("creating route table in vpc {}".format(vpc_id)) route_table = self.get_client().create_route_table(VpcId=vpc_id) - with self.depl._db: + with self.depl._state.db: self.state = self.UP self._state['vpcId'] = vpc_id self._state['routeTableId'] = route_table['RouteTable']['RouteTableId'] @@ -122,7 +122,7 @@ def realize_propagate_vpn_gtws(self, allow_recreate): GatewayId=vgw, RouteTableId=self._state['routeTableId']) - with self.depl._db: + with self.depl._state.db: self._state['propagatingVgws'] = new_vgws def realize_update_tag(self, allow_recreate): @@ -142,7 +142,7 @@ def _destroy(self): else: raise error - with self.depl._db: + with self.depl._state.db: self.state = self.MISSING self._state['vpcId'] = None self._state['routeTableId'] = None diff --git a/nixops/resources/vpc_route_table_association.py b/nixops/resources/vpc_route_table_association.py index 8b71ea2ff..e6d38de63 100644 --- a/nixops/resources/vpc_route_table_association.py +++ b/nixops/resources/vpc_route_table_association.py @@ -91,7 +91,7 @@ def realize_associate_route_table(self, allow_recreate): association = self.get_client().associate_route_table(RouteTableId=route_table_id, SubnetId=subnet_id) - with self.depl._db: + with self.depl._state.db: self.state = self.UP self._state['routeTableId'] = route_table_id self._state['subnetId'] = subnet_id @@ -108,7 +108,7 @@ def _destroy(self): else: raise error - with self.depl._db: + with self.depl._state.db: self.state = self.MISSING self._state['routeTableId'] = None self._state['subnetId'] = None diff --git a/nixops/resources/vpc_subnet.py b/nixops/resources/vpc_subnet.py index 00a3723bd..46d5ee845 100644 --- a/nixops/resources/vpc_subnet.py +++ b/nixops/resources/vpc_subnet.py @@ -114,7 +114,7 @@ def wait_for_subnet_available(self, subnet_id): raise Exception("couldn't find subnet {}, please run deploy with --check".format(subnet_id)) self.log_end(" done") - with self.depl._db: + with self.depl._state.db: self.state = self.UP def realize_create_subnet(self, allow_recreate): @@ -143,7 +143,7 @@ def realize_create_subnet(self, allow_recreate): self.subnet_id = subnet.get('SubnetId') self.zone = subnet.get('AvailabilityZone') - with self.depl._db: + with self.depl._state.db: self.state = self.STARTING self._state['subnetId'] = self.subnet_id self._state['cidrBlock'] = config['cidrBlock'] @@ -165,7 +165,7 @@ def realize_map_public_ip_on_launch(self, allow_recreate): MapPublicIpOnLaunch={'Value':config['mapPublicIpOnLaunch']}, SubnetId=self.subnet_id) - with self.depl._db: + with self.depl._state.db: self._state['mapPublicIpOnLaunch'] = config['mapPublicIpOnLaunch'] def realize_associate_ipv6_cidr_block(self, allow_recreate): @@ -181,7 +181,7 @@ def realize_associate_ipv6_cidr_block(self, allow_recreate): self.get_client().disassociate_subnet_cidr_block( AssociationId=self._state['associationId']) - with self.depl._db: + with self.depl._state.db: self._state["ipv6CidrBlock"] = config['ipv6CidrBlock'] if config['ipv6CidrBlock'] is not None: self._state['associationId'] = response['Ipv6CidrBlockAssociation']['AssociationId'] @@ -205,7 +205,7 @@ def _destroy(self): else: raise error - with self.depl._db: + with self.depl._state.db: self.state = self.MISSING self._state['subnetID'] = None self._state['region'] = None diff --git a/nixops/state.py b/nixops/state.py index 80cb8c740..1a529540d 100644 --- a/nixops/state.py +++ b/nixops/state.py @@ -11,7 +11,7 @@ class StateDict(collections.MutableMapping): # TODO implement __repr__ for convenience e.g debuging the structure def __init__(self, depl, id): super(StateDict, self).__init__() - self._db = depl._db + self._db = depl._state.db self.id = id def __setitem__(self, key, value): From 47684d5e3262040bb43d1ae58469c38cdb236c84 Mon Sep 17 00:00:00 2001 From: Matthew O'Gorman Date: Mon, 5 Feb 2018 10:47:06 -0500 Subject: [PATCH 052/123] integrate new state wrapper with one from fork --- nixops/resources/__init__.py | 2 +- nixops/resources/aws_vpn_connection.py | 2 +- nixops/resources/aws_vpn_connection_route.py | 2 +- nixops/resources/aws_vpn_gateway.py | 2 +- nixops/resources/vpc.py | 2 +- nixops/resources/vpc_customer_gateway.py | 2 +- nixops/resources/vpc_dhcp_options.py | 2 +- nixops/resources/vpc_egress_only_internet_gateway.py | 2 +- nixops/resources/vpc_endpoint.py | 2 +- nixops/resources/vpc_internet_gateway.py | 2 +- nixops/resources/vpc_nat_gateway.py | 2 +- nixops/resources/vpc_network_acl.py | 2 +- nixops/resources/vpc_network_interface.py | 2 +- nixops/resources/vpc_network_interface_attachment.py | 2 +- nixops/resources/vpc_route.py | 2 +- nixops/resources/vpc_route_table.py | 2 +- nixops/resources/vpc_route_table_association.py | 2 +- nixops/resources/vpc_subnet.py | 2 +- nixops/{ => state}/state.py | 0 19 files changed, 18 insertions(+), 18 deletions(-) rename nixops/{ => state}/state.py (100%) diff --git a/nixops/resources/__init__.py b/nixops/resources/__init__.py index f0d787837..728f56104 100644 --- a/nixops/resources/__init__.py +++ b/nixops/resources/__init__.py @@ -3,7 +3,7 @@ import re import nixops.util -from nixops.state import StateDict +from nixops.state.state import StateDict from nixops.diff import Diff, Handler class ResourceDefinition(object): diff --git a/nixops/resources/aws_vpn_connection.py b/nixops/resources/aws_vpn_connection.py index 080b3be71..9080650a9 100644 --- a/nixops/resources/aws_vpn_connection.py +++ b/nixops/resources/aws_vpn_connection.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from nixops.state import StateDict +from nixops.state.state import StateDict from nixops.diff import Diff, Handler import nixops.util import nixops.resources diff --git a/nixops/resources/aws_vpn_connection_route.py b/nixops/resources/aws_vpn_connection_route.py index 2da1631b0..c6477c917 100644 --- a/nixops/resources/aws_vpn_connection_route.py +++ b/nixops/resources/aws_vpn_connection_route.py @@ -7,7 +7,7 @@ from nixops.resources.ec2_common import EC2CommonState import nixops.ec2_utils from nixops.diff import Diff, Handler -from nixops.state import StateDict +from nixops.state.state import StateDict class AWSVPNConnectionRouteDefinition(nixops.resources.ResourceDefinition): """Definition of a VPN connection route""" diff --git a/nixops/resources/aws_vpn_gateway.py b/nixops/resources/aws_vpn_gateway.py index d97087878..f41babee5 100644 --- a/nixops/resources/aws_vpn_gateway.py +++ b/nixops/resources/aws_vpn_gateway.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from nixops.state import StateDict +from nixops.state.state import StateDict from nixops.diff import Diff, Handler import nixops.util import nixops.resources diff --git a/nixops/resources/vpc.py b/nixops/resources/vpc.py index 28d1344aa..6cb3c5d75 100755 --- a/nixops/resources/vpc.py +++ b/nixops/resources/vpc.py @@ -9,7 +9,7 @@ import nixops.resources from nixops.resources.ec2_common import EC2CommonState import nixops.ec2_utils -from nixops.state import StateDict +from nixops.state.state import StateDict from nixops.diff import Diff, Handler class VPCDefinition(nixops.resources.ResourceDefinition): diff --git a/nixops/resources/vpc_customer_gateway.py b/nixops/resources/vpc_customer_gateway.py index 129d7d846..9bc324058 100644 --- a/nixops/resources/vpc_customer_gateway.py +++ b/nixops/resources/vpc_customer_gateway.py @@ -7,7 +7,7 @@ import boto3 import botocore -from nixops.state import StateDict +from nixops.state.state import StateDict from nixops.diff import Diff, Handler import nixops.util import nixops.resources diff --git a/nixops/resources/vpc_dhcp_options.py b/nixops/resources/vpc_dhcp_options.py index 114692687..43000e8ad 100644 --- a/nixops/resources/vpc_dhcp_options.py +++ b/nixops/resources/vpc_dhcp_options.py @@ -12,7 +12,7 @@ import nixops.resources from nixops.resources.ec2_common import EC2CommonState import nixops.ec2_utils -from nixops.state import StateDict +from nixops.state.state import StateDict from nixops.diff import Diff, Handler class VPCDhcpOptionsDefinition(nixops.resources.ResourceDefinition): diff --git a/nixops/resources/vpc_egress_only_internet_gateway.py b/nixops/resources/vpc_egress_only_internet_gateway.py index d76791f60..7abcd5dfa 100644 --- a/nixops/resources/vpc_egress_only_internet_gateway.py +++ b/nixops/resources/vpc_egress_only_internet_gateway.py @@ -3,7 +3,7 @@ import boto3 import botocore -from nixops.state import StateDict +from nixops.state.state import StateDict from nixops.diff import Diff, Handler import nixops.util import nixops.resources diff --git a/nixops/resources/vpc_endpoint.py b/nixops/resources/vpc_endpoint.py index 55671dbb6..dd2070da4 100644 --- a/nixops/resources/vpc_endpoint.py +++ b/nixops/resources/vpc_endpoint.py @@ -2,7 +2,7 @@ import uuid -from nixops.state import StateDict +from nixops.state.state import StateDict from nixops.diff import Diff, Handler import nixops.util import nixops.resources diff --git a/nixops/resources/vpc_internet_gateway.py b/nixops/resources/vpc_internet_gateway.py index 4a3045b00..e260af14d 100644 --- a/nixops/resources/vpc_internet_gateway.py +++ b/nixops/resources/vpc_internet_gateway.py @@ -7,7 +7,7 @@ import boto3 import botocore -from nixops.state import StateDict +from nixops.state.state import StateDict from nixops.diff import Diff, Handler import nixops.util import nixops.resources diff --git a/nixops/resources/vpc_nat_gateway.py b/nixops/resources/vpc_nat_gateway.py index 7295e4e1d..c32c41dea 100644 --- a/nixops/resources/vpc_nat_gateway.py +++ b/nixops/resources/vpc_nat_gateway.py @@ -13,7 +13,7 @@ from nixops.resources.ec2_common import EC2CommonState import nixops.ec2_utils from nixops.diff import Diff, Handler -from nixops.state import StateDict +from nixops.state.state import StateDict class VPCNatGatewayDefinition(nixops.resources.ResourceDefinition): """Definition of a VPC NAT gateway""" diff --git a/nixops/resources/vpc_network_acl.py b/nixops/resources/vpc_network_acl.py index 6a5cf1b72..39ce50328 100644 --- a/nixops/resources/vpc_network_acl.py +++ b/nixops/resources/vpc_network_acl.py @@ -9,7 +9,7 @@ from nixops.resources.ec2_common import EC2CommonState import nixops.ec2_utils from nixops.diff import Diff, Handler -from nixops.state import StateDict +from nixops.state.state import StateDict class VPCNetworkAcldefinition(nixops.resources.ResourceDefinition): """definition of a vpc network ACL.""" diff --git a/nixops/resources/vpc_network_interface.py b/nixops/resources/vpc_network_interface.py index 8797e4a9b..84bf70a3a 100644 --- a/nixops/resources/vpc_network_interface.py +++ b/nixops/resources/vpc_network_interface.py @@ -10,7 +10,7 @@ from nixops.resources.ec2_common import EC2CommonState import nixops.ec2_utils from nixops.diff import Diff, Handler -from nixops.state import StateDict +from nixops.state.state import StateDict class VPCNetworkInterfaceDefinition(nixops.resources.ResourceDefinition): """Definition of a VPC network interface""" diff --git a/nixops/resources/vpc_network_interface_attachment.py b/nixops/resources/vpc_network_interface_attachment.py index cb22b924f..3ff610e76 100644 --- a/nixops/resources/vpc_network_interface_attachment.py +++ b/nixops/resources/vpc_network_interface_attachment.py @@ -12,7 +12,7 @@ from nixops.resources.ec2_common import EC2CommonState import nixops.ec2_utils from nixops.diff import Diff, Handler -from nixops.state import StateDict +from nixops.state.state import StateDict class VPCNetworkInterfaceAttachmentDefinition(nixops.resources.ResourceDefinition): """Definition of a VPC network interface attachment""" diff --git a/nixops/resources/vpc_route.py b/nixops/resources/vpc_route.py index 2bb88a014..3e7e686ba 100644 --- a/nixops/resources/vpc_route.py +++ b/nixops/resources/vpc_route.py @@ -10,7 +10,7 @@ from nixops.resources.ec2_common import EC2CommonState import nixops.ec2_utils from nixops.diff import Diff, Handler -from nixops.state import StateDict +from nixops.state.state import StateDict class VPCRouteDefinition(nixops.resources.ResourceDefinition): """Definition of a VPC route""" diff --git a/nixops/resources/vpc_route_table.py b/nixops/resources/vpc_route_table.py index a0fd0c123..504e0cc03 100644 --- a/nixops/resources/vpc_route_table.py +++ b/nixops/resources/vpc_route_table.py @@ -10,7 +10,7 @@ from nixops.resources.ec2_common import EC2CommonState import nixops.ec2_utils from nixops.diff import Diff, Handler -from nixops.state import StateDict +from nixops.state.state import StateDict class VPCRouteTableDefinition(nixops.resources.ResourceDefinition): """Definition of a VPC route table""" diff --git a/nixops/resources/vpc_route_table_association.py b/nixops/resources/vpc_route_table_association.py index e6d38de63..b1cc3d6c7 100644 --- a/nixops/resources/vpc_route_table_association.py +++ b/nixops/resources/vpc_route_table_association.py @@ -10,7 +10,7 @@ from nixops.resources.ec2_common import EC2CommonState import nixops.ec2_utils from nixops.diff import Diff, Handler -from nixops.state import StateDict +from nixops.state.state import StateDict class VPCRouteTableAssociationDefinition(nixops.resources.ResourceDefinition): """Definition of a VPC route table association""" diff --git a/nixops/resources/vpc_subnet.py b/nixops/resources/vpc_subnet.py index 46d5ee845..4031a6230 100644 --- a/nixops/resources/vpc_subnet.py +++ b/nixops/resources/vpc_subnet.py @@ -10,7 +10,7 @@ from nixops.resources.ec2_common import EC2CommonState import nixops.ec2_utils from nixops.diff import Diff, Handler -from nixops.state import StateDict +from nixops.state.state import StateDict class VPCSubnetDefinition(nixops.resources.ResourceDefinition): """Definition of a VPC subnet.""" diff --git a/nixops/state.py b/nixops/state/state.py similarity index 100% rename from nixops/state.py rename to nixops/state/state.py From b0f18b54fe155b7cd7645e7a9de7d6108d9f1aa7 Mon Sep 17 00:00:00 2001 From: Matthew O'Gorman Date: Tue, 6 Feb 2018 22:22:23 -0500 Subject: [PATCH 053/123] rewrite this to use the wrapper --- nixops/resources/vpc_subnet.py | 1 - nixops/state/state.py | 45 +++++++++------------------------- 2 files changed, 12 insertions(+), 34 deletions(-) diff --git a/nixops/resources/vpc_subnet.py b/nixops/resources/vpc_subnet.py index 4031a6230..016cab8fa 100644 --- a/nixops/resources/vpc_subnet.py +++ b/nixops/resources/vpc_subnet.py @@ -10,7 +10,6 @@ from nixops.resources.ec2_common import EC2CommonState import nixops.ec2_utils from nixops.diff import Diff, Handler -from nixops.state.state import StateDict class VPCSubnetDefinition(nixops.resources.ResourceDefinition): """Definition of a VPC subnet.""" diff --git a/nixops/state/state.py b/nixops/state/state.py index 1a529540d..e124806fd 100644 --- a/nixops/state/state.py +++ b/nixops/state/state.py @@ -11,50 +11,29 @@ class StateDict(collections.MutableMapping): # TODO implement __repr__ for convenience e.g debuging the structure def __init__(self, depl, id): super(StateDict, self).__init__() - self._db = depl._state.db + self._state = depl._state + self.uuid = depl.uuid self.id = id def __setitem__(self, key, value): - with self._db: - c = self._db.cursor() - if value == None: - c.execute("delete from ResourceAttrs where machine = ? and name = ?", - (self.id, key)) - else: - v = value - if isinstance(value, list): - v = json.dumps(value) - c.execute("insert or replace into ResourceAttrs(machine, name, value) values (?, ?, ?)", - (self.id, key, v)) + self._state.set_resource_attrs(self.uuid, self.id, {key:value}) def __getitem__(self, key): - with self._db: - c = self._db.cursor() - c.execute("select value from ResourceAttrs where machine = ? and name = ?", - (self.id, key)) - row = c.fetchone() - if row != None: - try: - return json.loads(row[0]) - except ValueError: - return row[0] - raise KeyError("couldn't find key {} in the state file".format(key)) + value = self._state.get_resource_attr(self.uuid, self.id, name) + try: + return json.loads(value) + except ValueError: + return value + raise KeyError("couldn't find key {} in the state file".format(key)) def __delitem__(self, key): - with self._db: - c.execute("delete from ResourceAttrs where machine = ? and name = ?", (self.id, key)) + self._state.del_resource_attr(self.uuid, self.id, key) def keys(self): # Generally the list of keys per ResourceAttrs is relatively small # so this should be also relatively fast. - _keys = [] - with self._db: - c = self._db.cursor() - c.execute("select name from ResourceAttrs where machine = ?", (self.id,)) - rows = c.fetchall() - for row in rows: - _keys.append(row[0]) - return _keys + attrs = self._state.get_all_resource_attrs(self.uuid, self.id) + return [key for key,value in attrs.iteritems()] def __iter__(self): return iter(self.keys()) From b3690b0e254958966aa5d60cd4d03fa8b5040cc5 Mon Sep 17 00:00:00 2001 From: Matthew O'Gorman Date: Tue, 6 Feb 2018 23:50:02 -0500 Subject: [PATCH 054/123] initial commit of moving over to sqlalchemy for all sql endpoints --- nixops/state/__init__.py | 6 +- nixops/state/mysql.py | 343 +++++++++++++++++++++++++++++++++++++++ release.nix | 1 + 3 files changed, 349 insertions(+), 1 deletion(-) create mode 100644 nixops/state/mysql.py diff --git a/nixops/state/__init__.py b/nixops/state/__init__.py index 214c19009..68a788745 100644 --- a/nixops/state/__init__.py +++ b/nixops/state/__init__.py @@ -2,6 +2,7 @@ import sys import sqlite3_file import json_file +import mysql class WrongStateSchemeException(Exception): pass @@ -9,7 +10,9 @@ class WrongStateSchemeException(Exception): def open(url): url = urlparse.urlparse(url) scheme = url.scheme - + print 'my name is mog' + print url.scheme + print url.path if scheme == "": scheme = "sqlite3" @@ -19,6 +22,7 @@ def raise_(ex): switcher = { "sqlite3": lambda(url): sqlite3_file.StateFile(url.path), "json": lambda(url): json_file.JsonFile(url.path), + "mysql": lambda(url): mysql.SQLConnection(url), } function = switcher.get(scheme, lambda(url): raise_(WrongStateSchemeException("Unknown state scheme!"))) diff --git a/nixops/state/mysql.py b/nixops/state/mysql.py new file mode 100644 index 000000000..1cde466b2 --- /dev/null +++ b/nixops/state/mysql.py @@ -0,0 +1,343 @@ +# -*- coding: utf-8 -*- + +import nixops.deployment +import os +import os.path +from pysqlite2 import dbapi2 as sqlite3 +import sys +import threading +import fcntl + +import sqlalchemy +from sqlalchemy.engine import Engine +from sqlalchemy import event + +def _subclasses(cls): + sub = cls.__subclasses__() + return [cls] if not sub else [g for s in sub for g in _subclasses(s)] + +def get_default_state_file(): + home = os.environ.get("HOME", "") + "/.nixops" + if not os.path.exists(home): + old_home = os.environ.get("HOME", "") + "/.charon" + if os.path.exists(old_home): + sys.stderr.write("renaming ‘{0}’ to ‘{1}’...\n".format(old_home, home)) + os.rename(old_home, home) + if os.path.exists(home + "/deployments.charon"): + os.rename(home + "/deployments.charon", home + "/deployments.nixops") + else: + os.makedirs(home, 0700) + return os.environ.get("NIXOPS_STATE", os.environ.get("CHARON_STATE", home + "/deployments.nixops")) + + +class SQLConnection(object): + """NixOps state file.""" + + current_schema = 3 + + @event.listens_for(Engine, "connect") + def set_sqlite_pragma(dbapi_connection, connection_record): + cursor = dbapi_connection.cursor() + db.execute("pragma journal_mode = wal") + db.execute("pragma foreign_keys = 1") + cursor.close() + + def __init__(self, db_uri): + self.db_uri = db_uri + + db_engine = sqlalchemy.create_engine(db_uri) + db = db_engine.connect() + # FIXME: this is not actually transactional, because pysqlite (not + # sqlite) does an implicit commit before "create table". + with db: + + c = db.cursor() + + # Get the schema version. + version = 0 # new database + if self._table_exists(c, 'SchemaVersion'): + c.execute("select version from SchemaVersion") + version = c.fetchone()[0] + elif self._table_exists(c, 'Deployments'): + version = 1 + + if version == self.current_schema: + pass + elif version == 0: + self._create_schema(c) + elif version < self.current_schema: + if version <= 1: self._upgrade_1_to_2(c) + if version <= 2: self._upgrade_2_to_3(c) + c.execute("update SchemaVersion set version = ?", (self.current_schema,)) + else: + raise Exception("this NixOps version is too old to deal with schema version {0}".format(version)) + + self.db = db + + + def close(self): + self.db.close() + + ############################################################################################### + ## Deployment + + def query_deployments(self): + """Return the UUIDs of all deployments in the database.""" + c = self.db.cursor() + c.execute("select uuid from Deployments") + res = c.fetchall() + return [x[0] for x in res] + + def get_all_deployments(self):r + """Return Deployment objects for every deployment in the database.""" + uuids = self.query_deployments() + res = [] + for uuid in uuids: + try: + res.append(self.open_deployment(uuid=uuid)) + except nixops.deployment.UnknownBackend as e: + sys.stderr.write("skipping deployment ‘{0}’: {1}\n".format(uuid, str(e))) + return res + + def _find_deployment(self, uuid=None): + c = self.db.cursor() + if not uuid: + c.execute("select uuid from Deployments") + else: + c.execute("select uuid from Deployments d where uuid = ? or exists (select 1 from DeploymentAttrs where deployment = d.uuid and name = 'name' and value = ?)", (uuid, uuid)) + res = c.fetchall() + if len(res) == 0: + if uuid: + # try the prefix match + c.execute("select uuid from Deployments where uuid glob ?", (uuid + '*', )) + res = c.fetchall() + if len(res) == 0: + return None + else: + return None + if len(res) > 1: + if uuid: + raise Exception("state file contains multiple deployments with the same name, so you should specify one using its UUID") + else: + raise Exception("state file contains multiple deployments, so you should specify which one to use using ‘-d’, or set the environment variable NIXOPS_DEPLOYMENT") + return nixops.deployment.Deployment(self, res[0][0], sys.stderr) + + def open_deployment(self, uuid=None): + """Open an existing deployment.""" + deployment = self._find_deployment(uuid=uuid) + if deployment: return deployment + raise Exception("could not find specified deployment in state file ‘{0}’".format(self.db_file)) + + def create_deployment(self, uuid=None): + """Create a new deployment.""" + if not uuid: + import uuid + uuid = str(uuid.uuid1()) + with self.db: + self.db.execute("insert into Deployments(uuid) values (?)", (uuid,)) + return nixops.deployment.Deployment(self, uuid, sys.stderr) + + def _delete_deployment(self, deployment_uuid): + """NOTE: This is UNSAFE, it's guarded in nixops/deployment.py. Do not call this function except from there!""" + self.db.execute("delete from Deployments where uuid = ?", (deployment_uuid,)) + + def clone_deployment(self, deployment_uuid): + with self.db: + new = self.create_deployment() + self.db.execute("insert into DeploymentAttrs (deployment, name, value) " + + "select ?, name, value from DeploymentAttrs where deployment = ?", + (new.uuid, deployment_uuid)) + new.configs_path = None + return new + + + + def get_resources_for(self, deployment): + """Get all the resources for a certain deployment""" + resources = {} + with self.db: + c = self.db.cursor() + c.execute("select id, name, type from Resources where deployment = ?", (deployment.uuid,)) + for (id, name, type) in c.fetchall(): + r = self._create_state(deployment, type, name, id) + resources[name] = r + return resources + + def set_deployment_attrs(self, deployment_uuid, attrs): + """Update deployment attributes in the state.""" + with self.db: + c = self.db.cursor() + for n, v in attrs.iteritems(): + if v == None: + c.execute("delete from DeploymentAttrs where deployment = ? and name = ?", (deployment_uuid, n)) + else: + c.execute("insert or replace into DeploymentAttrs(deployment, name, value) values (?, ?, ?)", + (deployment_uuid, n, v)) + + def del_deployment_attr(self, deployment_uuid, attr_name): + with self.db: + self.db.execute("delete from DeploymentAttrs where deployment = ? and name = ?", (deployment_uuid, attr_name)) + + + def get_deployment_attr(self, deployment_uuid, name): + """Get a deployment attribute from the state.""" + with self.db: + c = self.db.cursor() + c.execute("select value from DeploymentAttrs where deployment = ? and name = ?", (deployment_uuid, name)) + row = c.fetchone() + if row != None: return row[0] + return nixops.util.undefined + + def get_all_deployment_attrs(self, deployment_uuid): + with self.db: + c = self.db.cursor() + c.execute("select name, value from DeploymentAttrs where deployment = ?", (deployment_uuid)) + rows = c.fetchall() + res = {row[0]: row[1] for row in rows} + return res + + def get_deployment_lock(self, deployment): + lock_dir = os.environ.get("HOME", "") + "/.nixops/locks" + if not os.path.exists(lock_dir): os.makedirs(lock_dir, 0700) + lock_file_path = lock_dir + "/" + deployment.uuid + class DeploymentLock(object): + def __init__(self, logger, path): + self._lock_file_path = path + self._logger = logger + self._lock_file = None + def __enter__(self): + self._lock_file = open(self._lock_file_path, "w") + fcntl.fcntl(self._lock_file, fcntl.F_SETFD, fcntl.FD_CLOEXEC) + try: + fcntl.flock(self._lock_file, fcntl.LOCK_EX | fcntl.LOCK_NB) + except IOError: + self._logger.log( + "waiting for exclusive deployment lock..." + ) + fcntl.flock(self._lock_file, fcntl.LOCK_EX) + def __exit__(self, exception_type, exception_value, exception_traceback): + if self._lock_file: + self._lock_file.close() + return DeploymentLock(deployment.logger, lock_file_path) + + ############################################################################################### + ## Resources + + def create_resource(self, deployment, name, type): + c = self.db.cursor() + c.execute("select 1 from Resources where deployment = ? and name = ?", (deployment.uuid, name)) + if len(c.fetchall()) != 0: + raise Exception("resource already exists in database!") + c.execute("insert into Resources(deployment, name, type) values (?, ?, ?)", + (deployment.uuid, name, type)) + id = c.lastrowid + r = self._create_state(deployment, type, name, id) + return r + + def delete_resource(self, deployment_uuid, res_id): + with self.db: + self.db.execute("delete from Resources where deployment = ? and id = ?", (deployment_uuid, res_id)) + + def _rename_resource(self, deployment_uuid, resource_id, new_name): + """NOTE: Invariants are checked in nixops/deployment.py#rename""" + with self.db: + self.db.execute("update Resources set name = ? where deployment = ? and id = ?", (new_name, deployment_uuid, resource_id)) + + + def set_resource_attrs(self, _deployment_uuid, resource_id, attrs): + with self.db: + c = self.db.cursor() + for n, v in attrs.iteritems(): + if v == None: + c.execute("delete from ResourceAttrs where machine = ? and name = ?", (resource_id, n)) + else: + c.execute("insert or replace into ResourceAttrs(machine, name, value) values (?, ?, ?)", + (resource_id, n, v)) + + def del_resource_attr(self, _deployment_uuid, resource_id, name): + with self.db: + self.db.execute("delete from ResourceAttrs where machine = ? and name = ?", (resource_id, name)) + + def get_resource_attr(self, _deployment_uuid, resource_id, name): + """Get a machine attribute from the state file.""" + with self.db: + c = self.db.cursor() + c.execute("select value from ResourceAttrs where machine = ? and name = ?", (resource_id, name)) + row = c.fetchone() + if row != None: return row[0] + return nixops.util.undefined + + def get_all_resource_attrs(self, deployment_uuid, resource_id): + with self.db: + c = self.db.cursor() + c.execute("select name, value from ResourceAttrs where machine = ?", (resource_id,)) + rows = c.fetchall() + res = {row[0]: row[1] for row in rows} + return res + + ### STATE + def _create_state(self, depl, type, name, id): + """Create a resource state object of the desired type.""" + + for cls in _subclasses(nixops.resources.ResourceState): + if type == cls.get_type(): + return cls(depl, name, id) + + raise nixops.deployment.UnknownBackend("unknown resource type ‘{0}’".format(type)) + + + def _table_exists(self, c, table): + c.execute("select 1 from sqlite_master where name = ? and type='table'", (table,)); + return c.fetchone() != None + + def _create_schemaversion(self, c): + c.execute( + '''create table if not exists SchemaVersion( + version integer not null + );''') + + c.execute("insert into SchemaVersion(version) values (?)", (self.current_schema,)) + + def _create_schema(self, c): + self._create_schemaversion(c) + + c.execute( + '''create table if not exists Deployments( + uuid text primary key + );''') + + c.execute( + '''create table if not exists DeploymentAttrs( + deployment text not null, + name text not null, + value text not null, + primary key(deployment, name), + foreign key(deployment) references Deployments(uuid) on delete cascade + );''') + + c.execute( + '''create table if not exists Resources( + id integer primary key autoincrement, + deployment text not null, + name text not null, + type text not null, + foreign key(deployment) references Deployments(uuid) on delete cascade + );''') + + c.execute( + '''create table if not exists ResourceAttrs( + machine integer not null, + name text not null, + value text not null, + primary key(machine, name), + foreign key(machine) references Resources(id) on delete cascade + );''') + + def _upgrade_1_to_2(self, c): + sys.stderr.write("updating database schema from version 1 to 2...\n") + self._create_schemaversion(c) + + def _upgrade_2_to_3(self, c): + sys.stderr.write("updating database schema from version 2 to 3...\n") + c.execute("alter table Machines rename to Resources") + c.execute("alter table MachineAttrs rename to ResourceAttrs") diff --git a/release.nix b/release.nix index 0ae5deec6..7c3ac22d2 100644 --- a/release.nix +++ b/release.nix @@ -92,6 +92,7 @@ rec { azure-mgmt-storage adal # Go back to sqlite once Python 2.7.13 is released + sqlalchemy pysqlite datadog digital-ocean From 0b89b58a470c24b0c35295a94f0dcee8cbd932dd Mon Sep 17 00:00:00 2001 From: Matthew O'Gorman Date: Tue, 6 Feb 2018 23:53:33 -0500 Subject: [PATCH 055/123] typo letter --- nixops/state/mysql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nixops/state/mysql.py b/nixops/state/mysql.py index 1cde466b2..117a88c7f 100644 --- a/nixops/state/mysql.py +++ b/nixops/state/mysql.py @@ -88,7 +88,7 @@ def query_deployments(self): res = c.fetchall() return [x[0] for x in res] - def get_all_deployments(self):r + def get_all_deployments(self): """Return Deployment objects for every deployment in the database.""" uuids = self.query_deployments() res = [] From 1ab315691f7bc58e2e028bd94060fcaecd539b1a Mon Sep 17 00:00:00 2001 From: Matthew O'Gorman Date: Wed, 7 Feb 2018 00:25:25 -0500 Subject: [PATCH 056/123] working switcher code. now rewriting sql code to use sqlalchemy --- nixops/state/__init__.py | 18 +- nixops/state/{mysql.py => sql_connector.py} | 1 + nixops/state/sqlite3_file.py | 377 -------------------- scripts/nixops | 4 +- 4 files changed, 13 insertions(+), 387 deletions(-) rename nixops/state/{mysql.py => sql_connector.py} (99%) delete mode 100644 nixops/state/sqlite3_file.py diff --git a/nixops/state/__init__.py b/nixops/state/__init__.py index 68a788745..e29fa7dce 100644 --- a/nixops/state/__init__.py +++ b/nixops/state/__init__.py @@ -1,8 +1,8 @@ +import os import urlparse import sys -import sqlite3_file import json_file -import mysql +import sql_connector class WrongStateSchemeException(Exception): pass @@ -10,19 +10,21 @@ class WrongStateSchemeException(Exception): def open(url): url = urlparse.urlparse(url) scheme = url.scheme - print 'my name is mog' - print url.scheme - print url.path + ext = os.path.splitext(url.path)[1] + if scheme == "": - scheme = "sqlite3" + if ext == ".nixops": + scheme = "sql" + url = 'sqlite:///' + url.path + elif ext == ".json": + scheme = "json" def raise_(ex): raise ex switcher = { - "sqlite3": lambda(url): sqlite3_file.StateFile(url.path), "json": lambda(url): json_file.JsonFile(url.path), - "mysql": lambda(url): mysql.SQLConnection(url), + "sql": lambda(url): sql_connector.SQLConnection(url), } function = switcher.get(scheme, lambda(url): raise_(WrongStateSchemeException("Unknown state scheme!"))) diff --git a/nixops/state/mysql.py b/nixops/state/sql_connector.py similarity index 99% rename from nixops/state/mysql.py rename to nixops/state/sql_connector.py index 117a88c7f..332158791 100644 --- a/nixops/state/mysql.py +++ b/nixops/state/sql_connector.py @@ -43,6 +43,7 @@ def set_sqlite_pragma(dbapi_connection, connection_record): cursor.close() def __init__(self, db_uri): + print db_uri self.db_uri = db_uri db_engine = sqlalchemy.create_engine(db_uri) diff --git a/nixops/state/sqlite3_file.py b/nixops/state/sqlite3_file.py deleted file mode 100644 index 592874f9a..000000000 --- a/nixops/state/sqlite3_file.py +++ /dev/null @@ -1,377 +0,0 @@ -# -*- coding: utf-8 -*- - -import nixops.deployment -import os -import os.path -from pysqlite2 import dbapi2 as sqlite3 -import sys -import threading -import fcntl - -def _subclasses(cls): - sub = cls.__subclasses__() - return [cls] if not sub else [g for s in sub for g in _subclasses(s)] - -class Connection(sqlite3.Connection): - - def __init__(self, db_file, **kwargs): - db_exists = os.path.exists(db_file) - if not db_exists: - os.fdopen(os.open(db_file, os.O_WRONLY | os.O_CREAT, 0o600), 'w').close() - sqlite3.Connection.__init__(self, db_file, **kwargs) - self.db_file = db_file - self.nesting = 0 - self.lock = threading.RLock() - - # Implement Python's context management protocol so that "with db" - # automatically commits or rolls back. The difference with the - # parent's "with" implementation is that we nest, i.e. a commit or - # rollback is only done at the outer "with". - def __enter__(self): - self.lock.acquire() - if self.nesting == 0: - self.must_rollback = False - self.nesting = self.nesting + 1 - sqlite3.Connection.__enter__(self) - - - def __exit__(self, exception_type, exception_value, exception_traceback): - if exception_type != None: self.must_rollback = True - self.nesting = self.nesting - 1 - assert self.nesting >= 0 - if self.nesting == 0: - if self.must_rollback: - try: - self.rollback() - except sqlite3.ProgrammingError: - pass - else: - sqlite3.Connection.__exit__(self, exception_type, exception_value, exception_traceback) - self.lock.release() - - -def get_default_state_file(): - home = os.environ.get("HOME", "") + "/.nixops" - if not os.path.exists(home): - old_home = os.environ.get("HOME", "") + "/.charon" - if os.path.exists(old_home): - sys.stderr.write("renaming ‘{0}’ to ‘{1}’...\n".format(old_home, home)) - os.rename(old_home, home) - if os.path.exists(home + "/deployments.charon"): - os.rename(home + "/deployments.charon", home + "/deployments.nixops") - else: - os.makedirs(home, 0700) - return os.environ.get("NIXOPS_STATE", os.environ.get("CHARON_STATE", home + "/deployments.nixops")) - - -class StateFile(object): - """NixOps state file.""" - - current_schema = 3 - - def __init__(self, db_file): - self.db_file = db_file - - if os.path.splitext(db_file)[1] not in ['.nixops', '.charon']: - raise Exception("state file ‘{0}’ should have extension ‘.nixops’".format(db_file)) - db = sqlite3.connect(db_file, timeout=60, check_same_thread=False, factory=Connection, isolation_level=None) # FIXME - db.db_file = db_file - - db.execute("pragma journal_mode = wal") - db.execute("pragma foreign_keys = 1") - - # FIXME: this is not actually transactional, because pysqlite (not - # sqlite) does an implicit commit before "create table". - with db: - c = db.cursor() - - # Get the schema version. - version = 0 # new database - if self._table_exists(c, 'SchemaVersion'): - c.execute("select version from SchemaVersion") - version = c.fetchone()[0] - elif self._table_exists(c, 'Deployments'): - version = 1 - - if version == self.current_schema: - pass - elif version == 0: - self._create_schema(c) - elif version < self.current_schema: - if version <= 1: self._upgrade_1_to_2(c) - if version <= 2: self._upgrade_2_to_3(c) - c.execute("update SchemaVersion set version = ?", (self.current_schema,)) - else: - raise Exception("this NixOps version is too old to deal with schema version {0}".format(version)) - - self.db = db - - - def close(self): - self.db.close() - - ############################################################################################### - ## Deployment - - def query_deployments(self): - """Return the UUIDs of all deployments in the database.""" - c = self.db.cursor() - c.execute("select uuid from Deployments") - res = c.fetchall() - return [x[0] for x in res] - - def get_all_deployments(self): - """Return Deployment objects for every deployment in the database.""" - uuids = self.query_deployments() - res = [] - for uuid in uuids: - try: - res.append(self.open_deployment(uuid=uuid)) - except nixops.deployment.UnknownBackend as e: - sys.stderr.write("skipping deployment ‘{0}’: {1}\n".format(uuid, str(e))) - return res - - def _find_deployment(self, uuid=None): - c = self.db.cursor() - if not uuid: - c.execute("select uuid from Deployments") - else: - c.execute("select uuid from Deployments d where uuid = ? or exists (select 1 from DeploymentAttrs where deployment = d.uuid and name = 'name' and value = ?)", (uuid, uuid)) - res = c.fetchall() - if len(res) == 0: - if uuid: - # try the prefix match - c.execute("select uuid from Deployments where uuid glob ?", (uuid + '*', )) - res = c.fetchall() - if len(res) == 0: - return None - else: - return None - if len(res) > 1: - if uuid: - raise Exception("state file contains multiple deployments with the same name, so you should specify one using its UUID") - else: - raise Exception("state file contains multiple deployments, so you should specify which one to use using ‘-d’, or set the environment variable NIXOPS_DEPLOYMENT") - return nixops.deployment.Deployment(self, res[0][0], sys.stderr) - - def open_deployment(self, uuid=None): - """Open an existing deployment.""" - deployment = self._find_deployment(uuid=uuid) - if deployment: return deployment - raise Exception("could not find specified deployment in state file ‘{0}’".format(self.db_file)) - - def create_deployment(self, uuid=None): - """Create a new deployment.""" - if not uuid: - import uuid - uuid = str(uuid.uuid1()) - with self.db: - self.db.execute("insert into Deployments(uuid) values (?)", (uuid,)) - return nixops.deployment.Deployment(self, uuid, sys.stderr) - - def _delete_deployment(self, deployment_uuid): - """NOTE: This is UNSAFE, it's guarded in nixops/deployment.py. Do not call this function except from there!""" - self.db.execute("delete from Deployments where uuid = ?", (deployment_uuid,)) - - def clone_deployment(self, deployment_uuid): - with self.db: - new = self.create_deployment() - self.db.execute("insert into DeploymentAttrs (deployment, name, value) " + - "select ?, name, value from DeploymentAttrs where deployment = ?", - (new.uuid, deployment_uuid)) - new.configs_path = None - return new - - - - def get_resources_for(self, deployment): - """Get all the resources for a certain deployment""" - resources = {} - with self.db: - c = self.db.cursor() - c.execute("select id, name, type from Resources where deployment = ?", (deployment.uuid,)) - for (id, name, type) in c.fetchall(): - r = self._create_state(deployment, type, name, id) - resources[name] = r - return resources - - def set_deployment_attrs(self, deployment_uuid, attrs): - """Update deployment attributes in the state.""" - with self.db: - c = self.db.cursor() - for n, v in attrs.iteritems(): - if v == None: - c.execute("delete from DeploymentAttrs where deployment = ? and name = ?", (deployment_uuid, n)) - else: - c.execute("insert or replace into DeploymentAttrs(deployment, name, value) values (?, ?, ?)", - (deployment_uuid, n, v)) - - def del_deployment_attr(self, deployment_uuid, attr_name): - with self.db: - self.db.execute("delete from DeploymentAttrs where deployment = ? and name = ?", (deployment_uuid, attr_name)) - - - def get_deployment_attr(self, deployment_uuid, name): - """Get a deployment attribute from the state.""" - with self.db: - c = self.db.cursor() - c.execute("select value from DeploymentAttrs where deployment = ? and name = ?", (deployment_uuid, name)) - row = c.fetchone() - if row != None: return row[0] - return nixops.util.undefined - - def get_all_deployment_attrs(self, deployment_uuid): - with self.db: - c = self.db.cursor() - c.execute("select name, value from DeploymentAttrs where deployment = ?", (deployment_uuid)) - rows = c.fetchall() - res = {row[0]: row[1] for row in rows} - return res - - def get_deployment_lock(self, deployment): - lock_dir = os.environ.get("HOME", "") + "/.nixops/locks" - if not os.path.exists(lock_dir): os.makedirs(lock_dir, 0700) - lock_file_path = lock_dir + "/" + deployment.uuid - class DeploymentLock(object): - def __init__(self, logger, path): - self._lock_file_path = path - self._logger = logger - self._lock_file = None - def __enter__(self): - self._lock_file = open(self._lock_file_path, "w") - fcntl.fcntl(self._lock_file, fcntl.F_SETFD, fcntl.FD_CLOEXEC) - try: - fcntl.flock(self._lock_file, fcntl.LOCK_EX | fcntl.LOCK_NB) - except IOError: - self._logger.log( - "waiting for exclusive deployment lock..." - ) - fcntl.flock(self._lock_file, fcntl.LOCK_EX) - def __exit__(self, exception_type, exception_value, exception_traceback): - if self._lock_file: - self._lock_file.close() - return DeploymentLock(deployment.logger, lock_file_path) - - ############################################################################################### - ## Resources - - def create_resource(self, deployment, name, type): - c = self.db.cursor() - c.execute("select 1 from Resources where deployment = ? and name = ?", (deployment.uuid, name)) - if len(c.fetchall()) != 0: - raise Exception("resource already exists in database!") - c.execute("insert into Resources(deployment, name, type) values (?, ?, ?)", - (deployment.uuid, name, type)) - id = c.lastrowid - r = self._create_state(deployment, type, name, id) - return r - - def delete_resource(self, deployment_uuid, res_id): - with self.db: - self.db.execute("delete from Resources where deployment = ? and id = ?", (deployment_uuid, res_id)) - - def _rename_resource(self, deployment_uuid, resource_id, new_name): - """NOTE: Invariants are checked in nixops/deployment.py#rename""" - with self.db: - self.db.execute("update Resources set name = ? where deployment = ? and id = ?", (new_name, deployment_uuid, resource_id)) - - - def set_resource_attrs(self, _deployment_uuid, resource_id, attrs): - with self.db: - c = self.db.cursor() - for n, v in attrs.iteritems(): - if v == None: - c.execute("delete from ResourceAttrs where machine = ? and name = ?", (resource_id, n)) - else: - c.execute("insert or replace into ResourceAttrs(machine, name, value) values (?, ?, ?)", - (resource_id, n, v)) - - def del_resource_attr(self, _deployment_uuid, resource_id, name): - with self.db: - self.db.execute("delete from ResourceAttrs where machine = ? and name = ?", (resource_id, name)) - - def get_resource_attr(self, _deployment_uuid, resource_id, name): - """Get a machine attribute from the state file.""" - with self.db: - c = self.db.cursor() - c.execute("select value from ResourceAttrs where machine = ? and name = ?", (resource_id, name)) - row = c.fetchone() - if row != None: return row[0] - return nixops.util.undefined - - def get_all_resource_attrs(self, deployment_uuid, resource_id): - with self.db: - c = self.db.cursor() - c.execute("select name, value from ResourceAttrs where machine = ?", (resource_id,)) - rows = c.fetchall() - res = {row[0]: row[1] for row in rows} - return res - - ### STATE - def _create_state(self, depl, type, name, id): - """Create a resource state object of the desired type.""" - - for cls in _subclasses(nixops.resources.ResourceState): - if type == cls.get_type(): - return cls(depl, name, id) - - raise nixops.deployment.UnknownBackend("unknown resource type ‘{0}’".format(type)) - - - def _table_exists(self, c, table): - c.execute("select 1 from sqlite_master where name = ? and type='table'", (table,)); - return c.fetchone() != None - - def _create_schemaversion(self, c): - c.execute( - '''create table if not exists SchemaVersion( - version integer not null - );''') - - c.execute("insert into SchemaVersion(version) values (?)", (self.current_schema,)) - - def _create_schema(self, c): - self._create_schemaversion(c) - - c.execute( - '''create table if not exists Deployments( - uuid text primary key - );''') - - c.execute( - '''create table if not exists DeploymentAttrs( - deployment text not null, - name text not null, - value text not null, - primary key(deployment, name), - foreign key(deployment) references Deployments(uuid) on delete cascade - );''') - - c.execute( - '''create table if not exists Resources( - id integer primary key autoincrement, - deployment text not null, - name text not null, - type text not null, - foreign key(deployment) references Deployments(uuid) on delete cascade - );''') - - c.execute( - '''create table if not exists ResourceAttrs( - machine integer not null, - name text not null, - value text not null, - primary key(machine, name), - foreign key(machine) references Resources(id) on delete cascade - );''') - - def _upgrade_1_to_2(self, c): - sys.stderr.write("updating database schema from version 1 to 2...\n") - self._create_schemaversion(c) - - def _upgrade_2_to_3(self, c): - sys.stderr.write("updating database schema from version 2 to 3...\n") - c.execute("alter table Machines rename to Resources") - c.execute("alter table MachineAttrs rename to ResourceAttrs") - - diff --git a/scripts/nixops b/scripts/nixops index 0f7c95a5d..04ba3dbfb 100755 --- a/scripts/nixops +++ b/scripts/nixops @@ -5,7 +5,7 @@ from nixops import deployment from nixops.nix_expr import py2nix from nixops.parallel import MultipleExceptions, run_tasks -import nixops.state.sqlite3_file +import nixops.state.sql_connector import prettytable import argparse import os @@ -724,7 +724,7 @@ subparsers = parser.add_subparsers(help='sub-command help') def add_subparser(name, help): subparser = subparsers.add_parser(name, help=help) subparser.add_argument('--state', '-s', dest='state_url', metavar='FILE', - default=os.environ.get("NIXOPS_STATE_URL", nixops.state.sqlite3_file.get_default_state_file()), help='URL that points to the state provider.') + default=os.environ.get("NIXOPS_STATE_URL", nixops.state.sql_connector.get_default_state_file()), help='URL that points to the state provider.') subparser.add_argument('--deployment', '-d', dest='deployment', metavar='UUID_OR_NAME', default=os.environ.get("NIXOPS_DEPLOYMENT", os.environ.get("CHARON_DEPLOYMENT", None)), help='UUID or symbolic name of the deployment') subparser.add_argument('--debug', action='store_true', help='enable debug output') From 98290e6247a59aab37fddd287b6ad58dd2115c6c Mon Sep 17 00:00:00 2001 From: Matthew O'Gorman Date: Wed, 7 Feb 2018 02:05:57 -0500 Subject: [PATCH 057/123] fixed sqlalchemy sqlite support by using internal implementation. now to convert the rest of the file --- nixops/state/sql_connector.py | 19 +++++++++---------- release.nix | 1 - 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/nixops/state/sql_connector.py b/nixops/state/sql_connector.py index 332158791..3940aa3c6 100644 --- a/nixops/state/sql_connector.py +++ b/nixops/state/sql_connector.py @@ -3,19 +3,18 @@ import nixops.deployment import os import os.path -from pysqlite2 import dbapi2 as sqlite3 +import urlparse import sys import threading import fcntl import sqlalchemy -from sqlalchemy.engine import Engine -from sqlalchemy import event def _subclasses(cls): sub = cls.__subclasses__() return [cls] if not sub else [g for s in sub for g in _subclasses(s)] + def get_default_state_file(): home = os.environ.get("HOME", "") + "/.nixops" if not os.path.exists(home): @@ -35,19 +34,19 @@ class SQLConnection(object): current_schema = 3 - @event.listens_for(Engine, "connect") - def set_sqlite_pragma(dbapi_connection, connection_record): - cursor = dbapi_connection.cursor() - db.execute("pragma journal_mode = wal") - db.execute("pragma foreign_keys = 1") - cursor.close() - def __init__(self, db_uri): print db_uri + url = urlparse.urlparse(db_uri) + print url.scheme self.db_uri = db_uri db_engine = sqlalchemy.create_engine(db_uri) db = db_engine.connect() + + if url.scheme == "sqlite": + db.execute("pragma journal_mode = wal;") + db.execute("pragma foreign_keys;") + # FIXME: this is not actually transactional, because pysqlite (not # sqlite) does an implicit commit before "create table". with db: diff --git a/release.nix b/release.nix index 7c3ac22d2..9f9207147 100644 --- a/release.nix +++ b/release.nix @@ -93,7 +93,6 @@ rec { adal # Go back to sqlite once Python 2.7.13 is released sqlalchemy - pysqlite datadog digital-ocean ]; From 3863edc931c76b1ce3dc95aad49cc4012181aeca Mon Sep 17 00:00:00 2001 From: Matthew O'Gorman Date: Wed, 7 Feb 2018 06:09:59 -0500 Subject: [PATCH 058/123] working sqlalchemy --- nixops/state/sql_connector.py | 266 ++++++++++++++++++---------------- 1 file changed, 142 insertions(+), 124 deletions(-) diff --git a/nixops/state/sql_connector.py b/nixops/state/sql_connector.py index 3940aa3c6..1f96cfd25 100644 --- a/nixops/state/sql_connector.py +++ b/nixops/state/sql_connector.py @@ -20,7 +20,7 @@ def get_default_state_file(): if not os.path.exists(home): old_home = os.environ.get("HOME", "") + "/.charon" if os.path.exists(old_home): - sys.stderr.write("renaming ‘{0}’ to ‘{1}’...\n".format(old_home, home)) + sys.stderr.write("renaming {!r} to {!r}...\n".format(old_home, home)) os.rename(old_home, home) if os.path.exists(home + "/deployments.charon"): os.rename(home + "/deployments.charon", home + "/deployments.nixops") @@ -30,51 +30,41 @@ def get_default_state_file(): class SQLConnection(object): - """NixOps state file.""" + """NixOps db uri.""" current_schema = 3 def __init__(self, db_uri): - print db_uri url = urlparse.urlparse(db_uri) - print url.scheme self.db_uri = db_uri db_engine = sqlalchemy.create_engine(db_uri) db = db_engine.connect() - if url.scheme == "sqlite": db.execute("pragma journal_mode = wal;") db.execute("pragma foreign_keys;") - - # FIXME: this is not actually transactional, because pysqlite (not - # sqlite) does an implicit commit before "create table". - with db: - - c = db.cursor() - - # Get the schema version. - version = 0 # new database - if self._table_exists(c, 'SchemaVersion'): - c.execute("select version from SchemaVersion") - version = c.fetchone()[0] - elif self._table_exists(c, 'Deployments'): - version = 1 - - if version == self.current_schema: - pass - elif version == 0: - self._create_schema(c) - elif version < self.current_schema: - if version <= 1: self._upgrade_1_to_2(c) - if version <= 2: self._upgrade_2_to_3(c) - c.execute("update SchemaVersion set version = ?", (self.current_schema,)) - else: - raise Exception("this NixOps version is too old to deal with schema version {0}".format(version)) + version = 0 # new database + + if db_engine.dialect.has_table(db, 'SchemaVersion'): + version = db.execute("select version from SchemaVersion").scalar() + elif db_engine.dialect.has_table(db, 'Deployments'): + version = 1 + + if version == self.current_schema: + pass + elif version == 0: + self._create_schema(db) + elif version < self.current_schema: + if version <= 1: + self._upgrade_1_to_2(db) + if version <= 2: + self._upgrade_2_to_3(db) + db.execute("update SchemaVersion set version = {!r}".format(self.current_schema)) + else: + raise Exception("this NixOps version is too old to deal with schema version {!r}".format(version)) self.db = db - def close(self): self.db.close() @@ -83,10 +73,9 @@ def close(self): def query_deployments(self): """Return the UUIDs of all deployments in the database.""" - c = self.db.cursor() - c.execute("select uuid from Deployments") - res = c.fetchall() - return [x[0] for x in res] + rows = self.db.execute("select uuid from Deployments") + return [x[0] for x in rows] + def get_all_deployments(self): """Return Deployment objects for every deployment in the database.""" @@ -96,105 +85,131 @@ def get_all_deployments(self): try: res.append(self.open_deployment(uuid=uuid)) except nixops.deployment.UnknownBackend as e: - sys.stderr.write("skipping deployment ‘{0}’: {1}\n".format(uuid, str(e))) + sys.stderr.write("skipping deployment '{}': {!r}\n".format(uuid, str(e))) + print res + print 'done' return res + def _find_deployment(self, uuid=None): - c = self.db.cursor() if not uuid: - c.execute("select uuid from Deployments") + rows = self.db.execute("select count(uuid), uuid from Deployments") else: - c.execute("select uuid from Deployments d where uuid = ? or exists (select 1 from DeploymentAttrs where deployment = d.uuid and name = 'name' and value = ?)", (uuid, uuid)) - res = c.fetchall() - if len(res) == 0: + rows = self.db.execute("select count(uuid), uuid from Deployments d where uuid = '{}' or exists (select 1 from DeploymentAttrs where deployment = d.uuid and name = 'name' and value = '{}')".format(uuid, uuid)) + row_count = 0 + deployment = None + for row in rows: + row_count = row[0] + deployment = row[1] + break + + if row_count == 0: if uuid: # try the prefix match - c.execute("select uuid from Deployments where uuid glob ?", (uuid + '*', )) - res = c.fetchall() - if len(res) == 0: + rows = self.db.execute("select count(uuid), uuid from Deployments where uuid glob '{}'".format(uuid + '*')) + for row in rows: + row_count = row[0] + deployment = row[1] + break + + if row_count == 0: return None else: return None - if len(res) > 1: + + if row_count > 1: if uuid: raise Exception("state file contains multiple deployments with the same name, so you should specify one using its UUID") else: raise Exception("state file contains multiple deployments, so you should specify which one to use using ‘-d’, or set the environment variable NIXOPS_DEPLOYMENT") - return nixops.deployment.Deployment(self, res[0][0], sys.stderr) + print 'mog {}'.format(deployment) + return nixops.deployment.Deployment(self, deployment, sys.stderr) + def open_deployment(self, uuid=None): + print 'open a deployment' """Open an existing deployment.""" deployment = self._find_deployment(uuid=uuid) + print deployment if deployment: return deployment - raise Exception("could not find specified deployment in state file ‘{0}’".format(self.db_file)) + raise Exception("could not find specified deployment in state file {!r}".format(self.db_uri)) + def create_deployment(self, uuid=None): + print 'createing a deployment' """Create a new deployment.""" if not uuid: import uuid uuid = str(uuid.uuid1()) - with self.db: - self.db.execute("insert into Deployments(uuid) values (?)", (uuid,)) + self.db.execute("insert into Deployments(uuid) values ('{}')".format(uuid)) return nixops.deployment.Deployment(self, uuid, sys.stderr) + def _delete_deployment(self, deployment_uuid): """NOTE: This is UNSAFE, it's guarded in nixops/deployment.py. Do not call this function except from there!""" - self.db.execute("delete from Deployments where uuid = ?", (deployment_uuid,)) + self.db.execute("delete from Deployments where uuid = '{}'".format(deployment_uuid)) - def clone_deployment(self, deployment_uuid): - with self.db: - new = self.create_deployment() - self.db.execute("insert into DeploymentAttrs (deployment, name, value) " + - "select ?, name, value from DeploymentAttrs where deployment = ?", - (new.uuid, deployment_uuid)) - new.configs_path = None - return new + def clone_deployment(self, deployment_uuid): + new = self.create_deployment() + self.db.execute("insert into DeploymentAttrs (deployment, name, value) " + + "select '{}', name, value from DeploymentAttrs where deployment = '{}'" + .format(new.uuid, deployment_uuid) + ) + new.configs_path = None + return new def get_resources_for(self, deployment): """Get all the resources for a certain deployment""" resources = {} - with self.db: - c = self.db.cursor() - c.execute("select id, name, type from Resources where deployment = ?", (deployment.uuid,)) - for (id, name, type) in c.fetchall(): - r = self._create_state(deployment, type, name, id) - resources[name] = r + print 'get some things {}'.format(deployment.uuid) + rows = self.db.execute("select id, name, type from Resources where deployment = '{}'".format(deployment.uuid)) + print 'got some thangs {}'.format(rows) + for row in rows: + print 'asdf' + print row + for (id, name, type) in rows: + print 'WOOT {} {} {}'.format(id, name, type) + r = self._create_state(deployment, type, name, id) + resources[name] = r return resources + def set_deployment_attrs(self, deployment_uuid, attrs): """Update deployment attributes in the state.""" - with self.db: - c = self.db.cursor() - for n, v in attrs.iteritems(): - if v == None: - c.execute("delete from DeploymentAttrs where deployment = ? and name = ?", (deployment_uuid, n)) - else: - c.execute("insert or replace into DeploymentAttrs(deployment, name, value) values (?, ?, ?)", - (deployment_uuid, n, v)) + for name, value in attrs.iteritems(): + if value == None: + self.db.execute("delete from DeploymentAttrs where deployment = '{}' and name = '{}'" + .format(deployment_uuid, name) + ) + else: + self.db.execute("insert or replace into DeploymentAttrs(deployment, name, value) values ('{}', '{}', {!r})" + .format(deployment_uuid, name, value) + ) + def del_deployment_attr(self, deployment_uuid, attr_name): - with self.db: - self.db.execute("delete from DeploymentAttrs where deployment = ? and name = ?", (deployment_uuid, attr_name)) + self.db.execute("delete from DeploymentAttrs where deployment = '{}' and name = {!r}" + .format(deployment_uuid, attr_name) + ) def get_deployment_attr(self, deployment_uuid, name): """Get a deployment attribute from the state.""" - with self.db: - c = self.db.cursor() - c.execute("select value from DeploymentAttrs where deployment = ? and name = ?", (deployment_uuid, name)) - row = c.fetchone() - if row != None: return row[0] - return nixops.util.undefined + rows = self.db.execute("select value from DeploymentAttrs where deployment = '{}' and name = {!r}" + .format(deployment_uuid, name)) + for row in rows: + return row[0] + return nixops.util.undefined + def get_all_deployment_attrs(self, deployment_uuid): - with self.db: - c = self.db.cursor() - c.execute("select name, value from DeploymentAttrs where deployment = ?", (deployment_uuid)) - rows = c.fetchall() - res = {row[0]: row[1] for row in rows} - return res + rows = self.db.execute("select name, value from DeploymentAttrs where deployment = '{}'" + .format(deployment_uuid)) + res = {row[0]: row[1] for row in rows} + return res + def get_deployment_lock(self, deployment): lock_dir = os.environ.get("HOME", "") + "/.nixops/locks" @@ -224,56 +239,61 @@ def __exit__(self, exception_type, exception_value, exception_traceback): ## Resources def create_resource(self, deployment, name, type): - c = self.db.cursor() - c.execute("select 1 from Resources where deployment = ? and name = ?", (deployment.uuid, name)) - if len(c.fetchall()) != 0: + count = self.db.execute("select count(id) from Resources where deployment = '{}' and name = {!r}" + .format(deployment.uuid, name)).scalar() + + if count != 0: raise Exception("resource already exists in database!") - c.execute("insert into Resources(deployment, name, type) values (?, ?, ?)", - (deployment.uuid, name, type)) - id = c.lastrowid + + result = self.db.execute("insert into Resources(deployment, name, type) values ('{}', {!r}, {!r})" + .format(deployment.uuid, name, type)) + + id = result.lastrowid r = self._create_state(deployment, type, name, id) return r + def delete_resource(self, deployment_uuid, res_id): - with self.db: - self.db.execute("delete from Resources where deployment = ? and id = ?", (deployment_uuid, res_id)) + self.db.execute("delete from Resources where deployment = '{}' and id = {!r}" + .format(deployment_uuid, res_id)) + def _rename_resource(self, deployment_uuid, resource_id, new_name): """NOTE: Invariants are checked in nixops/deployment.py#rename""" - with self.db: - self.db.execute("update Resources set name = ? where deployment = ? and id = ?", (new_name, deployment_uuid, resource_id)) + self.db.execute("update Resources set name = '{}' where deployment = '{}' and id = {!r}" + .format(new_name, deployment_uuid, resource_id)) def set_resource_attrs(self, _deployment_uuid, resource_id, attrs): - with self.db: - c = self.db.cursor() - for n, v in attrs.iteritems(): - if v == None: - c.execute("delete from ResourceAttrs where machine = ? and name = ?", (resource_id, n)) - else: - c.execute("insert or replace into ResourceAttrs(machine, name, value) values (?, ?, ?)", - (resource_id, n, v)) + for name, value in attrs.iteritems(): + if value == None: + self.db.execute("delete from ResourceAttrs where machine = '{}' and name = '{}'" + .format(resource_id, name)) + else: + self.db.execute("insert or replace into ResourceAttrs(machine, name, value) values ('{}', '{}', {!r})" + .format(resource_id, name, value)) + def del_resource_attr(self, _deployment_uuid, resource_id, name): - with self.db: - self.db.execute("delete from ResourceAttrs where machine = ? and name = ?", (resource_id, name)) + self.db.execute("delete from ResourceAttrs where machine = {!r} and name = {!r}" + .format(resource_id, name)) + def get_resource_attr(self, _deployment_uuid, resource_id, name): """Get a machine attribute from the state file.""" - with self.db: - c = self.db.cursor() - c.execute("select value from ResourceAttrs where machine = ? and name = ?", (resource_id, name)) - row = c.fetchone() - if row != None: return row[0] - return nixops.util.undefined + rows = self.db.execute("select value from ResourceAttrs where machine = {!r} and name = {!r}" + .format(resource_id, name)) + if rows is not None: + return row[0][0] + return nixops.util.undefined + def get_all_resource_attrs(self, deployment_uuid, resource_id): - with self.db: - c = self.db.cursor() - c.execute("select name, value from ResourceAttrs where machine = ?", (resource_id,)) - rows = c.fetchall() - res = {row[0]: row[1] for row in rows} - return res + rows = self.db.execute("select name, value from ResourceAttrs where machine = {!r}" + .format(resource_id)) + res = {row[0]: row[1] for row in rows} + return res + ### STATE def _create_state(self, depl, type, name, id): @@ -283,12 +303,9 @@ def _create_state(self, depl, type, name, id): if type == cls.get_type(): return cls(depl, name, id) - raise nixops.deployment.UnknownBackend("unknown resource type ‘{0}’".format(type)) - + raise nixops.deployment.UnknownBackend("unknown resource type ‘{!r}’" + .format(type)) - def _table_exists(self, c, table): - c.execute("select 1 from sqlite_master where name = ? and type='table'", (table,)); - return c.fetchone() != None def _create_schemaversion(self, c): c.execute( @@ -296,7 +313,8 @@ def _create_schemaversion(self, c): version integer not null );''') - c.execute("insert into SchemaVersion(version) values (?)", (self.current_schema,)) + c.execute("insert into SchemaVersion(version) values ({!r})" + .format(self.current_schema)) def _create_schema(self, c): self._create_schemaversion(c) From c11756c6fc8a8af41c4f3ec6c5796286773f42ea Mon Sep 17 00:00:00 2001 From: Matthew O'Gorman Date: Wed, 7 Feb 2018 06:10:51 -0500 Subject: [PATCH 059/123] removed prints --- nixops/state/sql_connector.py | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/nixops/state/sql_connector.py b/nixops/state/sql_connector.py index 1f96cfd25..023ebc138 100644 --- a/nixops/state/sql_connector.py +++ b/nixops/state/sql_connector.py @@ -86,8 +86,7 @@ def get_all_deployments(self): res.append(self.open_deployment(uuid=uuid)) except nixops.deployment.UnknownBackend as e: sys.stderr.write("skipping deployment '{}': {!r}\n".format(uuid, str(e))) - print res - print 'done' + return res @@ -122,21 +121,18 @@ def _find_deployment(self, uuid=None): raise Exception("state file contains multiple deployments with the same name, so you should specify one using its UUID") else: raise Exception("state file contains multiple deployments, so you should specify which one to use using ‘-d’, or set the environment variable NIXOPS_DEPLOYMENT") - print 'mog {}'.format(deployment) return nixops.deployment.Deployment(self, deployment, sys.stderr) def open_deployment(self, uuid=None): - print 'open a deployment' """Open an existing deployment.""" deployment = self._find_deployment(uuid=uuid) - print deployment + if deployment: return deployment raise Exception("could not find specified deployment in state file {!r}".format(self.db_uri)) def create_deployment(self, uuid=None): - print 'createing a deployment' """Create a new deployment.""" if not uuid: import uuid @@ -163,14 +159,9 @@ def clone_deployment(self, deployment_uuid): def get_resources_for(self, deployment): """Get all the resources for a certain deployment""" resources = {} - print 'get some things {}'.format(deployment.uuid) + rows = self.db.execute("select id, name, type from Resources where deployment = '{}'".format(deployment.uuid)) - print 'got some thangs {}'.format(rows) - for row in rows: - print 'asdf' - print row for (id, name, type) in rows: - print 'WOOT {} {} {}'.format(id, name, type) r = self._create_state(deployment, type, name, id) resources[name] = r return resources From b42814f0e28a5440738847f5e550813df7d9c2e5 Mon Sep 17 00:00:00 2001 From: Matthew O'Gorman Date: Wed, 7 Feb 2018 06:44:40 -0500 Subject: [PATCH 060/123] beginning of porting to mysql as well. will need specific sql for creation of tables and database it looks like --- nixops/state/__init__.py | 17 ++++++++++------- nixops/state/sql_connector.py | 12 +++++++++--- release.nix | 1 + 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/nixops/state/__init__.py b/nixops/state/__init__.py index e29fa7dce..73149d325 100644 --- a/nixops/state/__init__.py +++ b/nixops/state/__init__.py @@ -8,14 +8,14 @@ class WrongStateSchemeException(Exception): pass def open(url): - url = urlparse.urlparse(url) - scheme = url.scheme - ext = os.path.splitext(url.path)[1] - + url_parsed = urlparse.urlparse(url) + scheme = url_parsed.scheme + ext = os.path.splitext(url)[1] + print 'a url {}'.format(url) if scheme == "": if ext == ".nixops": scheme = "sql" - url = 'sqlite:///' + url.path + url = 'sqlite:///' + url elif ext == ".json": scheme = "json" @@ -23,9 +23,12 @@ def raise_(ex): raise ex switcher = { - "json": lambda(url): json_file.JsonFile(url.path), - "sql": lambda(url): sql_connector.SQLConnection(url), + "json": lambda(url): json_file.JsonFile(url), + "sqlite": lambda(url): sql_connector.SQLConnection(url), + "mysql": lambda(url): sql_connector.SQLConnection(url), + "sql": lambda(url): sql_connector.SQLConnection(url) } + print 'a url {}'.format(url) function = switcher.get(scheme, lambda(url): raise_(WrongStateSchemeException("Unknown state scheme!"))) return function(url) diff --git a/nixops/state/sql_connector.py b/nixops/state/sql_connector.py index 023ebc138..9dec756a9 100644 --- a/nixops/state/sql_connector.py +++ b/nixops/state/sql_connector.py @@ -10,6 +10,9 @@ import sqlalchemy +import pymysql_sa +pymysql_sa.make_default_mysql_dialect() + def _subclasses(cls): sub = cls.__subclasses__() return [cls] if not sub else [g for s in sub for g in _subclasses(s)] @@ -38,7 +41,10 @@ def __init__(self, db_uri): url = urlparse.urlparse(db_uri) self.db_uri = db_uri + print 'oh no you didnt {}'.format(db_uri) + db_engine = sqlalchemy.create_engine(db_uri) + db = db_engine.connect() if url.scheme == "sqlite": db.execute("pragma journal_mode = wal;") @@ -312,13 +318,13 @@ def _create_schema(self, c): c.execute( '''create table if not exists Deployments( - uuid text primary key + uuid VARCHAR(36) primary key );''') c.execute( '''create table if not exists DeploymentAttrs( - deployment text not null, - name text not null, + deployment varchar(255) not null, + name varchar(255) not null, value text not null, primary key(deployment, name), foreign key(deployment) references Deployments(uuid) on delete cascade diff --git a/release.nix b/release.nix index 9f9207147..3e4962a1e 100644 --- a/release.nix +++ b/release.nix @@ -93,6 +93,7 @@ rec { adal # Go back to sqlite once Python 2.7.13 is released sqlalchemy + pymysqlsa datadog digital-ocean ]; From 19d31781693f6a42d28f6426603eddcaff52d667 Mon Sep 17 00:00:00 2001 From: Matthew O'Gorman Date: Wed, 7 Feb 2018 07:08:08 -0500 Subject: [PATCH 061/123] working mysql support --- nixops/state/__init__.py | 2 -- nixops/state/sql_connector.py | 41 +++++++++++++++++++++++------------ 2 files changed, 27 insertions(+), 16 deletions(-) diff --git a/nixops/state/__init__.py b/nixops/state/__init__.py index 73149d325..b5313de05 100644 --- a/nixops/state/__init__.py +++ b/nixops/state/__init__.py @@ -11,7 +11,6 @@ def open(url): url_parsed = urlparse.urlparse(url) scheme = url_parsed.scheme ext = os.path.splitext(url)[1] - print 'a url {}'.format(url) if scheme == "": if ext == ".nixops": scheme = "sql" @@ -29,6 +28,5 @@ def raise_(ex): "sql": lambda(url): sql_connector.SQLConnection(url) } - print 'a url {}'.format(url) function = switcher.get(scheme, lambda(url): raise_(WrongStateSchemeException("Unknown state scheme!"))) return function(url) diff --git a/nixops/state/sql_connector.py b/nixops/state/sql_connector.py index 9dec756a9..262529fc2 100644 --- a/nixops/state/sql_connector.py +++ b/nixops/state/sql_connector.py @@ -41,8 +41,6 @@ def __init__(self, db_uri): url = urlparse.urlparse(db_uri) self.db_uri = db_uri - print 'oh no you didnt {}'.format(db_uri) - db_engine = sqlalchemy.create_engine(db_uri) db = db_engine.connect() @@ -181,9 +179,14 @@ def set_deployment_attrs(self, deployment_uuid, attrs): .format(deployment_uuid, name) ) else: - self.db.execute("insert or replace into DeploymentAttrs(deployment, name, value) values ('{}', '{}', {!r})" - .format(deployment_uuid, name, value) - ) + if self.get_deployment_attr(deployment_uuid, name) == nixops.util.undefined: + self.db.execute("insert into DeploymentAttrs(deployment, name, value) values ('{}', '{}', {!r})" + .format(deployment_uuid, name, value) + ) + else: + self.db.execute("update DeploymentAttrs set value={!r} where deployment='{}' and name='{}'" + .format(value, deployment_uuid, name) + ) def del_deployment_attr(self, deployment_uuid, attr_name): @@ -267,8 +270,14 @@ def set_resource_attrs(self, _deployment_uuid, resource_id, attrs): self.db.execute("delete from ResourceAttrs where machine = '{}' and name = '{}'" .format(resource_id, name)) else: - self.db.execute("insert or replace into ResourceAttrs(machine, name, value) values ('{}', '{}', {!r})" - .format(resource_id, name, value)) + if get_resource_attr(deployment_uuid, name) == nixops.util.undefined: + self.db.execute("insert into ResourceAttrs(machine, name, value) values ('{}', '{}', {!r})" + .format(resource_id, name, value) + ) + else: + self.db.execute("update ResourceAttrs set value={!r} where machine='{}' and name='{}'" + .format(value, resource_id, name) + ) def del_resource_attr(self, _deployment_uuid, resource_id, name): @@ -330,20 +339,24 @@ def _create_schema(self, c): foreign key(deployment) references Deployments(uuid) on delete cascade );''') + autoincrement = 'autoincrement' + url = urlparse.urlparse(self.db_uri) + if url.scheme == 'mysql': + autoincrement = 'auto_increment' c.execute( '''create table if not exists Resources( - id integer primary key autoincrement, - deployment text not null, - name text not null, - type text not null, + id integer primary key {}, + deployment varchar(255) not null, + name varchar(255) not null, + type varchar(1024) not null, foreign key(deployment) references Deployments(uuid) on delete cascade - );''') + );'''.format(autoincrement)) c.execute( '''create table if not exists ResourceAttrs( machine integer not null, - name text not null, - value text not null, + name varchar(255) not null, + value varchar(1024) not null, primary key(machine, name), foreign key(machine) references Resources(id) on delete cascade );''') From 5d200eee165b6ec612b4e17d1788d3d5fd820a9f Mon Sep 17 00:00:00 2001 From: Matthew O'Gorman Date: Wed, 7 Feb 2018 12:43:12 -0500 Subject: [PATCH 062/123] possible row exception --- nixops/state/sql_connector.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nixops/state/sql_connector.py b/nixops/state/sql_connector.py index 262529fc2..1b7da92ad 100644 --- a/nixops/state/sql_connector.py +++ b/nixops/state/sql_connector.py @@ -287,10 +287,10 @@ def del_resource_attr(self, _deployment_uuid, resource_id, name): def get_resource_attr(self, _deployment_uuid, resource_id, name): """Get a machine attribute from the state file.""" - rows = self.db.execute("select value from ResourceAttrs where machine = {!r} and name = {!r}" + rows = self.db.execute("select value from ResourceAttrs where machine = '{}' and name = '{}'" .format(resource_id, name)) - if rows is not None: - return row[0][0] + for row in rows: + return row[0] return nixops.util.undefined From 6e71adc8d07e5ea1258e186d36cd2847124fed4e Mon Sep 17 00:00:00 2001 From: Matthew O'Gorman Date: Sat, 10 Feb 2018 17:16:30 -0500 Subject: [PATCH 063/123] better locking and some temporary debugging --- nixops/backends/virtualbox.py | 3 +- nixops/deployment.py | 1 + nixops/known_hosts.py | 1 + nixops/resources/__init__.py | 1 + nixops/state/sql_connector.py | 277 +++++++++++++++++++++------------- nixops/util.py | 3 + 6 files changed, 176 insertions(+), 110 deletions(-) diff --git a/nixops/backends/virtualbox.py b/nixops/backends/virtualbox.py index f5c53e909..975e28448 100644 --- a/nixops/backends/virtualbox.py +++ b/nixops/backends/virtualbox.py @@ -141,6 +141,7 @@ def _update_ip(self): capture_stdout=True).rstrip() if res[0:7] != "Value: ": return new_address = res[7:] + print 'called here with data' nixops.known_hosts.update(self.private_ipv4, new_address, self.public_host_key) self.private_ipv4 = new_address @@ -394,7 +395,7 @@ def destroy(self, wipe=False): self.state = self.STOPPED time.sleep(1) # hack to work around "machine locked" errors - + print 'called here with none?' nixops.known_hosts.update(self.private_ipv4, None, self.public_host_key) self._logged_exec(["VBoxManage", "unregistervm", "--delete", self.vm_id]) diff --git a/nixops/deployment.py b/nixops/deployment.py index 66e11b126..c2b3cd9d0 100644 --- a/nixops/deployment.py +++ b/nixops/deployment.py @@ -135,6 +135,7 @@ def _del_attr(self, name): # Removed it when moving the body to nixops/state/file.py. def _get_attr(self, name, default=nixops.util.undefined): """Get a deployment attribute from the state.""" + print 'get deploymante attr' return self._state.get_deployment_attr(self.uuid, name) def _create_resource(self, name, type): diff --git a/nixops/known_hosts.py b/nixops/known_hosts.py index fcd05b0c7..88d7fc11b 100644 --- a/nixops/known_hosts.py +++ b/nixops/known_hosts.py @@ -57,6 +57,7 @@ def add(ip_address, public_host_key): def update(prev_address, new_address, public_host_key): + print 'FOUND on UPDATE {} {} {}'.format(prev_address, new_address, public_host_key) assert public_host_key is not None # FIXME: this rewrites known_hosts twice. if prev_address is not None and prev_address != new_address: diff --git a/nixops/resources/__init__.py b/nixops/resources/__init__.py index 728f56104..778e24ad3 100644 --- a/nixops/resources/__init__.py +++ b/nixops/resources/__init__.py @@ -81,6 +81,7 @@ def _del_attr(self, name): # Have removed it in state/file.py. def _get_attr(self, name, default=nixops.util.undefined): """Get a machine attribute from the state file.""" + print 'get resource attr' return self.depl._state.get_resource_attr(self.depl.uuid, self.id, name) def export(self): diff --git a/nixops/state/sql_connector.py b/nixops/state/sql_connector.py index 1b7da92ad..1669f98da 100644 --- a/nixops/state/sql_connector.py +++ b/nixops/state/sql_connector.py @@ -8,7 +8,8 @@ import threading import fcntl -import sqlalchemy +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker import pymysql_sa pymysql_sa.make_default_mysql_dialect() @@ -32,6 +33,50 @@ def get_default_state_file(): return os.environ.get("NIXOPS_STATE", os.environ.get("CHARON_STATE", home + "/deployments.nixops")) +class SQLAlchemyDBConnection(object): + """SQLAlchemy database connection""" + + def __init__(self, connection_string): + print 'init' + self.connection_string = connection_string + self.session = None + self.nesting = 0 + self.lock = threading.RLock() + + def __enter__(self): + print 'enter' + self.lock.acquire() + if self.nesting == 0: + self.must_rollback = False + self.nesting = self.nesting + 1 + engine = create_engine(self.connection_string, echo = True) + Session = sessionmaker() + self.session = Session(bind=engine) + self.engine = engine + return self + + def __exit__(self, exception_type, excption_value, exception_traceback): + print 'exit' + if exception_type != None: + self.must_rollback = True + self.nesting = self.nesting - 1 + assert self.nesting >= 0 + if self.nesting == 0: + print 'not nested' + if self.must_rollback: + print 'MUST ROLL BACK' + try: + self.session.rollback() + except: + pass + else: + print 'closing' + self.session.flush() + self.session.commit() + self.session.close() + self.lock.release() + + class SQLConnection(object): """NixOps db uri.""" @@ -41,31 +86,30 @@ def __init__(self, db_uri): url = urlparse.urlparse(db_uri) self.db_uri = db_uri - db_engine = sqlalchemy.create_engine(db_uri) - - db = db_engine.connect() - if url.scheme == "sqlite": - db.execute("pragma journal_mode = wal;") - db.execute("pragma foreign_keys;") - version = 0 # new database - - if db_engine.dialect.has_table(db, 'SchemaVersion'): - version = db.execute("select version from SchemaVersion").scalar() - elif db_engine.dialect.has_table(db, 'Deployments'): - version = 1 - - if version == self.current_schema: - pass - elif version == 0: - self._create_schema(db) - elif version < self.current_schema: - if version <= 1: - self._upgrade_1_to_2(db) - if version <= 2: - self._upgrade_2_to_3(db) - db.execute("update SchemaVersion set version = {!r}".format(self.current_schema)) - else: - raise Exception("this NixOps version is too old to deal with schema version {!r}".format(version)) + db = SQLAlchemyDBConnection(db_uri) + with db: + if url.scheme == "sqlite": + db.session.execute("pragma journal_mode = wal;") + db.session.execute("pragma foreign_keys;") + with db: + version = 0 # new database + if db.engine.dialect.has_table(db.session, 'SchemaVersion'): + version = db.session.execute("select version from SchemaVersion").scalar() + elif db.engine.dialect.has_table(db.session, 'Deployments'): + version = 1 + + if version == self.current_schema: + pass + elif version == 0: + self._create_schema(db) + elif version < self.current_schema: + if version <= 1: + self._upgrade_1_to_2(db) + if version <= 2: + self._upgrade_2_to_3(db) + db.session.execute("update SchemaVersion set version = :current_schema", {'current_schema': self.current_schema}) + else: + raise Exception("this NixOps version is too old to deal with schema version {}".format(version)) self.db = db @@ -77,7 +121,7 @@ def close(self): def query_deployments(self): """Return the UUIDs of all deployments in the database.""" - rows = self.db.execute("select uuid from Deployments") + rows = self.db.session.execute("select uuid from Deployments") return [x[0] for x in rows] @@ -89,16 +133,16 @@ def get_all_deployments(self): try: res.append(self.open_deployment(uuid=uuid)) except nixops.deployment.UnknownBackend as e: - sys.stderr.write("skipping deployment '{}': {!r}\n".format(uuid, str(e))) + sys.stderr.write("skipping deployment '{}': {}\n".format(uuid, str(e))) return res def _find_deployment(self, uuid=None): if not uuid: - rows = self.db.execute("select count(uuid), uuid from Deployments") + rows = self.db.session.execute("select count(uuid), uuid from Deployments") else: - rows = self.db.execute("select count(uuid), uuid from Deployments d where uuid = '{}' or exists (select 1 from DeploymentAttrs where deployment = d.uuid and name = 'name' and value = '{}')".format(uuid, uuid)) + rows = self.db.session.execute("select count(uuid), uuid from Deployments d where uuid = :uuid or exists (select 1 from DeploymentAttrs where deployment = d.uuid and name = 'name' and value = :value)", {'uuid': uuid, 'value': uuid}) row_count = 0 deployment = None for row in rows: @@ -109,7 +153,7 @@ def _find_deployment(self, uuid=None): if row_count == 0: if uuid: # try the prefix match - rows = self.db.execute("select count(uuid), uuid from Deployments where uuid glob '{}'".format(uuid + '*')) + rows = self.db.session.execute("select count(uuid), uuid from Deployments where uuid glob :uuid".format(uuid + '*')) for row in rows: row_count = row[0] deployment = row[1] @@ -141,74 +185,82 @@ def create_deployment(self, uuid=None): if not uuid: import uuid uuid = str(uuid.uuid1()) - self.db.execute("insert into Deployments(uuid) values ('{}')".format(uuid)) + with self.db: + self.db.session.execute("insert into Deployments(uuid) values ('{}')".format(uuid)) return nixops.deployment.Deployment(self, uuid, sys.stderr) def _delete_deployment(self, deployment_uuid): """NOTE: This is UNSAFE, it's guarded in nixops/deployment.py. Do not call this function except from there!""" - self.db.execute("delete from Deployments where uuid = '{}'".format(deployment_uuid)) + self.db.session.execute("delete from Deployments where uuid = '{}'".format(deployment_uuid)) def clone_deployment(self, deployment_uuid): - new = self.create_deployment() - self.db.execute("insert into DeploymentAttrs (deployment, name, value) " + - "select '{}', name, value from DeploymentAttrs where deployment = '{}'" - .format(new.uuid, deployment_uuid) - ) - new.configs_path = None - return new + with self.db: + new = self.create_deployment() + self.db.session.execute("insert into DeploymentAttrs (deployment, name, value) " + + "select '{}', name, value from DeploymentAttrs where deployment = '{}'" + .format(new.uuid, deployment_uuid) + ) + new.configs_path = None + return new def get_resources_for(self, deployment): """Get all the resources for a certain deployment""" - resources = {} + with self.db: + resources = {} - rows = self.db.execute("select id, name, type from Resources where deployment = '{}'".format(deployment.uuid)) - for (id, name, type) in rows: - r = self._create_state(deployment, type, name, id) - resources[name] = r + rows = self.db.session.execute("select id, name, type from Resources where deployment = '{}'".format(deployment.uuid)) + for (id, name, type) in rows: + r = self._create_state(deployment, type, name, id) + resources[name] = r return resources def set_deployment_attrs(self, deployment_uuid, attrs): """Update deployment attributes in the state.""" - for name, value in attrs.iteritems(): - if value == None: - self.db.execute("delete from DeploymentAttrs where deployment = '{}' and name = '{}'" - .format(deployment_uuid, name) - ) - else: - if self.get_deployment_attr(deployment_uuid, name) == nixops.util.undefined: - self.db.execute("insert into DeploymentAttrs(deployment, name, value) values ('{}', '{}', {!r})" - .format(deployment_uuid, name, value) + with self.db: + for name, value in attrs.iteritems(): + if value == None: + self.db.session.execute("delete from DeploymentAttrs where deployment = '{}' and name = '{}'" + .format(deployment_uuid, name) ) else: - self.db.execute("update DeploymentAttrs set value={!r} where deployment='{}' and name='{}'" - .format(value, deployment_uuid, name) - ) + if self.get_deployment_attr(deployment_uuid, name) == nixops.util.undefined: + self.db.session.execute("insert into DeploymentAttrs(deployment, name, value) values ('{}', '{}', {!r})" + .format(deployment_uuid, name, value) + ) + else: + self.db.session.execute("update DeploymentAttrs set value={!r} where deployment='{}' and name='{}'" + .format(value, deployment_uuid, name) + ) def del_deployment_attr(self, deployment_uuid, attr_name): - self.db.execute("delete from DeploymentAttrs where deployment = '{}' and name = {!r}" - .format(deployment_uuid, attr_name) - ) + with self.db: + self.db.session.execute("delete from DeploymentAttrs where deployment = '{}' and name = {!r}" + .format(deployment_uuid, attr_name) + ) def get_deployment_attr(self, deployment_uuid, name): """Get a deployment attribute from the state.""" - rows = self.db.execute("select value from DeploymentAttrs where deployment = '{}' and name = {!r}" - .format(deployment_uuid, name)) - for row in rows: - return row[0] - return nixops.util.undefined + with self.db: + rows = self.db.session.execute("select value from DeploymentAttrs where deployment = '{}' and name = {!r}" + .format(deployment_uuid, name)) + for row in rows: + print 'here is the deployment {}'.format(row[0]) + return row[0] + return nixops.util.undefined def get_all_deployment_attrs(self, deployment_uuid): - rows = self.db.execute("select name, value from DeploymentAttrs where deployment = '{}'" - .format(deployment_uuid)) - res = {row[0]: row[1] for row in rows} - return res + with self.db: + rows = self.db.session.execute("select name, value from DeploymentAttrs where deployment = '{}'" + .format(deployment_uuid)) + res = {row[0]: row[1] for row in rows} + return res def get_deployment_lock(self, deployment): @@ -239,13 +291,13 @@ def __exit__(self, exception_type, exception_value, exception_traceback): ## Resources def create_resource(self, deployment, name, type): - count = self.db.execute("select count(id) from Resources where deployment = '{}' and name = {!r}" + count = self.db.session.execute("select count(id) from Resources where deployment = '{}' and name = {!r}" .format(deployment.uuid, name)).scalar() if count != 0: raise Exception("resource already exists in database!") - result = self.db.execute("insert into Resources(deployment, name, type) values ('{}', {!r}, {!r})" + result = self.db.session.execute("insert into Resources(deployment, name, type) values ('{}', {!r}, {!r})" .format(deployment.uuid, name, type)) id = result.lastrowid @@ -254,56 +306,63 @@ def create_resource(self, deployment, name, type): def delete_resource(self, deployment_uuid, res_id): - self.db.execute("delete from Resources where deployment = '{}' and id = {!r}" - .format(deployment_uuid, res_id)) + with self.db: + self.db.session.execute("delete from Resources where deployment = '{}' and id = {!r}" + .format(deployment_uuid, res_id)) def _rename_resource(self, deployment_uuid, resource_id, new_name): """NOTE: Invariants are checked in nixops/deployment.py#rename""" - self.db.execute("update Resources set name = '{}' where deployment = '{}' and id = {!r}" - .format(new_name, deployment_uuid, resource_id)) + with self.db: + self.db.session.execute("update Resources set name = '{}' where deployment = '{}' and id = {!r}" + .format(new_name, deployment_uuid, resource_id)) def set_resource_attrs(self, _deployment_uuid, resource_id, attrs): - for name, value in attrs.iteritems(): - if value == None: - self.db.execute("delete from ResourceAttrs where machine = '{}' and name = '{}'" - .format(resource_id, name)) - else: - if get_resource_attr(deployment_uuid, name) == nixops.util.undefined: - self.db.execute("insert into ResourceAttrs(machine, name, value) values ('{}', '{}', {!r})" - .format(resource_id, name, value) - ) + with self.db: + for name, value in attrs.iteritems(): + if value == None: + self.db.session.execute("delete from ResourceAttrs where machine = '{}' and name = '{}'" + .format(resource_id, name)) else: - self.db.execute("update ResourceAttrs set value={!r} where machine='{}' and name='{}'" - .format(value, resource_id, name) - ) + if self.get_resource_attr(_deployment_uuid, resource_id, name) == nixops.util.undefined: + self.db.session.execute("insert into ResourceAttrs(machine, name, value) values ('{}', '{}', '{}')" + .format(resource_id, name, value) + ) + else: + self.db.session.execute("update ResourceAttrs set value={!r} where machine='{}' and name='{}'" + .format(value, resource_id, name) + ) def del_resource_attr(self, _deployment_uuid, resource_id, name): - self.db.execute("delete from ResourceAttrs where machine = {!r} and name = {!r}" - .format(resource_id, name)) + with self.db: + self.db.session.execute("delete from ResourceAttrs where machine = {!r} and name = {!r}" + .format(resource_id, name)) def get_resource_attr(self, _deployment_uuid, resource_id, name): - """Get a machine attribute from the state file.""" - rows = self.db.execute("select value from ResourceAttrs where machine = '{}' and name = '{}'" - .format(resource_id, name)) - for row in rows: - return row[0] - return nixops.util.undefined + with self.db: + """Get a machine attribute from the state file.""" + rows = self.db.session.execute("select value from ResourceAttrs where machine = '{}' and name = '{}'" + .format(resource_id, name)) + for row in rows: + print 'here is the resource {}'.format(row[0]) + return row[0] + return nixops.util.undefined def get_all_resource_attrs(self, deployment_uuid, resource_id): - rows = self.db.execute("select name, value from ResourceAttrs where machine = {!r}" - .format(resource_id)) - res = {row[0]: row[1] for row in rows} - return res + with self.db: + rows = self.db.session.execute("select name, value from ResourceAttrs where machine = {!r}" + .format(resource_id)) + res = {row[0]: row[1] for row in rows} + return res ### STATE def _create_state(self, depl, type, name, id): - """Create a resource state object of the desired type.""" + """Create a resource state object of the desired type.""" for cls in _subclasses(nixops.resources.ResourceState): if type == cls.get_type(): @@ -314,23 +373,23 @@ def _create_state(self, depl, type, name, id): def _create_schemaversion(self, c): - c.execute( + c.session.execute( '''create table if not exists SchemaVersion( version integer not null );''') - c.execute("insert into SchemaVersion(version) values ({!r})" + c.session.execute("insert into SchemaVersion(version) values ({!r})" .format(self.current_schema)) def _create_schema(self, c): self._create_schemaversion(c) - c.execute( + c.session.execute( '''create table if not exists Deployments( uuid VARCHAR(36) primary key );''') - c.execute( + c.session.execute( '''create table if not exists DeploymentAttrs( deployment varchar(255) not null, name varchar(255) not null, @@ -343,7 +402,7 @@ def _create_schema(self, c): url = urlparse.urlparse(self.db_uri) if url.scheme == 'mysql': autoincrement = 'auto_increment' - c.execute( + c.session.execute( '''create table if not exists Resources( id integer primary key {}, deployment varchar(255) not null, @@ -352,11 +411,11 @@ def _create_schema(self, c): foreign key(deployment) references Deployments(uuid) on delete cascade );'''.format(autoincrement)) - c.execute( + c.session.execute( '''create table if not exists ResourceAttrs( machine integer not null, name varchar(255) not null, - value varchar(1024) not null, + value text not null, primary key(machine, name), foreign key(machine) references Resources(id) on delete cascade );''') @@ -367,5 +426,5 @@ def _upgrade_1_to_2(self, c): def _upgrade_2_to_3(self, c): sys.stderr.write("updating database schema from version 2 to 3...\n") - c.execute("alter table Machines rename to Resources") - c.execute("alter table MachineAttrs rename to ResourceAttrs") + c.session.execute("alter table Machines rename to Resources") + c.session.execute("alter table MachineAttrs rename to ResourceAttrs") diff --git a/nixops/util.py b/nixops/util.py index c5e22eceb..3ea7f5011 100644 --- a/nixops/util.py +++ b/nixops/util.py @@ -241,7 +241,9 @@ class Undefined: def attr_property(name, default, type=str): """Define a property that corresponds to a value in the NixOps state file.""" def get(self): + print 'getting attr {} {} {}'.format(name, default, type) s = self._get_attr(name, default) + print 'got s={}'.format(s) if s == undefined: if default != undefined: return copy.deepcopy(default) raise Exception("deployment attribute ‘{0}’ missing from state file".format(name)) @@ -252,6 +254,7 @@ def get(self): elif type is 'json': return json.loads(s) else: assert False def set(self, x): + print 'setting attr {} {} {}'.format(name, default, type) if x == default: self._del_attr(name) elif type is 'json': self._set_attr(name, json.dumps(x)) else: self._set_attr(name, x) From ceaf062f40a4bac94de6346a88e22b301e364015 Mon Sep 17 00:00:00 2001 From: Matthew O'Gorman Date: Mon, 12 Feb 2018 02:55:03 -0500 Subject: [PATCH 064/123] fix the past --- nixops/state/__init__.py | 11 +- nixops/state/sqlite_connector.py | 401 +++++++++++++++++++++++++++++++ 2 files changed, 407 insertions(+), 5 deletions(-) create mode 100644 nixops/state/sqlite_connector.py diff --git a/nixops/state/__init__.py b/nixops/state/__init__.py index b5313de05..ea811541b 100644 --- a/nixops/state/__init__.py +++ b/nixops/state/__init__.py @@ -3,18 +3,20 @@ import sys import json_file import sql_connector +import sqlite_connector class WrongStateSchemeException(Exception): pass def open(url): + print 'url = {}'.format(url) url_parsed = urlparse.urlparse(url) scheme = url_parsed.scheme ext = os.path.splitext(url)[1] if scheme == "": if ext == ".nixops": - scheme = "sql" - url = 'sqlite:///' + url + scheme = "sqlite" + url = 'sqlite://' + url elif ext == ".json": scheme = "json" @@ -23,10 +25,9 @@ def raise_(ex): switcher = { "json": lambda(url): json_file.JsonFile(url), - "sqlite": lambda(url): sql_connector.SQLConnection(url), "mysql": lambda(url): sql_connector.SQLConnection(url), - "sql": lambda(url): sql_connector.SQLConnection(url) + "sqlite": lambda(url): sqlite_connector.SQLiteConnection(url) } - function = switcher.get(scheme, lambda(url): raise_(WrongStateSchemeException("Unknown state scheme!"))) + function = switcher.get(scheme, lambda(url): raise_(WrongStateSchemeException("Unknown state scheme! {}".format(url)))) return function(url) diff --git a/nixops/state/sqlite_connector.py b/nixops/state/sqlite_connector.py new file mode 100644 index 000000000..16ac07114 --- /dev/null +++ b/nixops/state/sqlite_connector.py @@ -0,0 +1,401 @@ +# -*- coding: utf-8 -*- + + +import nixops.deployment +import os +import os.path +import urlparse +import sys +import threading +import fcntl +import sqlite3 + + +def _subclasses(cls): + sub = cls.__subclasses__() + return [cls] if not sub else [g for s in sub for g in _subclasses(s)] + + +class Connection(sqlite3.Connection): + + def __init__(self, db_file, **kwargs): + print 'a file {}'.format(db_file) + db_exists = os.path.exists(db_file) + if not db_exists: + print 'failed' + os.fdopen(os.open(db_file, os.O_WRONLY | os.O_CREAT, 0o600), 'w').close() + print 'opening' + sqlite3.Connection.__init__(self, db_file, **kwargs) + print 'double fail' + self.db_file = db_file + self.nesting = 0 + self.lock = threading.RLock() + + # Implement Python's context management protocol so that "with db" + # automatically commits or rolls back. The difference with the + # parent's "with" implementation is that we nest, i.e. a commit or + # rollback is only done at the outer "with". + def __enter__(self): + self.lock.acquire() + if self.nesting == 0: + self.must_rollback = False + self.nesting = self.nesting + 1 + sqlite3.Connection.__enter__(self) + + + def __exit__(self, exception_type, exception_value, exception_traceback): + if exception_type != None: self.must_rollback = True + self.nesting = self.nesting - 1 + assert self.nesting >= 0 + if self.nesting == 0: + if self.must_rollback: + try: + self.rollback() + except sqlite3.ProgrammingError: + pass + else: + sqlite3.Connection.__exit__(self, exception_type, exception_value, exception_traceback) + self.lock.release() + + +class SQLiteConnection(object): + """NixOps state file.""" + + current_schema = 3 + + def __init__(self, db_file): + print 'running {}'.format(db_file) + url = urlparse.urlparse(db_file) + self.db_file = url.netloc + url.path + print 'running {} {}'.format(self.db_file, urlparse.urlparse(db_file)) + if os.path.splitext(db_file)[1] not in ['.nixops', '.charon']: + raise Exception("state file ‘{0}’ should have extension ‘.nixops’".format(db_file)) + db = sqlite3.connect(self.db_file, timeout=60, check_same_thread=False, factory=Connection, isolation_level=None) # FIXME + db.db_file = db_file + + db.execute("pragma journal_mode = wal") + db.execute("pragma foreign_keys = 1") + + # FIXME: this is not actually transactional, because pysqlite (not + # sqlite) does an implicit commit before "create table". + with db: + c = db.cursor() + + # Get the schema version. + version = 0 # new database + if self._table_exists(c, 'SchemaVersion'): + c.execute("select version from SchemaVersion") + version = c.fetchone()[0] + elif self._table_exists(c, 'Deployments'): + version = 1 + + if version == self.current_schema: + pass + elif version == 0: + self._create_schema(c) + elif version < self.current_schema: + if version <= 1: self._upgrade_1_to_2(c) + if version <= 2: self._upgrade_2_to_3(c) + c.execute("update SchemaVersion set version = ?", (self.current_schema,)) + else: + raise Exception("this NixOps version is too old to deal with schema version {0}".format(version)) + + self.db = db + + def close(self): + self.db.close() + + def query_deployments(self): + """Return the UUIDs of all deployments in the database.""" + c = self.db.cursor() + c.execute("select uuid from Deployments") + res = c.fetchall() + return [x[0] for x in res] + + def get_all_deployments(self): + """Return Deployment objects for every deployment in the database.""" + uuids = self.query_deployments() + res = [] + for uuid in uuids: + try: + res.append(self.open_deployment(uuid=uuid)) + except nixops.deployment.UnknownBackend as e: + sys.stderr.write("skipping deployment ‘{0}’: {1}\n".format(uuid, str(e))) + return res + + def _find_deployment(self, uuid=None): + c = self.db.cursor() + if not uuid: + c.execute("select uuid from Deployments") + else: + c.execute("select uuid from Deployments d where uuid = ? or exists (select 1 from DeploymentAttrs where deployment = d.uuid and name = 'name' and value = ?)", (uuid, uuid)) + res = c.fetchall() + if len(res) == 0: + if uuid: + # try the prefix match + c.execute("select uuid from Deployments where uuid glob ?", (uuid + '*', )) + res = c.fetchall() + if len(res) == 0: + return None + else: + return None + if len(res) > 1: + if uuid: + raise Exception("state file contains multiple deployments with the same name, so you should specify one using its UUID") + else: + raise Exception("state file contains multiple deployments, so you should specify which one to use using ‘-d’, or set the environment variable NIXOPS_DEPLOYMENT") + return nixops.deployment.Deployment(self, res[0][0], sys.stderr) + + def open_deployment(self, uuid=None): + """Open an existing deployment.""" + deployment = self._find_deployment(uuid=uuid) + if deployment: return deployment + raise Exception("could not find specified deployment in state file ‘{0}’".format(self.db_file)) + + def create_deployment(self, uuid=None): + """Create a new deployment.""" + if not uuid: + import uuid + uuid = str(uuid.uuid1()) + with self.db: + self.db.execute("insert into Deployments(uuid) values (?)", (uuid,)) + return nixops.deployment.Deployment(self, uuid, sys.stderr) + + def _table_exists(self, c, table): + c.execute("select 1 from sqlite_master where name = ? and type='table'", (table,)); + return c.fetchone() != None + + def _delete_deployment(self, deployment_uuid): + """NOTE: This is UNSAFE, it's guarded in nixops/deployment.py. Do not call this function except from there!""" + with self.db: + self.db.execute("delete from Deployments where uuid = '{}'".format(deployment_uuid)) + + def clone_deployment(self, deployment_uuid): + with self.db: + new = self.create_deployment() + self.db.execute("insert into DeploymentAttrs (deployment, name, value) " + + "select '{}', name, value from DeploymentAttrs where deployment = '{}'" + .format(new.uuid, deployment_uuid) + ) + new.configs_path = None + return new + + + def _create_schemaversion(self, c): + c.execute( + '''create table if not exists SchemaVersion( + version integer not null + );''') + + c.execute("insert into SchemaVersion(version) values (?)", (self.current_schema,)) + + def _create_schema(self, c): + self._create_schemaversion(c) + + c.execute( + '''create table if not exists Deployments( + uuid text primary key + );''') + + c.execute( + '''create table if not exists DeploymentAttrs( + deployment text not null, + name text not null, + value text not null, + primary key(deployment, name), + foreign key(deployment) references Deployments(uuid) on delete cascade + );''') + + c.execute( + '''create table if not exists Resources( + id integer primary key autoincrement, + deployment text not null, + name text not null, + type text not null, + foreign key(deployment) references Deployments(uuid) on delete cascade + );''') + + c.execute( + '''create table if not exists ResourceAttrs( + machine integer not null, + name text not null, + value text not null, + primary key(machine, name), + foreign key(machine) references Resources(id) on delete cascade + );''') + + def _upgrade_1_to_2(self, c): + sys.stderr.write("updating database schema from version 1 to 2...\n") + self._create_schemaversion(c) + + def _upgrade_2_to_3(self, c): + sys.stderr.write("updating database schema from version 2 to 3...\n") + c.execute("alter table Machines rename to Resources") + c.execute("alter table MachineAttrs rename to ResourceAttrs") + +# ############################################################################################### +# ## Deployment + def get_resources_for(self, deployment): + """Get all the resources for a certain deployment""" + with self.db: + resources = {} + + rows = self.db.execute("select id, name, type from Resources where deployment = '{}'".format(deployment.uuid)).fetchall() + for (id, name, type) in rows: + r = self._create_state(deployment, type, name, id) + resources[name] = r + return resources + + + def set_deployment_attrs(self, deployment_uuid, attrs): + """Update deployment attributes in the state.""" + with self.db: + for name, value in attrs.iteritems(): + if value == None: + self.db.execute("delete from DeploymentAttrs where deployment = '{}' and name = '{}'" + .format(deployment_uuid, name) + ) + else: + if self.get_deployment_attr(deployment_uuid, name) == nixops.util.undefined: + self.db.execute("insert into DeploymentAttrs(deployment, name, value) values ('{}', '{}', {!r})" + .format(deployment_uuid, name, value) + ) + else: + self.db.execute("update DeploymentAttrs set value={!r} where deployment='{}' and name='{}'" + .format(value, deployment_uuid, name) + ) + + + def del_deployment_attr(self, deployment_uuid, attr_name): + with self.db: + self.db.execute("delete from DeploymentAttrs where deployment = '{}' and name = {!r}" + .format(deployment_uuid, attr_name) + ) + + + def get_deployment_attr(self, deployment_uuid, name): + """Get a deployment attribute from the state.""" + with self.db: + rows = self.db.execute("select value from DeploymentAttrs where deployment = '{}' and name = {!r}" + .format(deployment_uuid, name)).fetchall() + for row in rows: + return row[0] + return nixops.util.undefined + + + def get_all_deployment_attrs(self, deployment_uuid): + with self.db: + rows = self.db.execute("select name, value from DeploymentAttrs where deployment = '{}'" + .format(deployment_uuid)).fetchall() + res = {row[0]: row[1] for row in rows} + return res + + + def get_deployment_lock(self, deployment): + lock_dir = os.environ.get("HOME", "") + "/.nixops/locks" + if not os.path.exists(lock_dir): os.makedirs(lock_dir, 0700) + lock_file_path = lock_dir + "/" + deployment.uuid + class DeploymentLock(object): + def __init__(self, logger, path): + self._lock_file_path = path + self._logger = logger + self._lock_file = None + def __enter__(self): + self._lock_file = open(self._lock_file_path, "w") + fcntl.fcntl(self._lock_file, fcntl.F_SETFD, fcntl.FD_CLOEXEC) + try: + fcntl.flock(self._lock_file, fcntl.LOCK_EX | fcntl.LOCK_NB) + except IOError: + self._logger.log( + "waiting for exclusive deployment lock..." + ) + fcntl.flock(self._lock_file, fcntl.LOCK_EX) + def __exit__(self, exception_type, exception_value, exception_traceback): + if self._lock_file: + self._lock_file.close() + return DeploymentLock(deployment.logger, lock_file_path) + +# ############################################################################################### +# ## Resources + + def create_resource(self, deployment, name, type): + count = self.db.execute("select count(id) from Resources where deployment = '{}' and name = {!r}" + .format(deployment.uuid, name)).fetchone()[0] + + if count != 0: + raise Exception("resource already exists in database!") + + result = self.db.execute("insert into Resources(deployment, name, type) values ('{}', {!r}, {!r})" + .format(deployment.uuid, name, type)) + + id = result.lastrowid + r = self._create_state(deployment, type, name, id) + return r + + + def delete_resource(self, deployment_uuid, res_id): + with self.db: + self.db.execute("delete from Resources where deployment = '{}' and id = {!r}" + .format(deployment_uuid, res_id)) + + + def _rename_resource(self, deployment_uuid, resource_id, new_name): + """NOTE: Invariants are checked in nixops/deployment.py#rename""" + with self.db: + self.db.execute("update Resources set name = '{}' where deployment = '{}' and id = {!r}" + .format(new_name, deployment_uuid, resource_id)) + + + def set_resource_attrs(self, _deployment_uuid, resource_id, attrs): + with self.db: + for name, value in attrs.iteritems(): + if value == None: + self.db.execute("delete from ResourceAttrs where machine = '{}' and name = '{}'" + .format(resource_id, name)) + else: + if self.get_resource_attr(_deployment_uuid, resource_id, name) == nixops.util.undefined: + self.db.execute("insert into ResourceAttrs(machine, name, value) values ('{}', '{}', '{}')" + .format(resource_id, name, value) + ) + else: + self.db.execute("update ResourceAttrs set value={!r} where machine='{}' and name='{}'" + .format(value, resource_id, name) + ) + + + def del_resource_attr(self, _deployment_uuid, resource_id, name): + with self.db: + self.db.execute("delete from ResourceAttrs where machine = {!r} and name = {!r}" + .format(resource_id, name)) + + + def get_resource_attr(self, _deployment_uuid, resource_id, name): + with self.db: + """Get a machine attribute from the state file.""" + rows = self.db.execute("select value from ResourceAttrs where machine = '{}' and name = '{}'" + .format(resource_id, name)).fetchall() + for row in rows: + print 'here is the resource {}'.format(row[0]) + return row[0] + return nixops.util.undefined + + + def get_all_resource_attrs(self, deployment_uuid, resource_id): + with self.db: + rows = self.db.execute("select name, value from ResourceAttrs where machine = {!r}" + .format(resource_id)).fetchall() + res = {row[0]: row[1] for row in rows} + return res + + +# ### STATE + + def _create_state(self, depl, type, name, id): + """Create a resource state object of the desired type.""" + + for cls in _subclasses(nixops.resources.ResourceState): + if type == cls.get_type(): + return cls(depl, name, id) + + raise nixops.deployment.UnknownBackend("unknown resource type ‘{!r}’" + .format(type)) From 6f4f855963720226a70e4839fd74b480f1717652 Mon Sep 17 00:00:00 2001 From: Matthew O'Gorman Date: Mon, 12 Feb 2018 07:48:37 -0500 Subject: [PATCH 065/123] fixed unit tests --- tests/__init__.py | 7 ++++--- tests/functional/__init__.py | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/__init__.py b/tests/__init__.py index b586d2885..2c25dc760 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -3,14 +3,15 @@ import sys import threading from os import path -import nixops.state.sqlite3_file +import nixops.state _multiprocess_shared_ = True db_file = '%s/test.nixops' % (path.dirname(__file__)) def setup(): - nixops.state.sqlite3_file.StateFile(db_file).close() + state = nixops.state.open(db_file) + state.db.close() def destroy(sf, uuid): depl = sf.open_deployment(uuid) @@ -27,7 +28,7 @@ def destroy(sf, uuid): depl.logger.log("deployment ‘{0}’ destroyed".format(uuid)) def teardown(): - sf = nixops.state.sqlite3_file.StateFile(db_file) + sf = nixops.state.open(db_file) uuids = sf.query_deployments() threads = [] for uuid in uuids: diff --git a/tests/functional/__init__.py b/tests/functional/__init__.py index e721b2132..98f8b8015 100644 --- a/tests/functional/__init__.py +++ b/tests/functional/__init__.py @@ -1,6 +1,6 @@ import os from os import path -import nixops.state.sqlite3_file +import nixops.state from tests import db_file @@ -8,7 +8,7 @@ class DatabaseUsingTest(object): _multiprocess_can_split_ = True def setup(self): - self.sf = nixops.state.sqlite3_file.StateFile(db_file) + self.sf = nixops.state.open(db_file) def teardown(self): self.sf.close() From 28ac4b2357b0aa667631c77e45eb23d4984cfad6 Mon Sep 17 00:00:00 2001 From: Matthew O'Gorman Date: Tue, 13 Feb 2018 14:21:18 -0500 Subject: [PATCH 066/123] move function into the core state file --- nixops/state/__init__.py | 15 +++++++++++++++ scripts/nixops | 4 ++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/nixops/state/__init__.py b/nixops/state/__init__.py index ea811541b..51def7b47 100644 --- a/nixops/state/__init__.py +++ b/nixops/state/__init__.py @@ -8,6 +8,21 @@ class WrongStateSchemeException(Exception): pass + +def get_default_state_file(): + home = os.environ.get("HOME", "") + "/.nixops" + if not os.path.exists(home): + old_home = os.environ.get("HOME", "") + "/.charon" + if os.path.exists(old_home): + sys.stderr.write("renaming {!r} to {!r}...\n".format(old_home, home)) + os.rename(old_home, home) + if os.path.exists(home + "/deployments.charon"): + os.rename(home + "/deployments.charon", home + "/deployments.nixops") + else: + os.makedirs(home, 0700) + return os.environ.get("NIXOPS_STATE", os.environ.get("CHARON_STATE", home + "/deployments.nixops")) + + def open(url): print 'url = {}'.format(url) url_parsed = urlparse.urlparse(url) diff --git a/scripts/nixops b/scripts/nixops index 04ba3dbfb..90fdf170b 100755 --- a/scripts/nixops +++ b/scripts/nixops @@ -5,7 +5,7 @@ from nixops import deployment from nixops.nix_expr import py2nix from nixops.parallel import MultipleExceptions, run_tasks -import nixops.state.sql_connector +import nixops.state import prettytable import argparse import os @@ -724,7 +724,7 @@ subparsers = parser.add_subparsers(help='sub-command help') def add_subparser(name, help): subparser = subparsers.add_parser(name, help=help) subparser.add_argument('--state', '-s', dest='state_url', metavar='FILE', - default=os.environ.get("NIXOPS_STATE_URL", nixops.state.sql_connector.get_default_state_file()), help='URL that points to the state provider.') + default=os.environ.get("NIXOPS_STATE_URL", nixops.state.get_default_state_file()), help='URL that points to the state provider.') subparser.add_argument('--deployment', '-d', dest='deployment', metavar='UUID_OR_NAME', default=os.environ.get("NIXOPS_DEPLOYMENT", os.environ.get("CHARON_DEPLOYMENT", None)), help='UUID or symbolic name of the deployment') subparser.add_argument('--debug', action='store_true', help='enable debug output') From 759b96b7bea640006733f9e7bd89292b95da5011 Mon Sep 17 00:00:00 2001 From: Matthew O'Gorman Date: Wed, 14 Feb 2018 12:05:58 -0500 Subject: [PATCH 067/123] this simplifies to kv / sqlite for now so that we can move forward on this --- nixops/resources/__init__.py | 2 +- nixops/resources/aws_vpn_connection.py | 2 +- nixops/resources/aws_vpn_connection_route.py | 2 +- nixops/resources/aws_vpn_gateway.py | 2 +- nixops/resources/vpc.py | 2 +- nixops/resources/vpc_customer_gateway.py | 2 +- nixops/resources/vpc_dhcp_options.py | 2 +- .../vpc_egress_only_internet_gateway.py | 2 +- nixops/resources/vpc_endpoint.py | 2 +- nixops/resources/vpc_internet_gateway.py | 2 +- nixops/resources/vpc_nat_gateway.py | 2 +- nixops/resources/vpc_network_acl.py | 2 +- nixops/resources/vpc_network_interface.py | 2 +- .../vpc_network_interface_attachment.py | 2 +- nixops/resources/vpc_route.py | 2 +- nixops/resources/vpc_route_table.py | 2 +- .../resources/vpc_route_table_association.py | 2 +- nixops/state/__init__.py | 45 +- nixops/state/etcd.py | 0 nixops/state/sql_connector.py | 430 ------------------ nixops/state/state.py | 42 -- 21 files changed, 60 insertions(+), 491 deletions(-) delete mode 100644 nixops/state/etcd.py delete mode 100644 nixops/state/sql_connector.py delete mode 100644 nixops/state/state.py diff --git a/nixops/resources/__init__.py b/nixops/resources/__init__.py index 778e24ad3..804a1d9e4 100644 --- a/nixops/resources/__init__.py +++ b/nixops/resources/__init__.py @@ -3,7 +3,7 @@ import re import nixops.util -from nixops.state.state import StateDict +from nixops.state import StateDict from nixops.diff import Diff, Handler class ResourceDefinition(object): diff --git a/nixops/resources/aws_vpn_connection.py b/nixops/resources/aws_vpn_connection.py index 9080650a9..080b3be71 100644 --- a/nixops/resources/aws_vpn_connection.py +++ b/nixops/resources/aws_vpn_connection.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from nixops.state.state import StateDict +from nixops.state import StateDict from nixops.diff import Diff, Handler import nixops.util import nixops.resources diff --git a/nixops/resources/aws_vpn_connection_route.py b/nixops/resources/aws_vpn_connection_route.py index c6477c917..2da1631b0 100644 --- a/nixops/resources/aws_vpn_connection_route.py +++ b/nixops/resources/aws_vpn_connection_route.py @@ -7,7 +7,7 @@ from nixops.resources.ec2_common import EC2CommonState import nixops.ec2_utils from nixops.diff import Diff, Handler -from nixops.state.state import StateDict +from nixops.state import StateDict class AWSVPNConnectionRouteDefinition(nixops.resources.ResourceDefinition): """Definition of a VPN connection route""" diff --git a/nixops/resources/aws_vpn_gateway.py b/nixops/resources/aws_vpn_gateway.py index f41babee5..d97087878 100644 --- a/nixops/resources/aws_vpn_gateway.py +++ b/nixops/resources/aws_vpn_gateway.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from nixops.state.state import StateDict +from nixops.state import StateDict from nixops.diff import Diff, Handler import nixops.util import nixops.resources diff --git a/nixops/resources/vpc.py b/nixops/resources/vpc.py index 6cb3c5d75..28d1344aa 100755 --- a/nixops/resources/vpc.py +++ b/nixops/resources/vpc.py @@ -9,7 +9,7 @@ import nixops.resources from nixops.resources.ec2_common import EC2CommonState import nixops.ec2_utils -from nixops.state.state import StateDict +from nixops.state import StateDict from nixops.diff import Diff, Handler class VPCDefinition(nixops.resources.ResourceDefinition): diff --git a/nixops/resources/vpc_customer_gateway.py b/nixops/resources/vpc_customer_gateway.py index 9bc324058..129d7d846 100644 --- a/nixops/resources/vpc_customer_gateway.py +++ b/nixops/resources/vpc_customer_gateway.py @@ -7,7 +7,7 @@ import boto3 import botocore -from nixops.state.state import StateDict +from nixops.state import StateDict from nixops.diff import Diff, Handler import nixops.util import nixops.resources diff --git a/nixops/resources/vpc_dhcp_options.py b/nixops/resources/vpc_dhcp_options.py index 43000e8ad..114692687 100644 --- a/nixops/resources/vpc_dhcp_options.py +++ b/nixops/resources/vpc_dhcp_options.py @@ -12,7 +12,7 @@ import nixops.resources from nixops.resources.ec2_common import EC2CommonState import nixops.ec2_utils -from nixops.state.state import StateDict +from nixops.state import StateDict from nixops.diff import Diff, Handler class VPCDhcpOptionsDefinition(nixops.resources.ResourceDefinition): diff --git a/nixops/resources/vpc_egress_only_internet_gateway.py b/nixops/resources/vpc_egress_only_internet_gateway.py index 7abcd5dfa..d76791f60 100644 --- a/nixops/resources/vpc_egress_only_internet_gateway.py +++ b/nixops/resources/vpc_egress_only_internet_gateway.py @@ -3,7 +3,7 @@ import boto3 import botocore -from nixops.state.state import StateDict +from nixops.state import StateDict from nixops.diff import Diff, Handler import nixops.util import nixops.resources diff --git a/nixops/resources/vpc_endpoint.py b/nixops/resources/vpc_endpoint.py index dd2070da4..55671dbb6 100644 --- a/nixops/resources/vpc_endpoint.py +++ b/nixops/resources/vpc_endpoint.py @@ -2,7 +2,7 @@ import uuid -from nixops.state.state import StateDict +from nixops.state import StateDict from nixops.diff import Diff, Handler import nixops.util import nixops.resources diff --git a/nixops/resources/vpc_internet_gateway.py b/nixops/resources/vpc_internet_gateway.py index e260af14d..4a3045b00 100644 --- a/nixops/resources/vpc_internet_gateway.py +++ b/nixops/resources/vpc_internet_gateway.py @@ -7,7 +7,7 @@ import boto3 import botocore -from nixops.state.state import StateDict +from nixops.state import StateDict from nixops.diff import Diff, Handler import nixops.util import nixops.resources diff --git a/nixops/resources/vpc_nat_gateway.py b/nixops/resources/vpc_nat_gateway.py index c32c41dea..7295e4e1d 100644 --- a/nixops/resources/vpc_nat_gateway.py +++ b/nixops/resources/vpc_nat_gateway.py @@ -13,7 +13,7 @@ from nixops.resources.ec2_common import EC2CommonState import nixops.ec2_utils from nixops.diff import Diff, Handler -from nixops.state.state import StateDict +from nixops.state import StateDict class VPCNatGatewayDefinition(nixops.resources.ResourceDefinition): """Definition of a VPC NAT gateway""" diff --git a/nixops/resources/vpc_network_acl.py b/nixops/resources/vpc_network_acl.py index 39ce50328..6a5cf1b72 100644 --- a/nixops/resources/vpc_network_acl.py +++ b/nixops/resources/vpc_network_acl.py @@ -9,7 +9,7 @@ from nixops.resources.ec2_common import EC2CommonState import nixops.ec2_utils from nixops.diff import Diff, Handler -from nixops.state.state import StateDict +from nixops.state import StateDict class VPCNetworkAcldefinition(nixops.resources.ResourceDefinition): """definition of a vpc network ACL.""" diff --git a/nixops/resources/vpc_network_interface.py b/nixops/resources/vpc_network_interface.py index 84bf70a3a..8797e4a9b 100644 --- a/nixops/resources/vpc_network_interface.py +++ b/nixops/resources/vpc_network_interface.py @@ -10,7 +10,7 @@ from nixops.resources.ec2_common import EC2CommonState import nixops.ec2_utils from nixops.diff import Diff, Handler -from nixops.state.state import StateDict +from nixops.state import StateDict class VPCNetworkInterfaceDefinition(nixops.resources.ResourceDefinition): """Definition of a VPC network interface""" diff --git a/nixops/resources/vpc_network_interface_attachment.py b/nixops/resources/vpc_network_interface_attachment.py index 3ff610e76..cb22b924f 100644 --- a/nixops/resources/vpc_network_interface_attachment.py +++ b/nixops/resources/vpc_network_interface_attachment.py @@ -12,7 +12,7 @@ from nixops.resources.ec2_common import EC2CommonState import nixops.ec2_utils from nixops.diff import Diff, Handler -from nixops.state.state import StateDict +from nixops.state import StateDict class VPCNetworkInterfaceAttachmentDefinition(nixops.resources.ResourceDefinition): """Definition of a VPC network interface attachment""" diff --git a/nixops/resources/vpc_route.py b/nixops/resources/vpc_route.py index 3e7e686ba..2bb88a014 100644 --- a/nixops/resources/vpc_route.py +++ b/nixops/resources/vpc_route.py @@ -10,7 +10,7 @@ from nixops.resources.ec2_common import EC2CommonState import nixops.ec2_utils from nixops.diff import Diff, Handler -from nixops.state.state import StateDict +from nixops.state import StateDict class VPCRouteDefinition(nixops.resources.ResourceDefinition): """Definition of a VPC route""" diff --git a/nixops/resources/vpc_route_table.py b/nixops/resources/vpc_route_table.py index 504e0cc03..a0fd0c123 100644 --- a/nixops/resources/vpc_route_table.py +++ b/nixops/resources/vpc_route_table.py @@ -10,7 +10,7 @@ from nixops.resources.ec2_common import EC2CommonState import nixops.ec2_utils from nixops.diff import Diff, Handler -from nixops.state.state import StateDict +from nixops.state import StateDict class VPCRouteTableDefinition(nixops.resources.ResourceDefinition): """Definition of a VPC route table""" diff --git a/nixops/resources/vpc_route_table_association.py b/nixops/resources/vpc_route_table_association.py index b1cc3d6c7..e6d38de63 100644 --- a/nixops/resources/vpc_route_table_association.py +++ b/nixops/resources/vpc_route_table_association.py @@ -10,7 +10,7 @@ from nixops.resources.ec2_common import EC2CommonState import nixops.ec2_utils from nixops.diff import Diff, Handler -from nixops.state.state import StateDict +from nixops.state import StateDict class VPCRouteTableAssociationDefinition(nixops.resources.ResourceDefinition): """Definition of a VPC route table association""" diff --git a/nixops/state/__init__.py b/nixops/state/__init__.py index 51def7b47..38de8f823 100644 --- a/nixops/state/__init__.py +++ b/nixops/state/__init__.py @@ -1,10 +1,13 @@ +import json +import collections import os import urlparse import sys import json_file -import sql_connector import sqlite_connector +import nixops.util + class WrongStateSchemeException(Exception): pass @@ -40,9 +43,47 @@ def raise_(ex): switcher = { "json": lambda(url): json_file.JsonFile(url), - "mysql": lambda(url): sql_connector.SQLConnection(url), "sqlite": lambda(url): sqlite_connector.SQLiteConnection(url) } function = switcher.get(scheme, lambda(url): raise_(WrongStateSchemeException("Unknown state scheme! {}".format(url)))) return function(url) + + +class StateDict(collections.MutableMapping): + """ + An implementation of a MutableMapping container providing + a python dict like behavior for the NixOps state file. + """ + # TODO implement __repr__ for convenience e.g debuging the structure + def __init__(self, depl, id): + super(StateDict, self).__init__() + self._state = depl._state + self.uuid = depl.uuid + self.id = id + + def __setitem__(self, key, value): + self._state.set_resource_attrs(self.uuid, self.id, {key:value}) + + def __getitem__(self, key): + value = self._state.get_resource_attr(self.uuid, self.id, name) + try: + return json.loads(value) + except ValueError: + return value + raise KeyError("couldn't find key {} in the state file".format(key)) + + def __delitem__(self, key): + self._state.del_resource_attr(self.uuid, self.id, key) + + def keys(self): + # Generally the list of keys per ResourceAttrs is relatively small + # so this should be also relatively fast. + attrs = self._state.get_all_resource_attrs(self.uuid, self.id) + return [key for key,value in attrs.iteritems()] + + def __iter__(self): + return iter(self.keys()) + + def __len__(self): + return len(self.keys()) diff --git a/nixops/state/etcd.py b/nixops/state/etcd.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/nixops/state/sql_connector.py b/nixops/state/sql_connector.py deleted file mode 100644 index 1669f98da..000000000 --- a/nixops/state/sql_connector.py +++ /dev/null @@ -1,430 +0,0 @@ -# -*- coding: utf-8 -*- - -import nixops.deployment -import os -import os.path -import urlparse -import sys -import threading -import fcntl - -from sqlalchemy import create_engine -from sqlalchemy.orm import sessionmaker - -import pymysql_sa -pymysql_sa.make_default_mysql_dialect() - -def _subclasses(cls): - sub = cls.__subclasses__() - return [cls] if not sub else [g for s in sub for g in _subclasses(s)] - - -def get_default_state_file(): - home = os.environ.get("HOME", "") + "/.nixops" - if not os.path.exists(home): - old_home = os.environ.get("HOME", "") + "/.charon" - if os.path.exists(old_home): - sys.stderr.write("renaming {!r} to {!r}...\n".format(old_home, home)) - os.rename(old_home, home) - if os.path.exists(home + "/deployments.charon"): - os.rename(home + "/deployments.charon", home + "/deployments.nixops") - else: - os.makedirs(home, 0700) - return os.environ.get("NIXOPS_STATE", os.environ.get("CHARON_STATE", home + "/deployments.nixops")) - - -class SQLAlchemyDBConnection(object): - """SQLAlchemy database connection""" - - def __init__(self, connection_string): - print 'init' - self.connection_string = connection_string - self.session = None - self.nesting = 0 - self.lock = threading.RLock() - - def __enter__(self): - print 'enter' - self.lock.acquire() - if self.nesting == 0: - self.must_rollback = False - self.nesting = self.nesting + 1 - engine = create_engine(self.connection_string, echo = True) - Session = sessionmaker() - self.session = Session(bind=engine) - self.engine = engine - return self - - def __exit__(self, exception_type, excption_value, exception_traceback): - print 'exit' - if exception_type != None: - self.must_rollback = True - self.nesting = self.nesting - 1 - assert self.nesting >= 0 - if self.nesting == 0: - print 'not nested' - if self.must_rollback: - print 'MUST ROLL BACK' - try: - self.session.rollback() - except: - pass - else: - print 'closing' - self.session.flush() - self.session.commit() - self.session.close() - self.lock.release() - - -class SQLConnection(object): - """NixOps db uri.""" - - current_schema = 3 - - def __init__(self, db_uri): - url = urlparse.urlparse(db_uri) - self.db_uri = db_uri - - db = SQLAlchemyDBConnection(db_uri) - with db: - if url.scheme == "sqlite": - db.session.execute("pragma journal_mode = wal;") - db.session.execute("pragma foreign_keys;") - with db: - version = 0 # new database - if db.engine.dialect.has_table(db.session, 'SchemaVersion'): - version = db.session.execute("select version from SchemaVersion").scalar() - elif db.engine.dialect.has_table(db.session, 'Deployments'): - version = 1 - - if version == self.current_schema: - pass - elif version == 0: - self._create_schema(db) - elif version < self.current_schema: - if version <= 1: - self._upgrade_1_to_2(db) - if version <= 2: - self._upgrade_2_to_3(db) - db.session.execute("update SchemaVersion set version = :current_schema", {'current_schema': self.current_schema}) - else: - raise Exception("this NixOps version is too old to deal with schema version {}".format(version)) - - self.db = db - - def close(self): - self.db.close() - - ############################################################################################### - ## Deployment - - def query_deployments(self): - """Return the UUIDs of all deployments in the database.""" - rows = self.db.session.execute("select uuid from Deployments") - return [x[0] for x in rows] - - - def get_all_deployments(self): - """Return Deployment objects for every deployment in the database.""" - uuids = self.query_deployments() - res = [] - for uuid in uuids: - try: - res.append(self.open_deployment(uuid=uuid)) - except nixops.deployment.UnknownBackend as e: - sys.stderr.write("skipping deployment '{}': {}\n".format(uuid, str(e))) - - return res - - - def _find_deployment(self, uuid=None): - if not uuid: - rows = self.db.session.execute("select count(uuid), uuid from Deployments") - else: - rows = self.db.session.execute("select count(uuid), uuid from Deployments d where uuid = :uuid or exists (select 1 from DeploymentAttrs where deployment = d.uuid and name = 'name' and value = :value)", {'uuid': uuid, 'value': uuid}) - row_count = 0 - deployment = None - for row in rows: - row_count = row[0] - deployment = row[1] - break - - if row_count == 0: - if uuid: - # try the prefix match - rows = self.db.session.execute("select count(uuid), uuid from Deployments where uuid glob :uuid".format(uuid + '*')) - for row in rows: - row_count = row[0] - deployment = row[1] - break - - if row_count == 0: - return None - else: - return None - - if row_count > 1: - if uuid: - raise Exception("state file contains multiple deployments with the same name, so you should specify one using its UUID") - else: - raise Exception("state file contains multiple deployments, so you should specify which one to use using ‘-d’, or set the environment variable NIXOPS_DEPLOYMENT") - return nixops.deployment.Deployment(self, deployment, sys.stderr) - - - def open_deployment(self, uuid=None): - """Open an existing deployment.""" - deployment = self._find_deployment(uuid=uuid) - - if deployment: return deployment - raise Exception("could not find specified deployment in state file {!r}".format(self.db_uri)) - - - def create_deployment(self, uuid=None): - """Create a new deployment.""" - if not uuid: - import uuid - uuid = str(uuid.uuid1()) - with self.db: - self.db.session.execute("insert into Deployments(uuid) values ('{}')".format(uuid)) - return nixops.deployment.Deployment(self, uuid, sys.stderr) - - - def _delete_deployment(self, deployment_uuid): - """NOTE: This is UNSAFE, it's guarded in nixops/deployment.py. Do not call this function except from there!""" - self.db.session.execute("delete from Deployments where uuid = '{}'".format(deployment_uuid)) - - - def clone_deployment(self, deployment_uuid): - with self.db: - new = self.create_deployment() - self.db.session.execute("insert into DeploymentAttrs (deployment, name, value) " + - "select '{}', name, value from DeploymentAttrs where deployment = '{}'" - .format(new.uuid, deployment_uuid) - ) - new.configs_path = None - return new - - - def get_resources_for(self, deployment): - """Get all the resources for a certain deployment""" - with self.db: - resources = {} - - rows = self.db.session.execute("select id, name, type from Resources where deployment = '{}'".format(deployment.uuid)) - for (id, name, type) in rows: - r = self._create_state(deployment, type, name, id) - resources[name] = r - return resources - - - def set_deployment_attrs(self, deployment_uuid, attrs): - """Update deployment attributes in the state.""" - with self.db: - for name, value in attrs.iteritems(): - if value == None: - self.db.session.execute("delete from DeploymentAttrs where deployment = '{}' and name = '{}'" - .format(deployment_uuid, name) - ) - else: - if self.get_deployment_attr(deployment_uuid, name) == nixops.util.undefined: - self.db.session.execute("insert into DeploymentAttrs(deployment, name, value) values ('{}', '{}', {!r})" - .format(deployment_uuid, name, value) - ) - else: - self.db.session.execute("update DeploymentAttrs set value={!r} where deployment='{}' and name='{}'" - .format(value, deployment_uuid, name) - ) - - - def del_deployment_attr(self, deployment_uuid, attr_name): - with self.db: - self.db.session.execute("delete from DeploymentAttrs where deployment = '{}' and name = {!r}" - .format(deployment_uuid, attr_name) - ) - - - def get_deployment_attr(self, deployment_uuid, name): - """Get a deployment attribute from the state.""" - with self.db: - rows = self.db.session.execute("select value from DeploymentAttrs where deployment = '{}' and name = {!r}" - .format(deployment_uuid, name)) - for row in rows: - print 'here is the deployment {}'.format(row[0]) - return row[0] - return nixops.util.undefined - - - def get_all_deployment_attrs(self, deployment_uuid): - with self.db: - rows = self.db.session.execute("select name, value from DeploymentAttrs where deployment = '{}'" - .format(deployment_uuid)) - res = {row[0]: row[1] for row in rows} - return res - - - def get_deployment_lock(self, deployment): - lock_dir = os.environ.get("HOME", "") + "/.nixops/locks" - if not os.path.exists(lock_dir): os.makedirs(lock_dir, 0700) - lock_file_path = lock_dir + "/" + deployment.uuid - class DeploymentLock(object): - def __init__(self, logger, path): - self._lock_file_path = path - self._logger = logger - self._lock_file = None - def __enter__(self): - self._lock_file = open(self._lock_file_path, "w") - fcntl.fcntl(self._lock_file, fcntl.F_SETFD, fcntl.FD_CLOEXEC) - try: - fcntl.flock(self._lock_file, fcntl.LOCK_EX | fcntl.LOCK_NB) - except IOError: - self._logger.log( - "waiting for exclusive deployment lock..." - ) - fcntl.flock(self._lock_file, fcntl.LOCK_EX) - def __exit__(self, exception_type, exception_value, exception_traceback): - if self._lock_file: - self._lock_file.close() - return DeploymentLock(deployment.logger, lock_file_path) - - ############################################################################################### - ## Resources - - def create_resource(self, deployment, name, type): - count = self.db.session.execute("select count(id) from Resources where deployment = '{}' and name = {!r}" - .format(deployment.uuid, name)).scalar() - - if count != 0: - raise Exception("resource already exists in database!") - - result = self.db.session.execute("insert into Resources(deployment, name, type) values ('{}', {!r}, {!r})" - .format(deployment.uuid, name, type)) - - id = result.lastrowid - r = self._create_state(deployment, type, name, id) - return r - - - def delete_resource(self, deployment_uuid, res_id): - with self.db: - self.db.session.execute("delete from Resources where deployment = '{}' and id = {!r}" - .format(deployment_uuid, res_id)) - - - def _rename_resource(self, deployment_uuid, resource_id, new_name): - """NOTE: Invariants are checked in nixops/deployment.py#rename""" - with self.db: - self.db.session.execute("update Resources set name = '{}' where deployment = '{}' and id = {!r}" - .format(new_name, deployment_uuid, resource_id)) - - - def set_resource_attrs(self, _deployment_uuid, resource_id, attrs): - with self.db: - for name, value in attrs.iteritems(): - if value == None: - self.db.session.execute("delete from ResourceAttrs where machine = '{}' and name = '{}'" - .format(resource_id, name)) - else: - if self.get_resource_attr(_deployment_uuid, resource_id, name) == nixops.util.undefined: - self.db.session.execute("insert into ResourceAttrs(machine, name, value) values ('{}', '{}', '{}')" - .format(resource_id, name, value) - ) - else: - self.db.session.execute("update ResourceAttrs set value={!r} where machine='{}' and name='{}'" - .format(value, resource_id, name) - ) - - - def del_resource_attr(self, _deployment_uuid, resource_id, name): - with self.db: - self.db.session.execute("delete from ResourceAttrs where machine = {!r} and name = {!r}" - .format(resource_id, name)) - - - def get_resource_attr(self, _deployment_uuid, resource_id, name): - with self.db: - """Get a machine attribute from the state file.""" - rows = self.db.session.execute("select value from ResourceAttrs where machine = '{}' and name = '{}'" - .format(resource_id, name)) - for row in rows: - print 'here is the resource {}'.format(row[0]) - return row[0] - return nixops.util.undefined - - - def get_all_resource_attrs(self, deployment_uuid, resource_id): - with self.db: - rows = self.db.session.execute("select name, value from ResourceAttrs where machine = {!r}" - .format(resource_id)) - res = {row[0]: row[1] for row in rows} - return res - - - ### STATE - def _create_state(self, depl, type, name, id): - """Create a resource state object of the desired type.""" - - for cls in _subclasses(nixops.resources.ResourceState): - if type == cls.get_type(): - return cls(depl, name, id) - - raise nixops.deployment.UnknownBackend("unknown resource type ‘{!r}’" - .format(type)) - - - def _create_schemaversion(self, c): - c.session.execute( - '''create table if not exists SchemaVersion( - version integer not null - );''') - - c.session.execute("insert into SchemaVersion(version) values ({!r})" - .format(self.current_schema)) - - def _create_schema(self, c): - self._create_schemaversion(c) - - c.session.execute( - '''create table if not exists Deployments( - uuid VARCHAR(36) primary key - );''') - - c.session.execute( - '''create table if not exists DeploymentAttrs( - deployment varchar(255) not null, - name varchar(255) not null, - value text not null, - primary key(deployment, name), - foreign key(deployment) references Deployments(uuid) on delete cascade - );''') - - autoincrement = 'autoincrement' - url = urlparse.urlparse(self.db_uri) - if url.scheme == 'mysql': - autoincrement = 'auto_increment' - c.session.execute( - '''create table if not exists Resources( - id integer primary key {}, - deployment varchar(255) not null, - name varchar(255) not null, - type varchar(1024) not null, - foreign key(deployment) references Deployments(uuid) on delete cascade - );'''.format(autoincrement)) - - c.session.execute( - '''create table if not exists ResourceAttrs( - machine integer not null, - name varchar(255) not null, - value text not null, - primary key(machine, name), - foreign key(machine) references Resources(id) on delete cascade - );''') - - def _upgrade_1_to_2(self, c): - sys.stderr.write("updating database schema from version 1 to 2...\n") - self._create_schemaversion(c) - - def _upgrade_2_to_3(self, c): - sys.stderr.write("updating database schema from version 2 to 3...\n") - c.session.execute("alter table Machines rename to Resources") - c.session.execute("alter table MachineAttrs rename to ResourceAttrs") diff --git a/nixops/state/state.py b/nixops/state/state.py deleted file mode 100644 index e124806fd..000000000 --- a/nixops/state/state.py +++ /dev/null @@ -1,42 +0,0 @@ -import json -import collections - -import nixops.util - -class StateDict(collections.MutableMapping): - """ - An implementation of a MutableMapping container providing - a python dict like behavior for the NixOps state file. - """ - # TODO implement __repr__ for convenience e.g debuging the structure - def __init__(self, depl, id): - super(StateDict, self).__init__() - self._state = depl._state - self.uuid = depl.uuid - self.id = id - - def __setitem__(self, key, value): - self._state.set_resource_attrs(self.uuid, self.id, {key:value}) - - def __getitem__(self, key): - value = self._state.get_resource_attr(self.uuid, self.id, name) - try: - return json.loads(value) - except ValueError: - return value - raise KeyError("couldn't find key {} in the state file".format(key)) - - def __delitem__(self, key): - self._state.del_resource_attr(self.uuid, self.id, key) - - def keys(self): - # Generally the list of keys per ResourceAttrs is relatively small - # so this should be also relatively fast. - attrs = self._state.get_all_resource_attrs(self.uuid, self.id) - return [key for key,value in attrs.iteritems()] - - def __iter__(self): - return iter(self.keys()) - - def __len__(self): - return len(self.keys()) From cc70b2a5b622a3deb7bd7a0eb6511197a6385c7c Mon Sep 17 00:00:00 2001 From: Matthew O'Gorman Date: Wed, 14 Feb 2018 13:17:48 -0500 Subject: [PATCH 068/123] testing this theory --- nixops/resources/__init__.py | 2 +- nixops/resources/aws_vpn_connection.py | 2 +- nixops/resources/aws_vpn_connection_route.py | 2 +- nixops/resources/aws_vpn_gateway.py | 2 +- nixops/resources/vpc.py | 2 +- nixops/resources/vpc_customer_gateway.py | 2 +- nixops/resources/vpc_dhcp_options.py | 2 +- .../vpc_egress_only_internet_gateway.py | 2 +- nixops/resources/vpc_endpoint.py | 2 +- nixops/resources/vpc_internet_gateway.py | 2 +- nixops/resources/vpc_nat_gateway.py | 2 +- nixops/resources/vpc_network_acl.py | 2 +- nixops/resources/vpc_network_interface.py | 2 +- .../vpc_network_interface_attachment.py | 2 +- nixops/resources/vpc_route.py | 2 +- nixops/resources/vpc_route_table.py | 2 +- .../resources/vpc_route_table_association.py | 2 +- nixops/state/__init__.py | 43 ------------------- nixops/state/state.py | 42 ++++++++++++++++++ release.nix | 3 -- 20 files changed, 59 insertions(+), 63 deletions(-) create mode 100644 nixops/state/state.py diff --git a/nixops/resources/__init__.py b/nixops/resources/__init__.py index 804a1d9e4..778e24ad3 100644 --- a/nixops/resources/__init__.py +++ b/nixops/resources/__init__.py @@ -3,7 +3,7 @@ import re import nixops.util -from nixops.state import StateDict +from nixops.state.state import StateDict from nixops.diff import Diff, Handler class ResourceDefinition(object): diff --git a/nixops/resources/aws_vpn_connection.py b/nixops/resources/aws_vpn_connection.py index 080b3be71..9080650a9 100644 --- a/nixops/resources/aws_vpn_connection.py +++ b/nixops/resources/aws_vpn_connection.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from nixops.state import StateDict +from nixops.state.state import StateDict from nixops.diff import Diff, Handler import nixops.util import nixops.resources diff --git a/nixops/resources/aws_vpn_connection_route.py b/nixops/resources/aws_vpn_connection_route.py index 2da1631b0..c6477c917 100644 --- a/nixops/resources/aws_vpn_connection_route.py +++ b/nixops/resources/aws_vpn_connection_route.py @@ -7,7 +7,7 @@ from nixops.resources.ec2_common import EC2CommonState import nixops.ec2_utils from nixops.diff import Diff, Handler -from nixops.state import StateDict +from nixops.state.state import StateDict class AWSVPNConnectionRouteDefinition(nixops.resources.ResourceDefinition): """Definition of a VPN connection route""" diff --git a/nixops/resources/aws_vpn_gateway.py b/nixops/resources/aws_vpn_gateway.py index d97087878..f41babee5 100644 --- a/nixops/resources/aws_vpn_gateway.py +++ b/nixops/resources/aws_vpn_gateway.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from nixops.state import StateDict +from nixops.state.state import StateDict from nixops.diff import Diff, Handler import nixops.util import nixops.resources diff --git a/nixops/resources/vpc.py b/nixops/resources/vpc.py index 28d1344aa..6cb3c5d75 100755 --- a/nixops/resources/vpc.py +++ b/nixops/resources/vpc.py @@ -9,7 +9,7 @@ import nixops.resources from nixops.resources.ec2_common import EC2CommonState import nixops.ec2_utils -from nixops.state import StateDict +from nixops.state.state import StateDict from nixops.diff import Diff, Handler class VPCDefinition(nixops.resources.ResourceDefinition): diff --git a/nixops/resources/vpc_customer_gateway.py b/nixops/resources/vpc_customer_gateway.py index 129d7d846..9bc324058 100644 --- a/nixops/resources/vpc_customer_gateway.py +++ b/nixops/resources/vpc_customer_gateway.py @@ -7,7 +7,7 @@ import boto3 import botocore -from nixops.state import StateDict +from nixops.state.state import StateDict from nixops.diff import Diff, Handler import nixops.util import nixops.resources diff --git a/nixops/resources/vpc_dhcp_options.py b/nixops/resources/vpc_dhcp_options.py index 114692687..43000e8ad 100644 --- a/nixops/resources/vpc_dhcp_options.py +++ b/nixops/resources/vpc_dhcp_options.py @@ -12,7 +12,7 @@ import nixops.resources from nixops.resources.ec2_common import EC2CommonState import nixops.ec2_utils -from nixops.state import StateDict +from nixops.state.state import StateDict from nixops.diff import Diff, Handler class VPCDhcpOptionsDefinition(nixops.resources.ResourceDefinition): diff --git a/nixops/resources/vpc_egress_only_internet_gateway.py b/nixops/resources/vpc_egress_only_internet_gateway.py index d76791f60..7abcd5dfa 100644 --- a/nixops/resources/vpc_egress_only_internet_gateway.py +++ b/nixops/resources/vpc_egress_only_internet_gateway.py @@ -3,7 +3,7 @@ import boto3 import botocore -from nixops.state import StateDict +from nixops.state.state import StateDict from nixops.diff import Diff, Handler import nixops.util import nixops.resources diff --git a/nixops/resources/vpc_endpoint.py b/nixops/resources/vpc_endpoint.py index 55671dbb6..dd2070da4 100644 --- a/nixops/resources/vpc_endpoint.py +++ b/nixops/resources/vpc_endpoint.py @@ -2,7 +2,7 @@ import uuid -from nixops.state import StateDict +from nixops.state.state import StateDict from nixops.diff import Diff, Handler import nixops.util import nixops.resources diff --git a/nixops/resources/vpc_internet_gateway.py b/nixops/resources/vpc_internet_gateway.py index 4a3045b00..e260af14d 100644 --- a/nixops/resources/vpc_internet_gateway.py +++ b/nixops/resources/vpc_internet_gateway.py @@ -7,7 +7,7 @@ import boto3 import botocore -from nixops.state import StateDict +from nixops.state.state import StateDict from nixops.diff import Diff, Handler import nixops.util import nixops.resources diff --git a/nixops/resources/vpc_nat_gateway.py b/nixops/resources/vpc_nat_gateway.py index 7295e4e1d..c32c41dea 100644 --- a/nixops/resources/vpc_nat_gateway.py +++ b/nixops/resources/vpc_nat_gateway.py @@ -13,7 +13,7 @@ from nixops.resources.ec2_common import EC2CommonState import nixops.ec2_utils from nixops.diff import Diff, Handler -from nixops.state import StateDict +from nixops.state.state import StateDict class VPCNatGatewayDefinition(nixops.resources.ResourceDefinition): """Definition of a VPC NAT gateway""" diff --git a/nixops/resources/vpc_network_acl.py b/nixops/resources/vpc_network_acl.py index 6a5cf1b72..39ce50328 100644 --- a/nixops/resources/vpc_network_acl.py +++ b/nixops/resources/vpc_network_acl.py @@ -9,7 +9,7 @@ from nixops.resources.ec2_common import EC2CommonState import nixops.ec2_utils from nixops.diff import Diff, Handler -from nixops.state import StateDict +from nixops.state.state import StateDict class VPCNetworkAcldefinition(nixops.resources.ResourceDefinition): """definition of a vpc network ACL.""" diff --git a/nixops/resources/vpc_network_interface.py b/nixops/resources/vpc_network_interface.py index 8797e4a9b..84bf70a3a 100644 --- a/nixops/resources/vpc_network_interface.py +++ b/nixops/resources/vpc_network_interface.py @@ -10,7 +10,7 @@ from nixops.resources.ec2_common import EC2CommonState import nixops.ec2_utils from nixops.diff import Diff, Handler -from nixops.state import StateDict +from nixops.state.state import StateDict class VPCNetworkInterfaceDefinition(nixops.resources.ResourceDefinition): """Definition of a VPC network interface""" diff --git a/nixops/resources/vpc_network_interface_attachment.py b/nixops/resources/vpc_network_interface_attachment.py index cb22b924f..3ff610e76 100644 --- a/nixops/resources/vpc_network_interface_attachment.py +++ b/nixops/resources/vpc_network_interface_attachment.py @@ -12,7 +12,7 @@ from nixops.resources.ec2_common import EC2CommonState import nixops.ec2_utils from nixops.diff import Diff, Handler -from nixops.state import StateDict +from nixops.state.state import StateDict class VPCNetworkInterfaceAttachmentDefinition(nixops.resources.ResourceDefinition): """Definition of a VPC network interface attachment""" diff --git a/nixops/resources/vpc_route.py b/nixops/resources/vpc_route.py index 2bb88a014..3e7e686ba 100644 --- a/nixops/resources/vpc_route.py +++ b/nixops/resources/vpc_route.py @@ -10,7 +10,7 @@ from nixops.resources.ec2_common import EC2CommonState import nixops.ec2_utils from nixops.diff import Diff, Handler -from nixops.state import StateDict +from nixops.state.state import StateDict class VPCRouteDefinition(nixops.resources.ResourceDefinition): """Definition of a VPC route""" diff --git a/nixops/resources/vpc_route_table.py b/nixops/resources/vpc_route_table.py index a0fd0c123..504e0cc03 100644 --- a/nixops/resources/vpc_route_table.py +++ b/nixops/resources/vpc_route_table.py @@ -10,7 +10,7 @@ from nixops.resources.ec2_common import EC2CommonState import nixops.ec2_utils from nixops.diff import Diff, Handler -from nixops.state import StateDict +from nixops.state.state import StateDict class VPCRouteTableDefinition(nixops.resources.ResourceDefinition): """Definition of a VPC route table""" diff --git a/nixops/resources/vpc_route_table_association.py b/nixops/resources/vpc_route_table_association.py index e6d38de63..b1cc3d6c7 100644 --- a/nixops/resources/vpc_route_table_association.py +++ b/nixops/resources/vpc_route_table_association.py @@ -10,7 +10,7 @@ from nixops.resources.ec2_common import EC2CommonState import nixops.ec2_utils from nixops.diff import Diff, Handler -from nixops.state import StateDict +from nixops.state.state import StateDict class VPCRouteTableAssociationDefinition(nixops.resources.ResourceDefinition): """Definition of a VPC route table association""" diff --git a/nixops/state/__init__.py b/nixops/state/__init__.py index 38de8f823..761b21d7b 100644 --- a/nixops/state/__init__.py +++ b/nixops/state/__init__.py @@ -1,13 +1,9 @@ -import json -import collections import os import urlparse import sys import json_file import sqlite_connector -import nixops.util - class WrongStateSchemeException(Exception): pass @@ -48,42 +44,3 @@ def raise_(ex): function = switcher.get(scheme, lambda(url): raise_(WrongStateSchemeException("Unknown state scheme! {}".format(url)))) return function(url) - - -class StateDict(collections.MutableMapping): - """ - An implementation of a MutableMapping container providing - a python dict like behavior for the NixOps state file. - """ - # TODO implement __repr__ for convenience e.g debuging the structure - def __init__(self, depl, id): - super(StateDict, self).__init__() - self._state = depl._state - self.uuid = depl.uuid - self.id = id - - def __setitem__(self, key, value): - self._state.set_resource_attrs(self.uuid, self.id, {key:value}) - - def __getitem__(self, key): - value = self._state.get_resource_attr(self.uuid, self.id, name) - try: - return json.loads(value) - except ValueError: - return value - raise KeyError("couldn't find key {} in the state file".format(key)) - - def __delitem__(self, key): - self._state.del_resource_attr(self.uuid, self.id, key) - - def keys(self): - # Generally the list of keys per ResourceAttrs is relatively small - # so this should be also relatively fast. - attrs = self._state.get_all_resource_attrs(self.uuid, self.id) - return [key for key,value in attrs.iteritems()] - - def __iter__(self): - return iter(self.keys()) - - def __len__(self): - return len(self.keys()) diff --git a/nixops/state/state.py b/nixops/state/state.py new file mode 100644 index 000000000..681387557 --- /dev/null +++ b/nixops/state/state.py @@ -0,0 +1,42 @@ +import json +import collections +import nixops.util + + +class StateDict(collections.MutableMapping): + """ + An implementation of a MutableMapping container providing + a python dict like behavior for the NixOps state file. + """ + # TODO implement __repr__ for convenience e.g debuging the structure + def __init__(self, depl, id): + super(StateDict, self).__init__() + self._state = depl._state + self.uuid = depl.uuid + self.id = id + + def __setitem__(self, key, value): + self._state.set_resource_attrs(self.uuid, self.id, {key:value}) + + def __getitem__(self, key): + value = self._state.get_resource_attr(self.uuid, self.id, name) + try: + return json.loads(value) + except ValueError: + return value + raise KeyError("couldn't find key {} in the state file".format(key)) + + def __delitem__(self, key): + self._state.del_resource_attr(self.uuid, self.id, key) + + def keys(self): + # Generally the list of keys per ResourceAttrs is relatively small + # so this should be also relatively fast. + attrs = self._state.get_all_resource_attrs(self.uuid, self.id) + return [key for key,value in attrs.iteritems()] + + def __iter__(self): + return iter(self.keys()) + + def __len__(self): + return len(self.keys()) diff --git a/release.nix b/release.nix index 3e4962a1e..098bd2628 100644 --- a/release.nix +++ b/release.nix @@ -91,9 +91,6 @@ rec { azure-mgmt-resource azure-mgmt-storage adal - # Go back to sqlite once Python 2.7.13 is released - sqlalchemy - pymysqlsa datadog digital-ocean ]; From 4585685972a76caac1eae5c41729c59a7c3abed9 Mon Sep 17 00:00:00 2001 From: Matthew O'Gorman Date: Wed, 14 Feb 2018 13:19:11 -0500 Subject: [PATCH 069/123] move class around --- nixops/resources/__init__.py | 2 +- nixops/resources/aws_vpn_connection.py | 2 +- nixops/resources/aws_vpn_connection_route.py | 2 +- nixops/resources/aws_vpn_gateway.py | 2 +- nixops/resources/vpc.py | 2 +- nixops/resources/vpc_customer_gateway.py | 2 +- nixops/resources/vpc_dhcp_options.py | 2 +- .../vpc_egress_only_internet_gateway.py | 2 +- nixops/resources/vpc_endpoint.py | 2 +- nixops/resources/vpc_internet_gateway.py | 2 +- nixops/resources/vpc_nat_gateway.py | 2 +- nixops/resources/vpc_network_acl.py | 2 +- nixops/resources/vpc_network_interface.py | 2 +- .../vpc_network_interface_attachment.py | 2 +- nixops/resources/vpc_route.py | 2 +- nixops/resources/vpc_route_table.py | 2 +- .../resources/vpc_route_table_association.py | 2 +- nixops/state/__init__.py | 42 +++++++++++++++++++ nixops/state/state.py | 42 ------------------- 19 files changed, 59 insertions(+), 59 deletions(-) delete mode 100644 nixops/state/state.py diff --git a/nixops/resources/__init__.py b/nixops/resources/__init__.py index 778e24ad3..804a1d9e4 100644 --- a/nixops/resources/__init__.py +++ b/nixops/resources/__init__.py @@ -3,7 +3,7 @@ import re import nixops.util -from nixops.state.state import StateDict +from nixops.state import StateDict from nixops.diff import Diff, Handler class ResourceDefinition(object): diff --git a/nixops/resources/aws_vpn_connection.py b/nixops/resources/aws_vpn_connection.py index 9080650a9..080b3be71 100644 --- a/nixops/resources/aws_vpn_connection.py +++ b/nixops/resources/aws_vpn_connection.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from nixops.state.state import StateDict +from nixops.state import StateDict from nixops.diff import Diff, Handler import nixops.util import nixops.resources diff --git a/nixops/resources/aws_vpn_connection_route.py b/nixops/resources/aws_vpn_connection_route.py index c6477c917..2da1631b0 100644 --- a/nixops/resources/aws_vpn_connection_route.py +++ b/nixops/resources/aws_vpn_connection_route.py @@ -7,7 +7,7 @@ from nixops.resources.ec2_common import EC2CommonState import nixops.ec2_utils from nixops.diff import Diff, Handler -from nixops.state.state import StateDict +from nixops.state import StateDict class AWSVPNConnectionRouteDefinition(nixops.resources.ResourceDefinition): """Definition of a VPN connection route""" diff --git a/nixops/resources/aws_vpn_gateway.py b/nixops/resources/aws_vpn_gateway.py index f41babee5..d97087878 100644 --- a/nixops/resources/aws_vpn_gateway.py +++ b/nixops/resources/aws_vpn_gateway.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from nixops.state.state import StateDict +from nixops.state import StateDict from nixops.diff import Diff, Handler import nixops.util import nixops.resources diff --git a/nixops/resources/vpc.py b/nixops/resources/vpc.py index 6cb3c5d75..28d1344aa 100755 --- a/nixops/resources/vpc.py +++ b/nixops/resources/vpc.py @@ -9,7 +9,7 @@ import nixops.resources from nixops.resources.ec2_common import EC2CommonState import nixops.ec2_utils -from nixops.state.state import StateDict +from nixops.state import StateDict from nixops.diff import Diff, Handler class VPCDefinition(nixops.resources.ResourceDefinition): diff --git a/nixops/resources/vpc_customer_gateway.py b/nixops/resources/vpc_customer_gateway.py index 9bc324058..129d7d846 100644 --- a/nixops/resources/vpc_customer_gateway.py +++ b/nixops/resources/vpc_customer_gateway.py @@ -7,7 +7,7 @@ import boto3 import botocore -from nixops.state.state import StateDict +from nixops.state import StateDict from nixops.diff import Diff, Handler import nixops.util import nixops.resources diff --git a/nixops/resources/vpc_dhcp_options.py b/nixops/resources/vpc_dhcp_options.py index 43000e8ad..114692687 100644 --- a/nixops/resources/vpc_dhcp_options.py +++ b/nixops/resources/vpc_dhcp_options.py @@ -12,7 +12,7 @@ import nixops.resources from nixops.resources.ec2_common import EC2CommonState import nixops.ec2_utils -from nixops.state.state import StateDict +from nixops.state import StateDict from nixops.diff import Diff, Handler class VPCDhcpOptionsDefinition(nixops.resources.ResourceDefinition): diff --git a/nixops/resources/vpc_egress_only_internet_gateway.py b/nixops/resources/vpc_egress_only_internet_gateway.py index 7abcd5dfa..d76791f60 100644 --- a/nixops/resources/vpc_egress_only_internet_gateway.py +++ b/nixops/resources/vpc_egress_only_internet_gateway.py @@ -3,7 +3,7 @@ import boto3 import botocore -from nixops.state.state import StateDict +from nixops.state import StateDict from nixops.diff import Diff, Handler import nixops.util import nixops.resources diff --git a/nixops/resources/vpc_endpoint.py b/nixops/resources/vpc_endpoint.py index dd2070da4..55671dbb6 100644 --- a/nixops/resources/vpc_endpoint.py +++ b/nixops/resources/vpc_endpoint.py @@ -2,7 +2,7 @@ import uuid -from nixops.state.state import StateDict +from nixops.state import StateDict from nixops.diff import Diff, Handler import nixops.util import nixops.resources diff --git a/nixops/resources/vpc_internet_gateway.py b/nixops/resources/vpc_internet_gateway.py index e260af14d..4a3045b00 100644 --- a/nixops/resources/vpc_internet_gateway.py +++ b/nixops/resources/vpc_internet_gateway.py @@ -7,7 +7,7 @@ import boto3 import botocore -from nixops.state.state import StateDict +from nixops.state import StateDict from nixops.diff import Diff, Handler import nixops.util import nixops.resources diff --git a/nixops/resources/vpc_nat_gateway.py b/nixops/resources/vpc_nat_gateway.py index c32c41dea..7295e4e1d 100644 --- a/nixops/resources/vpc_nat_gateway.py +++ b/nixops/resources/vpc_nat_gateway.py @@ -13,7 +13,7 @@ from nixops.resources.ec2_common import EC2CommonState import nixops.ec2_utils from nixops.diff import Diff, Handler -from nixops.state.state import StateDict +from nixops.state import StateDict class VPCNatGatewayDefinition(nixops.resources.ResourceDefinition): """Definition of a VPC NAT gateway""" diff --git a/nixops/resources/vpc_network_acl.py b/nixops/resources/vpc_network_acl.py index 39ce50328..6a5cf1b72 100644 --- a/nixops/resources/vpc_network_acl.py +++ b/nixops/resources/vpc_network_acl.py @@ -9,7 +9,7 @@ from nixops.resources.ec2_common import EC2CommonState import nixops.ec2_utils from nixops.diff import Diff, Handler -from nixops.state.state import StateDict +from nixops.state import StateDict class VPCNetworkAcldefinition(nixops.resources.ResourceDefinition): """definition of a vpc network ACL.""" diff --git a/nixops/resources/vpc_network_interface.py b/nixops/resources/vpc_network_interface.py index 84bf70a3a..8797e4a9b 100644 --- a/nixops/resources/vpc_network_interface.py +++ b/nixops/resources/vpc_network_interface.py @@ -10,7 +10,7 @@ from nixops.resources.ec2_common import EC2CommonState import nixops.ec2_utils from nixops.diff import Diff, Handler -from nixops.state.state import StateDict +from nixops.state import StateDict class VPCNetworkInterfaceDefinition(nixops.resources.ResourceDefinition): """Definition of a VPC network interface""" diff --git a/nixops/resources/vpc_network_interface_attachment.py b/nixops/resources/vpc_network_interface_attachment.py index 3ff610e76..cb22b924f 100644 --- a/nixops/resources/vpc_network_interface_attachment.py +++ b/nixops/resources/vpc_network_interface_attachment.py @@ -12,7 +12,7 @@ from nixops.resources.ec2_common import EC2CommonState import nixops.ec2_utils from nixops.diff import Diff, Handler -from nixops.state.state import StateDict +from nixops.state import StateDict class VPCNetworkInterfaceAttachmentDefinition(nixops.resources.ResourceDefinition): """Definition of a VPC network interface attachment""" diff --git a/nixops/resources/vpc_route.py b/nixops/resources/vpc_route.py index 3e7e686ba..2bb88a014 100644 --- a/nixops/resources/vpc_route.py +++ b/nixops/resources/vpc_route.py @@ -10,7 +10,7 @@ from nixops.resources.ec2_common import EC2CommonState import nixops.ec2_utils from nixops.diff import Diff, Handler -from nixops.state.state import StateDict +from nixops.state import StateDict class VPCRouteDefinition(nixops.resources.ResourceDefinition): """Definition of a VPC route""" diff --git a/nixops/resources/vpc_route_table.py b/nixops/resources/vpc_route_table.py index 504e0cc03..a0fd0c123 100644 --- a/nixops/resources/vpc_route_table.py +++ b/nixops/resources/vpc_route_table.py @@ -10,7 +10,7 @@ from nixops.resources.ec2_common import EC2CommonState import nixops.ec2_utils from nixops.diff import Diff, Handler -from nixops.state.state import StateDict +from nixops.state import StateDict class VPCRouteTableDefinition(nixops.resources.ResourceDefinition): """Definition of a VPC route table""" diff --git a/nixops/resources/vpc_route_table_association.py b/nixops/resources/vpc_route_table_association.py index b1cc3d6c7..e6d38de63 100644 --- a/nixops/resources/vpc_route_table_association.py +++ b/nixops/resources/vpc_route_table_association.py @@ -10,7 +10,7 @@ from nixops.resources.ec2_common import EC2CommonState import nixops.ec2_utils from nixops.diff import Diff, Handler -from nixops.state.state import StateDict +from nixops.state import StateDict class VPCRouteTableAssociationDefinition(nixops.resources.ResourceDefinition): """Definition of a VPC route table association""" diff --git a/nixops/state/__init__.py b/nixops/state/__init__.py index 761b21d7b..92ac1f9ab 100644 --- a/nixops/state/__init__.py +++ b/nixops/state/__init__.py @@ -3,6 +3,9 @@ import sys import json_file import sqlite_connector +import json +import collections +import nixops.util class WrongStateSchemeException(Exception): pass @@ -44,3 +47,42 @@ def raise_(ex): function = switcher.get(scheme, lambda(url): raise_(WrongStateSchemeException("Unknown state scheme! {}".format(url)))) return function(url) + + +class StateDict(collections.MutableMapping): + """ + An implementation of a MutableMapping container providing + a python dict like behavior for the NixOps state file. + """ + # TODO implement __repr__ for convenience e.g debuging the structure + def __init__(self, depl, id): + super(StateDict, self).__init__() + self._state = depl._state + self.uuid = depl.uuid + self.id = id + + def __setitem__(self, key, value): + self._state.set_resource_attrs(self.uuid, self.id, {key:value}) + + def __getitem__(self, key): + value = self._state.get_resource_attr(self.uuid, self.id, name) + try: + return json.loads(value) + except ValueError: + return value + raise KeyError("couldn't find key {} in the state file".format(key)) + + def __delitem__(self, key): + self._state.del_resource_attr(self.uuid, self.id, key) + + def keys(self): + # Generally the list of keys per ResourceAttrs is relatively small + # so this should be also relatively fast. + attrs = self._state.get_all_resource_attrs(self.uuid, self.id) + return [key for key,value in attrs.iteritems()] + + def __iter__(self): + return iter(self.keys()) + + def __len__(self): + return len(self.keys()) diff --git a/nixops/state/state.py b/nixops/state/state.py deleted file mode 100644 index 681387557..000000000 --- a/nixops/state/state.py +++ /dev/null @@ -1,42 +0,0 @@ -import json -import collections -import nixops.util - - -class StateDict(collections.MutableMapping): - """ - An implementation of a MutableMapping container providing - a python dict like behavior for the NixOps state file. - """ - # TODO implement __repr__ for convenience e.g debuging the structure - def __init__(self, depl, id): - super(StateDict, self).__init__() - self._state = depl._state - self.uuid = depl.uuid - self.id = id - - def __setitem__(self, key, value): - self._state.set_resource_attrs(self.uuid, self.id, {key:value}) - - def __getitem__(self, key): - value = self._state.get_resource_attr(self.uuid, self.id, name) - try: - return json.loads(value) - except ValueError: - return value - raise KeyError("couldn't find key {} in the state file".format(key)) - - def __delitem__(self, key): - self._state.del_resource_attr(self.uuid, self.id, key) - - def keys(self): - # Generally the list of keys per ResourceAttrs is relatively small - # so this should be also relatively fast. - attrs = self._state.get_all_resource_attrs(self.uuid, self.id) - return [key for key,value in attrs.iteritems()] - - def __iter__(self): - return iter(self.keys()) - - def __len__(self): - return len(self.keys()) From 07278c80a0c55ce222d5f86378d56f66862c8c77 Mon Sep 17 00:00:00 2001 From: Matthew O'Gorman Date: Wed, 14 Feb 2018 13:21:06 -0500 Subject: [PATCH 070/123] test --- nixops/resources/__init__.py | 2 +- nixops/resources/aws_vpn_connection.py | 2 +- nixops/resources/aws_vpn_connection_route.py | 2 +- nixops/resources/aws_vpn_gateway.py | 2 +- nixops/resources/vpc.py | 2 +- nixops/resources/vpc_customer_gateway.py | 2 +- nixops/resources/vpc_dhcp_options.py | 2 +- .../vpc_egress_only_internet_gateway.py | 2 +- nixops/resources/vpc_endpoint.py | 2 +- nixops/resources/vpc_internet_gateway.py | 2 +- nixops/resources/vpc_nat_gateway.py | 2 +- nixops/resources/vpc_network_acl.py | 2 +- nixops/resources/vpc_network_interface.py | 2 +- .../vpc_network_interface_attachment.py | 2 +- nixops/resources/vpc_route.py | 2 +- nixops/resources/vpc_route_table.py | 2 +- .../resources/vpc_route_table_association.py | 2 +- nixops/state/__init__.py | 42 ------------------- nixops/state/state_helper.py | 42 +++++++++++++++++++ 19 files changed, 59 insertions(+), 59 deletions(-) create mode 100644 nixops/state/state_helper.py diff --git a/nixops/resources/__init__.py b/nixops/resources/__init__.py index 804a1d9e4..710be33ff 100644 --- a/nixops/resources/__init__.py +++ b/nixops/resources/__init__.py @@ -3,7 +3,7 @@ import re import nixops.util -from nixops.state import StateDict +from nixops.state.state_helper import StateDict from nixops.diff import Diff, Handler class ResourceDefinition(object): diff --git a/nixops/resources/aws_vpn_connection.py b/nixops/resources/aws_vpn_connection.py index 080b3be71..c6962e660 100644 --- a/nixops/resources/aws_vpn_connection.py +++ b/nixops/resources/aws_vpn_connection.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from nixops.state import StateDict +from nixops.state.state_helper import StateDict from nixops.diff import Diff, Handler import nixops.util import nixops.resources diff --git a/nixops/resources/aws_vpn_connection_route.py b/nixops/resources/aws_vpn_connection_route.py index 2da1631b0..a7f099f76 100644 --- a/nixops/resources/aws_vpn_connection_route.py +++ b/nixops/resources/aws_vpn_connection_route.py @@ -7,7 +7,7 @@ from nixops.resources.ec2_common import EC2CommonState import nixops.ec2_utils from nixops.diff import Diff, Handler -from nixops.state import StateDict +from nixops.state.state_helper import StateDict class AWSVPNConnectionRouteDefinition(nixops.resources.ResourceDefinition): """Definition of a VPN connection route""" diff --git a/nixops/resources/aws_vpn_gateway.py b/nixops/resources/aws_vpn_gateway.py index d97087878..7137e3526 100644 --- a/nixops/resources/aws_vpn_gateway.py +++ b/nixops/resources/aws_vpn_gateway.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from nixops.state import StateDict +from nixops.state.state_helper import StateDict from nixops.diff import Diff, Handler import nixops.util import nixops.resources diff --git a/nixops/resources/vpc.py b/nixops/resources/vpc.py index 28d1344aa..400aa53b5 100755 --- a/nixops/resources/vpc.py +++ b/nixops/resources/vpc.py @@ -9,7 +9,7 @@ import nixops.resources from nixops.resources.ec2_common import EC2CommonState import nixops.ec2_utils -from nixops.state import StateDict +from nixops.state.state_helper import StateDict from nixops.diff import Diff, Handler class VPCDefinition(nixops.resources.ResourceDefinition): diff --git a/nixops/resources/vpc_customer_gateway.py b/nixops/resources/vpc_customer_gateway.py index 129d7d846..ece177cba 100644 --- a/nixops/resources/vpc_customer_gateway.py +++ b/nixops/resources/vpc_customer_gateway.py @@ -7,7 +7,7 @@ import boto3 import botocore -from nixops.state import StateDict +from nixops.state.state_helper import StateDict from nixops.diff import Diff, Handler import nixops.util import nixops.resources diff --git a/nixops/resources/vpc_dhcp_options.py b/nixops/resources/vpc_dhcp_options.py index 114692687..0fd7be400 100644 --- a/nixops/resources/vpc_dhcp_options.py +++ b/nixops/resources/vpc_dhcp_options.py @@ -12,7 +12,7 @@ import nixops.resources from nixops.resources.ec2_common import EC2CommonState import nixops.ec2_utils -from nixops.state import StateDict +from nixops.state.state_helper import StateDict from nixops.diff import Diff, Handler class VPCDhcpOptionsDefinition(nixops.resources.ResourceDefinition): diff --git a/nixops/resources/vpc_egress_only_internet_gateway.py b/nixops/resources/vpc_egress_only_internet_gateway.py index d76791f60..aba415dae 100644 --- a/nixops/resources/vpc_egress_only_internet_gateway.py +++ b/nixops/resources/vpc_egress_only_internet_gateway.py @@ -3,7 +3,7 @@ import boto3 import botocore -from nixops.state import StateDict +from nixops.state.state_helper import StateDict from nixops.diff import Diff, Handler import nixops.util import nixops.resources diff --git a/nixops/resources/vpc_endpoint.py b/nixops/resources/vpc_endpoint.py index 55671dbb6..5759b5846 100644 --- a/nixops/resources/vpc_endpoint.py +++ b/nixops/resources/vpc_endpoint.py @@ -2,7 +2,7 @@ import uuid -from nixops.state import StateDict +from nixops.state.state_helper import StateDict from nixops.diff import Diff, Handler import nixops.util import nixops.resources diff --git a/nixops/resources/vpc_internet_gateway.py b/nixops/resources/vpc_internet_gateway.py index 4a3045b00..236a8f285 100644 --- a/nixops/resources/vpc_internet_gateway.py +++ b/nixops/resources/vpc_internet_gateway.py @@ -7,7 +7,7 @@ import boto3 import botocore -from nixops.state import StateDict +from nixops.state.state_helper import StateDict from nixops.diff import Diff, Handler import nixops.util import nixops.resources diff --git a/nixops/resources/vpc_nat_gateway.py b/nixops/resources/vpc_nat_gateway.py index 7295e4e1d..c58aa63df 100644 --- a/nixops/resources/vpc_nat_gateway.py +++ b/nixops/resources/vpc_nat_gateway.py @@ -13,7 +13,7 @@ from nixops.resources.ec2_common import EC2CommonState import nixops.ec2_utils from nixops.diff import Diff, Handler -from nixops.state import StateDict +from nixops.state.state_helper import StateDict class VPCNatGatewayDefinition(nixops.resources.ResourceDefinition): """Definition of a VPC NAT gateway""" diff --git a/nixops/resources/vpc_network_acl.py b/nixops/resources/vpc_network_acl.py index 6a5cf1b72..667f6c5de 100644 --- a/nixops/resources/vpc_network_acl.py +++ b/nixops/resources/vpc_network_acl.py @@ -9,7 +9,7 @@ from nixops.resources.ec2_common import EC2CommonState import nixops.ec2_utils from nixops.diff import Diff, Handler -from nixops.state import StateDict +from nixops.state.state_helper import StateDict class VPCNetworkAcldefinition(nixops.resources.ResourceDefinition): """definition of a vpc network ACL.""" diff --git a/nixops/resources/vpc_network_interface.py b/nixops/resources/vpc_network_interface.py index 8797e4a9b..b867423d0 100644 --- a/nixops/resources/vpc_network_interface.py +++ b/nixops/resources/vpc_network_interface.py @@ -10,7 +10,7 @@ from nixops.resources.ec2_common import EC2CommonState import nixops.ec2_utils from nixops.diff import Diff, Handler -from nixops.state import StateDict +from nixops.state.state_helper import StateDict class VPCNetworkInterfaceDefinition(nixops.resources.ResourceDefinition): """Definition of a VPC network interface""" diff --git a/nixops/resources/vpc_network_interface_attachment.py b/nixops/resources/vpc_network_interface_attachment.py index cb22b924f..ae8c1113b 100644 --- a/nixops/resources/vpc_network_interface_attachment.py +++ b/nixops/resources/vpc_network_interface_attachment.py @@ -12,7 +12,7 @@ from nixops.resources.ec2_common import EC2CommonState import nixops.ec2_utils from nixops.diff import Diff, Handler -from nixops.state import StateDict +from nixops.state.state_helper import StateDict class VPCNetworkInterfaceAttachmentDefinition(nixops.resources.ResourceDefinition): """Definition of a VPC network interface attachment""" diff --git a/nixops/resources/vpc_route.py b/nixops/resources/vpc_route.py index 2bb88a014..399e92772 100644 --- a/nixops/resources/vpc_route.py +++ b/nixops/resources/vpc_route.py @@ -10,7 +10,7 @@ from nixops.resources.ec2_common import EC2CommonState import nixops.ec2_utils from nixops.diff import Diff, Handler -from nixops.state import StateDict +from nixops.state.state_helper import StateDict class VPCRouteDefinition(nixops.resources.ResourceDefinition): """Definition of a VPC route""" diff --git a/nixops/resources/vpc_route_table.py b/nixops/resources/vpc_route_table.py index a0fd0c123..ef05c7d9d 100644 --- a/nixops/resources/vpc_route_table.py +++ b/nixops/resources/vpc_route_table.py @@ -10,7 +10,7 @@ from nixops.resources.ec2_common import EC2CommonState import nixops.ec2_utils from nixops.diff import Diff, Handler -from nixops.state import StateDict +from nixops.state.state_helper import StateDict class VPCRouteTableDefinition(nixops.resources.ResourceDefinition): """Definition of a VPC route table""" diff --git a/nixops/resources/vpc_route_table_association.py b/nixops/resources/vpc_route_table_association.py index e6d38de63..f4b5d14b6 100644 --- a/nixops/resources/vpc_route_table_association.py +++ b/nixops/resources/vpc_route_table_association.py @@ -10,7 +10,7 @@ from nixops.resources.ec2_common import EC2CommonState import nixops.ec2_utils from nixops.diff import Diff, Handler -from nixops.state import StateDict +from nixops.state.state_helper import StateDict class VPCRouteTableAssociationDefinition(nixops.resources.ResourceDefinition): """Definition of a VPC route table association""" diff --git a/nixops/state/__init__.py b/nixops/state/__init__.py index 92ac1f9ab..761b21d7b 100644 --- a/nixops/state/__init__.py +++ b/nixops/state/__init__.py @@ -3,9 +3,6 @@ import sys import json_file import sqlite_connector -import json -import collections -import nixops.util class WrongStateSchemeException(Exception): pass @@ -47,42 +44,3 @@ def raise_(ex): function = switcher.get(scheme, lambda(url): raise_(WrongStateSchemeException("Unknown state scheme! {}".format(url)))) return function(url) - - -class StateDict(collections.MutableMapping): - """ - An implementation of a MutableMapping container providing - a python dict like behavior for the NixOps state file. - """ - # TODO implement __repr__ for convenience e.g debuging the structure - def __init__(self, depl, id): - super(StateDict, self).__init__() - self._state = depl._state - self.uuid = depl.uuid - self.id = id - - def __setitem__(self, key, value): - self._state.set_resource_attrs(self.uuid, self.id, {key:value}) - - def __getitem__(self, key): - value = self._state.get_resource_attr(self.uuid, self.id, name) - try: - return json.loads(value) - except ValueError: - return value - raise KeyError("couldn't find key {} in the state file".format(key)) - - def __delitem__(self, key): - self._state.del_resource_attr(self.uuid, self.id, key) - - def keys(self): - # Generally the list of keys per ResourceAttrs is relatively small - # so this should be also relatively fast. - attrs = self._state.get_all_resource_attrs(self.uuid, self.id) - return [key for key,value in attrs.iteritems()] - - def __iter__(self): - return iter(self.keys()) - - def __len__(self): - return len(self.keys()) diff --git a/nixops/state/state_helper.py b/nixops/state/state_helper.py new file mode 100644 index 000000000..681387557 --- /dev/null +++ b/nixops/state/state_helper.py @@ -0,0 +1,42 @@ +import json +import collections +import nixops.util + + +class StateDict(collections.MutableMapping): + """ + An implementation of a MutableMapping container providing + a python dict like behavior for the NixOps state file. + """ + # TODO implement __repr__ for convenience e.g debuging the structure + def __init__(self, depl, id): + super(StateDict, self).__init__() + self._state = depl._state + self.uuid = depl.uuid + self.id = id + + def __setitem__(self, key, value): + self._state.set_resource_attrs(self.uuid, self.id, {key:value}) + + def __getitem__(self, key): + value = self._state.get_resource_attr(self.uuid, self.id, name) + try: + return json.loads(value) + except ValueError: + return value + raise KeyError("couldn't find key {} in the state file".format(key)) + + def __delitem__(self, key): + self._state.del_resource_attr(self.uuid, self.id, key) + + def keys(self): + # Generally the list of keys per ResourceAttrs is relatively small + # so this should be also relatively fast. + attrs = self._state.get_all_resource_attrs(self.uuid, self.id) + return [key for key,value in attrs.iteritems()] + + def __iter__(self): + return iter(self.keys()) + + def __len__(self): + return len(self.keys()) From 76ffca7dd5d937fcaf0796f3e191ebd7e1bead53 Mon Sep 17 00:00:00 2001 From: Matthew O'Gorman Date: Wed, 14 Feb 2018 14:17:44 -0500 Subject: [PATCH 071/123] remove debugging --- nixops/backends/virtualbox.py | 6 +++--- nixops/deployment.py | 1 - nixops/known_hosts.py | 1 - nixops/resources/__init__.py | 2 +- nixops/state/__init__.py | 1 - nixops/state/sqlite_connector.py | 10 +++------- nixops/util.py | 4 +--- 7 files changed, 8 insertions(+), 17 deletions(-) diff --git a/nixops/backends/virtualbox.py b/nixops/backends/virtualbox.py index 975e28448..01ab95a0d 100644 --- a/nixops/backends/virtualbox.py +++ b/nixops/backends/virtualbox.py @@ -141,7 +141,7 @@ def _update_ip(self): capture_stdout=True).rstrip() if res[0:7] != "Value: ": return new_address = res[7:] - print 'called here with data' + nixops.known_hosts.update(self.private_ipv4, new_address, self.public_host_key) self.private_ipv4 = new_address @@ -355,7 +355,7 @@ def create(self, defn, check, allow_reboot, allow_recreate): "--nictype2", "virtio", "--nic2", "hostonly", "--hostonlyadapter2", "vboxnet0", - "--nestedpaging", "off", + "--nestedpaging", "off",s "--paravirtprovider", "kvm" ] vcpus = defn.config["virtualbox"]["vcpu"] # None or integer @@ -395,7 +395,7 @@ def destroy(self, wipe=False): self.state = self.STOPPED time.sleep(1) # hack to work around "machine locked" errors - print 'called here with none?' + nixops.known_hosts.update(self.private_ipv4, None, self.public_host_key) self._logged_exec(["VBoxManage", "unregistervm", "--delete", self.vm_id]) diff --git a/nixops/deployment.py b/nixops/deployment.py index c2b3cd9d0..66e11b126 100644 --- a/nixops/deployment.py +++ b/nixops/deployment.py @@ -135,7 +135,6 @@ def _del_attr(self, name): # Removed it when moving the body to nixops/state/file.py. def _get_attr(self, name, default=nixops.util.undefined): """Get a deployment attribute from the state.""" - print 'get deploymante attr' return self._state.get_deployment_attr(self.uuid, name) def _create_resource(self, name, type): diff --git a/nixops/known_hosts.py b/nixops/known_hosts.py index 88d7fc11b..fcd05b0c7 100644 --- a/nixops/known_hosts.py +++ b/nixops/known_hosts.py @@ -57,7 +57,6 @@ def add(ip_address, public_host_key): def update(prev_address, new_address, public_host_key): - print 'FOUND on UPDATE {} {} {}'.format(prev_address, new_address, public_host_key) assert public_host_key is not None # FIXME: this rewrites known_hosts twice. if prev_address is not None and prev_address != new_address: diff --git a/nixops/resources/__init__.py b/nixops/resources/__init__.py index 710be33ff..d53f7f5fa 100644 --- a/nixops/resources/__init__.py +++ b/nixops/resources/__init__.py @@ -81,7 +81,7 @@ def _del_attr(self, name): # Have removed it in state/file.py. def _get_attr(self, name, default=nixops.util.undefined): """Get a machine attribute from the state file.""" - print 'get resource attr' + return self.depl._state.get_resource_attr(self.depl.uuid, self.id, name) def export(self): diff --git a/nixops/state/__init__.py b/nixops/state/__init__.py index 761b21d7b..10a52c27d 100644 --- a/nixops/state/__init__.py +++ b/nixops/state/__init__.py @@ -23,7 +23,6 @@ def get_default_state_file(): def open(url): - print 'url = {}'.format(url) url_parsed = urlparse.urlparse(url) scheme = url_parsed.scheme ext = os.path.splitext(url)[1] diff --git a/nixops/state/sqlite_connector.py b/nixops/state/sqlite_connector.py index 16ac07114..46eb26d15 100644 --- a/nixops/state/sqlite_connector.py +++ b/nixops/state/sqlite_connector.py @@ -19,14 +19,12 @@ def _subclasses(cls): class Connection(sqlite3.Connection): def __init__(self, db_file, **kwargs): - print 'a file {}'.format(db_file) db_exists = os.path.exists(db_file) if not db_exists: - print 'failed' os.fdopen(os.open(db_file, os.O_WRONLY | os.O_CREAT, 0o600), 'w').close() - print 'opening' + sqlite3.Connection.__init__(self, db_file, **kwargs) - print 'double fail' + self.db_file = db_file self.nesting = 0 self.lock = threading.RLock() @@ -64,10 +62,9 @@ class SQLiteConnection(object): current_schema = 3 def __init__(self, db_file): - print 'running {}'.format(db_file) url = urlparse.urlparse(db_file) self.db_file = url.netloc + url.path - print 'running {} {}'.format(self.db_file, urlparse.urlparse(db_file)) + if os.path.splitext(db_file)[1] not in ['.nixops', '.charon']: raise Exception("state file ‘{0}’ should have extension ‘.nixops’".format(db_file)) db = sqlite3.connect(self.db_file, timeout=60, check_same_thread=False, factory=Connection, isolation_level=None) # FIXME @@ -375,7 +372,6 @@ def get_resource_attr(self, _deployment_uuid, resource_id, name): rows = self.db.execute("select value from ResourceAttrs where machine = '{}' and name = '{}'" .format(resource_id, name)).fetchall() for row in rows: - print 'here is the resource {}'.format(row[0]) return row[0] return nixops.util.undefined diff --git a/nixops/util.py b/nixops/util.py index 3ea7f5011..c0d399c01 100644 --- a/nixops/util.py +++ b/nixops/util.py @@ -241,9 +241,8 @@ class Undefined: def attr_property(name, default, type=str): """Define a property that corresponds to a value in the NixOps state file.""" def get(self): - print 'getting attr {} {} {}'.format(name, default, type) s = self._get_attr(name, default) - print 'got s={}'.format(s) + if s == undefined: if default != undefined: return copy.deepcopy(default) raise Exception("deployment attribute ‘{0}’ missing from state file".format(name)) @@ -254,7 +253,6 @@ def get(self): elif type is 'json': return json.loads(s) else: assert False def set(self, x): - print 'setting attr {} {} {}'.format(name, default, type) if x == default: self._del_attr(name) elif type is 'json': self._set_attr(name, json.dumps(x)) else: self._set_attr(name, x) From 6cf8c0262e0ad930b5f61abbf07420d2b4f2bfbd Mon Sep 17 00:00:00 2001 From: Matthew O'Gorman Date: Wed, 14 Feb 2018 14:41:34 -0500 Subject: [PATCH 072/123] adding unit test and testing it --- nixops/state/json_file.py | 2 +- tests/__init__.py | 1 + tests/functional/__init__.py | 12 +++++++++++- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/nixops/state/json_file.py b/nixops/state/json_file.py index 873b27ab5..9561d4cd2 100644 --- a/nixops/state/json_file.py +++ b/nixops/state/json_file.py @@ -121,7 +121,7 @@ class JsonFile(object): def __init__(self, json_file): self.file_path = json_file - + print "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX JSON XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" if os.path.splitext(json_file)[1] not in ['.json']: raise Exception("state file ‘{0}’ should have extension ‘.json’".format(json_file)) diff --git a/tests/__init__.py b/tests/__init__.py index 2c25dc760..5587a6d4e 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -8,6 +8,7 @@ _multiprocess_shared_ = True db_file = '%s/test.nixops' % (path.dirname(__file__)) +json_file = '%s/test.json' % (path.dirname(__file__)) def setup(): state = nixops.state.open(db_file) diff --git a/tests/functional/__init__.py b/tests/functional/__init__.py index 98f8b8015..c3b3bb895 100644 --- a/tests/functional/__init__.py +++ b/tests/functional/__init__.py @@ -2,7 +2,7 @@ from os import path import nixops.state from tests import db_file - +from tests import json_file class DatabaseUsingTest(object): _multiprocess_can_split_ = True @@ -12,3 +12,13 @@ def setup(self): def teardown(self): self.sf.close() + + +class JSONUsingTest(object): + _multiprocess_can_split_ = True + + def setup(self): + self.sf = nixops.state.open(json_file) + + def teardown(self): + self.sf.close() From f656d58a8151617aa13f1906c807ea0190ee39ad Mon Sep 17 00:00:00 2001 From: Matthew O'Gorman Date: Wed, 14 Feb 2018 14:53:49 -0500 Subject: [PATCH 073/123] new unit tests --- nixops/backends/virtualbox.py | 2 +- tests/functional/generic_json_deployment_test.py | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 tests/functional/generic_json_deployment_test.py diff --git a/nixops/backends/virtualbox.py b/nixops/backends/virtualbox.py index 01ab95a0d..151095505 100644 --- a/nixops/backends/virtualbox.py +++ b/nixops/backends/virtualbox.py @@ -355,7 +355,7 @@ def create(self, defn, check, allow_reboot, allow_recreate): "--nictype2", "virtio", "--nic2", "hostonly", "--hostonlyadapter2", "vboxnet0", - "--nestedpaging", "off",s + "--nestedpaging", "off", "--paravirtprovider", "kvm" ] vcpus = defn.config["virtualbox"]["vcpu"] # None or integer diff --git a/tests/functional/generic_json_deployment_test.py b/tests/functional/generic_json_deployment_test.py new file mode 100644 index 000000000..041fd53dc --- /dev/null +++ b/tests/functional/generic_json_deployment_test.py @@ -0,0 +1,12 @@ +import os +import subprocess + +from nose import SkipTest + +from tests.functional import JSONUsingTest + +class GenericJsonDeploymentTest(JSONUsingTest): + def setup(self): + super(GenericJsonDeploymentTest,self).setup() + self.depl = self.sf.create_deployment() + self.depl.logger.set_autoresponse("y") From 83c4af9cdb4e42022c44a11794f92e9e2e3d45e0 Mon Sep 17 00:00:00 2001 From: Matthew O'Gorman Date: Wed, 14 Feb 2018 15:28:30 -0500 Subject: [PATCH 074/123] remove debugging --- nixops/state/json_file.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nixops/state/json_file.py b/nixops/state/json_file.py index 9561d4cd2..873b27ab5 100644 --- a/nixops/state/json_file.py +++ b/nixops/state/json_file.py @@ -121,7 +121,7 @@ class JsonFile(object): def __init__(self, json_file): self.file_path = json_file - print "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX JSON XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" + if os.path.splitext(json_file)[1] not in ['.json']: raise Exception("state file ‘{0}’ should have extension ‘.json’".format(json_file)) From 4cea4d22058cd8c65b2b252eb85f88b43376b33c Mon Sep 17 00:00:00 2001 From: Matthew O'Gorman Date: Sun, 18 Feb 2018 21:43:39 -0500 Subject: [PATCH 075/123] fix variable name issue and move subclasses to common spot among backends --- nixops/state/json_file.py | 5 +---- nixops/state/sqlite_connector.py | 7 ++----- nixops/state/state_helper.py | 7 ++++++- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/nixops/state/json_file.py b/nixops/state/json_file.py index 873b27ab5..c3923f53a 100644 --- a/nixops/state/json_file.py +++ b/nixops/state/json_file.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- import nixops.deployment +from nixops.state.state_helper import _subclasses import os import os.path import sys @@ -12,10 +13,6 @@ from uuid import uuid1 as gen_uuid -def _subclasses(cls): - sub = cls.__subclasses__() - return [cls] if not sub else [g for s in sub for g in _subclasses(s)] - class TransactionalJsonFile: """ Transactional access to a JSON file, with support diff --git a/nixops/state/sqlite_connector.py b/nixops/state/sqlite_connector.py index 46eb26d15..3d409b0d1 100644 --- a/nixops/state/sqlite_connector.py +++ b/nixops/state/sqlite_connector.py @@ -2,18 +2,15 @@ import nixops.deployment +from nixops.state.state_helper import _subclasses import os import os.path import urlparse import sys import threading import fcntl -import sqlite3 - -def _subclasses(cls): - sub = cls.__subclasses__() - return [cls] if not sub else [g for s in sub for g in _subclasses(s)] +import sqlite3 class Connection(sqlite3.Connection): diff --git a/nixops/state/state_helper.py b/nixops/state/state_helper.py index 681387557..f05e2fdd1 100644 --- a/nixops/state/state_helper.py +++ b/nixops/state/state_helper.py @@ -3,6 +3,11 @@ import nixops.util +def _subclasses(cls): + sub = cls.__subclasses__() + return [cls] if not sub else [g for s in sub for g in _subclasses(s)] + + class StateDict(collections.MutableMapping): """ An implementation of a MutableMapping container providing @@ -19,7 +24,7 @@ def __setitem__(self, key, value): self._state.set_resource_attrs(self.uuid, self.id, {key:value}) def __getitem__(self, key): - value = self._state.get_resource_attr(self.uuid, self.id, name) + value = self._state.get_resource_attr(self.uuid, self.id, key) try: return json.loads(value) except ValueError: From f477f2a78d895286505fc99854e9d1f8a820b7cb Mon Sep 17 00:00:00 2001 From: Matthew O'Gorman Date: Mon, 19 Feb 2018 01:21:42 -0500 Subject: [PATCH 076/123] work on unit tests --- nixops/state/sqlite_connector.py | 3 ++- nixops/state/state_helper.py | 9 +++++---- setup.py | 2 +- tests/__init__.py | 14 +++++++------- tests/functional/__init__.py | 8 ++++---- tests/functional/generic_deployment_test.py | 2 +- tests/functional/generic_json_deployment_test.py | 2 +- tests/functional/test_deleting_deletes.py | 2 +- tests/functional/test_query_deployments.py | 4 ++-- tests/functional/vpc.py | 1 + 10 files changed, 25 insertions(+), 22 deletions(-) diff --git a/nixops/state/sqlite_connector.py b/nixops/state/sqlite_connector.py index 3d409b0d1..ab07c879c 100644 --- a/nixops/state/sqlite_connector.py +++ b/nixops/state/sqlite_connector.py @@ -368,6 +368,7 @@ def get_resource_attr(self, _deployment_uuid, resource_id, name): """Get a machine attribute from the state file.""" rows = self.db.execute("select value from ResourceAttrs where machine = '{}' and name = '{}'" .format(resource_id, name)).fetchall() + for row in rows: return row[0] return nixops.util.undefined @@ -375,7 +376,7 @@ def get_resource_attr(self, _deployment_uuid, resource_id, name): def get_all_resource_attrs(self, deployment_uuid, resource_id): with self.db: - rows = self.db.execute("select name, value from ResourceAttrs where machine = {!r}" + rows = self.db.execute("select name, value from ResourceAttrs where machine = '{}'" .format(resource_id)).fetchall() res = {row[0]: row[1] for row in rows} return res diff --git a/nixops/state/state_helper.py b/nixops/state/state_helper.py index f05e2fdd1..12bfa225e 100644 --- a/nixops/state/state_helper.py +++ b/nixops/state/state_helper.py @@ -25,10 +25,11 @@ def __setitem__(self, key, value): def __getitem__(self, key): value = self._state.get_resource_attr(self.uuid, self.id, key) - try: - return json.loads(value) - except ValueError: - return value + if value != nixops.util.undefined: + try: + return json.loads(value) + except ValueError: + return value raise KeyError("couldn't find key {} in the state file".format(key)) def __delitem__(self, key): diff --git a/setup.py b/setup.py index ed09010e7..ca08bbfb8 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ def run(self): author='Eelco Dolstra', author_email='eelco.dolstra@logicblox.com', scripts=['scripts/nixops'], - packages=['nixops', 'nixops.resources', 'nixops.backends'], + packages=['nixops', 'nixops.resources', 'nixops.backends', 'nixops.state'], package_data={'nixops': ['data/nixos-infect']}, cmdclass={'test': TestCommand} ) diff --git a/tests/__init__.py b/tests/__init__.py index 5587a6d4e..7b760ec24 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -14,8 +14,8 @@ def setup(): state = nixops.state.open(db_file) state.db.close() -def destroy(sf, uuid): - depl = sf.open_deployment(uuid) +def destroy(state, uuid): + depl = state.open_deployment(uuid) depl.logger.set_autoresponse("y") try: depl.clean_backups(keep=0) @@ -29,17 +29,17 @@ def destroy(sf, uuid): depl.logger.log("deployment ‘{0}’ destroyed".format(uuid)) def teardown(): - sf = nixops.state.open(db_file) - uuids = sf.query_deployments() + state = nixops.state.open(db_file) + uuids = state.query_deployments() threads = [] for uuid in uuids: - threads.append(threading.Thread(target=destroy,args=(sf, uuid))) + threads.append(threading.Thread(target=destroy,args=(state, uuid))) for thread in threads: thread.start() for thread in threads: thread.join() - uuids_left = sf.query_deployments() - sf.close() + uuids_left = state.query_deployments() + state.close() if not uuids_left: os.remove(db_file) else: diff --git a/tests/functional/__init__.py b/tests/functional/__init__.py index c3b3bb895..3c5d0c540 100644 --- a/tests/functional/__init__.py +++ b/tests/functional/__init__.py @@ -8,17 +8,17 @@ class DatabaseUsingTest(object): _multiprocess_can_split_ = True def setup(self): - self.sf = nixops.state.open(db_file) + self.state = nixops.state.open(db_file) def teardown(self): - self.sf.close() + self.state.close() class JSONUsingTest(object): _multiprocess_can_split_ = True def setup(self): - self.sf = nixops.state.open(json_file) + self.state = nixops.state.open(json_file) def teardown(self): - self.sf.close() + self.state.close() diff --git a/tests/functional/generic_deployment_test.py b/tests/functional/generic_deployment_test.py index 26085fb82..f40781423 100644 --- a/tests/functional/generic_deployment_test.py +++ b/tests/functional/generic_deployment_test.py @@ -8,5 +8,5 @@ class GenericDeploymentTest(DatabaseUsingTest): def setup(self): super(GenericDeploymentTest,self).setup() - self.depl = self.sf.create_deployment() + self.depl = self.state.create_deployment() self.depl.logger.set_autoresponse("y") diff --git a/tests/functional/generic_json_deployment_test.py b/tests/functional/generic_json_deployment_test.py index 041fd53dc..ef6f3b04d 100644 --- a/tests/functional/generic_json_deployment_test.py +++ b/tests/functional/generic_json_deployment_test.py @@ -8,5 +8,5 @@ class GenericJsonDeploymentTest(JSONUsingTest): def setup(self): super(GenericJsonDeploymentTest,self).setup() - self.depl = self.sf.create_deployment() + self.depl = self.state.create_deployment() self.depl.logger.set_autoresponse("y") diff --git a/tests/functional/test_deleting_deletes.py b/tests/functional/test_deleting_deletes.py index df8c32bec..83ca9859f 100644 --- a/tests/functional/test_deleting_deletes.py +++ b/tests/functional/test_deleting_deletes.py @@ -7,4 +7,4 @@ class TestDeletingDeletes(single_machine_test.SingleMachineTest): def run_check(self): uuid = self.depl.uuid self.depl.delete() - tools.assert_raises(Exception, self.sf.open_deployment, (uuid,)) + tools.assert_raises(Exception, self.state.open_deployment, (uuid,)) diff --git a/tests/functional/test_query_deployments.py b/tests/functional/test_query_deployments.py index 88bc08760..c045586dc 100644 --- a/tests/functional/test_query_deployments.py +++ b/tests/functional/test_query_deployments.py @@ -6,7 +6,7 @@ class TestQueryDeployments(DatabaseUsingTest): def test_shows_all_deployments(self): depls = [] for i in range(10): - depls.append(self.sf.create_deployment()) - uuids = self.sf.query_deployments() + depls.append(self.state.create_deployment()) + uuids = self.state.query_deployments() for depl in depls: tools.assert_true(any([ depl.uuid == uuid for uuid in uuids ])) diff --git a/tests/functional/vpc.py b/tests/functional/vpc.py index b6cdb0c1e..25e778d6f 100644 --- a/tests/functional/vpc.py +++ b/tests/functional/vpc.py @@ -20,6 +20,7 @@ def setup(self): self.depl.nix_exprs = [ base_spec ] self.exprs_dir = nixops.util.SelfDeletingDir(tempfile.mkdtemp("nixos-tests")) + def test_deploy_vpc(self): self.depl.deploy() vpc_resource = self.depl.get_typed_resource("vpc-test", "vpc") From 36f24792118e853d684d27853ddd706615aa813f Mon Sep 17 00:00:00 2001 From: Matthew O'Gorman Date: Mon, 19 Feb 2018 02:06:26 -0500 Subject: [PATCH 077/123] make json match the behavior of sqlite --- nixops/state/json_file.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/nixops/state/json_file.py b/nixops/state/json_file.py index c3923f53a..8e0cd64ce 100644 --- a/nixops/state/json_file.py +++ b/nixops/state/json_file.py @@ -152,15 +152,19 @@ def get_all_deployments(self): def _find_deployment(self, uuid=None): all_deployments = self.db.read()["deployments"] found = [] + + # if nothing exists no reason to check for things + if not all_deployments: + return None + if not uuid: - found = all_deployments - if not found: + found = filter(lambda(id): all_deployments[id]["attributes"].get("name"), all_deployments) + else: found = filter(lambda(id): id == uuid, all_deployments) - if not found: - found = filter(lambda(id): all_deployments[id]["attributes"].get("name") == uuid, all_deployments) - - if not found: - found = filter(lambda(id): id.startswith(uuid), all_deployments) + if not found: + found = filter(lambda(id): all_deployments[id]["attributes"].get("name") == uuid, all_deployments) + if not found: + found = filter(lambda(id): id.startswith(uuid), all_deployments) if not found: return None From 0a836b6d19aacb04c11deba0c5b9dcbd3fb5f70f Mon Sep 17 00:00:00 2001 From: Matthew O'Gorman Date: Mon, 19 Feb 2018 04:36:24 -0500 Subject: [PATCH 078/123] misnamed variable --- nixops/state/json_file.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nixops/state/json_file.py b/nixops/state/json_file.py index 8e0cd64ce..ef8ff87d7 100644 --- a/nixops/state/json_file.py +++ b/nixops/state/json_file.py @@ -260,7 +260,7 @@ def get_deployment_attr(self, deployment_uuid, name): def get_all_deployment_attrs(self, deployment_uuid): with self.db: state = self.db.read() - return copy.deepcopy(state["deployments"][deployment]["attributes"]) + return copy.deepcopy(state["deployments"][deployment_uuid]["attributes"]) def get_deployment_lock(self, deployment): lock_dir = os.environ.get("HOME", "") + "/.nixops/locks" From 6be85a1c04608fc55344b337330b2f29ff1cdc67 Mon Sep 17 00:00:00 2001 From: srghma Date: Sat, 9 Jun 2018 20:32:58 +0300 Subject: [PATCH 079/123] fix: OperationalError no such column True -> quote booleans --- nixops/state/sqlite_connector.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/nixops/state/sqlite_connector.py b/nixops/state/sqlite_connector.py index ab07c879c..5dff118a7 100644 --- a/nixops/state/sqlite_connector.py +++ b/nixops/state/sqlite_connector.py @@ -251,18 +251,18 @@ def set_deployment_attrs(self, deployment_uuid, attrs): ) else: if self.get_deployment_attr(deployment_uuid, name) == nixops.util.undefined: - self.db.execute("insert into DeploymentAttrs(deployment, name, value) values ('{}', '{}', {!r})" + self.db.execute("insert into DeploymentAttrs(deployment, name, value) values ('{}', '{}', '{}')" .format(deployment_uuid, name, value) ) else: - self.db.execute("update DeploymentAttrs set value={!r} where deployment='{}' and name='{}'" + self.db.execute("update DeploymentAttrs set value='{}' where deployment='{}' and name='{}'" .format(value, deployment_uuid, name) ) def del_deployment_attr(self, deployment_uuid, attr_name): with self.db: - self.db.execute("delete from DeploymentAttrs where deployment = '{}' and name = {!r}" + self.db.execute("delete from DeploymentAttrs where deployment = '{}' and name = '{}'" .format(deployment_uuid, attr_name) ) @@ -313,13 +313,13 @@ def __exit__(self, exception_type, exception_value, exception_traceback): # ## Resources def create_resource(self, deployment, name, type): - count = self.db.execute("select count(id) from Resources where deployment = '{}' and name = {!r}" + count = self.db.execute("select count(id) from Resources where deployment = '{}' and name = '{}'" .format(deployment.uuid, name)).fetchone()[0] if count != 0: raise Exception("resource already exists in database!") - result = self.db.execute("insert into Resources(deployment, name, type) values ('{}', {!r}, {!r})" + result = self.db.execute("insert into Resources(deployment, name, type) values ('{}', '{}', '{}')" .format(deployment.uuid, name, type)) id = result.lastrowid @@ -329,14 +329,14 @@ def create_resource(self, deployment, name, type): def delete_resource(self, deployment_uuid, res_id): with self.db: - self.db.execute("delete from Resources where deployment = '{}' and id = {!r}" + self.db.execute("delete from Resources where deployment = '{}' and id = '{}'" .format(deployment_uuid, res_id)) def _rename_resource(self, deployment_uuid, resource_id, new_name): """NOTE: Invariants are checked in nixops/deployment.py#rename""" with self.db: - self.db.execute("update Resources set name = '{}' where deployment = '{}' and id = {!r}" + self.db.execute("update Resources set name = '{}' where deployment = '{}' and id = '{}'" .format(new_name, deployment_uuid, resource_id)) @@ -352,14 +352,14 @@ def set_resource_attrs(self, _deployment_uuid, resource_id, attrs): .format(resource_id, name, value) ) else: - self.db.execute("update ResourceAttrs set value={!r} where machine='{}' and name='{}'" + self.db.execute("update ResourceAttrs set value='{}' where machine='{}' and name='{}'" .format(value, resource_id, name) ) def del_resource_attr(self, _deployment_uuid, resource_id, name): with self.db: - self.db.execute("delete from ResourceAttrs where machine = {!r} and name = {!r}" + self.db.execute("delete from ResourceAttrs where machine = '{}' and name = '{}'" .format(resource_id, name)) From 666561ca5ac5da7d9fbc8a134a9354355f323c97 Mon Sep 17 00:00:00 2001 From: srghma Date: Sat, 9 Jun 2018 22:23:07 +0300 Subject: [PATCH 080/123] fix: OperationalError no such column True -> use sqlite execute escaping --- nixops/state/sqlite_connector.py | 207 +++++++++++++++++-------------- 1 file changed, 117 insertions(+), 90 deletions(-) diff --git a/nixops/state/sqlite_connector.py b/nixops/state/sqlite_connector.py index 5dff118a7..8c9042d8b 100644 --- a/nixops/state/sqlite_connector.py +++ b/nixops/state/sqlite_connector.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- - import nixops.deployment from nixops.state.state_helper import _subclasses import os @@ -14,11 +13,11 @@ class Connection(sqlite3.Connection): - def __init__(self, db_file, **kwargs): db_exists = os.path.exists(db_file) if not db_exists: - os.fdopen(os.open(db_file, os.O_WRONLY | os.O_CREAT, 0o600), 'w').close() + os.fdopen(os.open(db_file, os.O_WRONLY | os.O_CREAT, 0o600), + 'w').close() sqlite3.Connection.__init__(self, db_file, **kwargs) @@ -37,7 +36,6 @@ def __enter__(self): self.nesting = self.nesting + 1 sqlite3.Connection.__enter__(self) - def __exit__(self, exception_type, exception_value, exception_traceback): if exception_type != None: self.must_rollback = True self.nesting = self.nesting - 1 @@ -49,7 +47,8 @@ def __exit__(self, exception_type, exception_value, exception_traceback): except sqlite3.ProgrammingError: pass else: - sqlite3.Connection.__exit__(self, exception_type, exception_value, exception_traceback) + sqlite3.Connection.__exit__( + self, exception_type, exception_value, exception_traceback) self.lock.release() @@ -63,8 +62,15 @@ def __init__(self, db_file): self.db_file = url.netloc + url.path if os.path.splitext(db_file)[1] not in ['.nixops', '.charon']: - raise Exception("state file ‘{0}’ should have extension ‘.nixops’".format(db_file)) - db = sqlite3.connect(self.db_file, timeout=60, check_same_thread=False, factory=Connection, isolation_level=None) # FIXME + raise Exception( + "state file ‘{0}’ should have extension ‘.nixops’".format( + db_file)) + db = sqlite3.connect( + self.db_file, + timeout=60, + check_same_thread=False, + factory=Connection, + isolation_level=None) # FIXME db.db_file = db_file db.execute("pragma journal_mode = wal") @@ -76,7 +82,7 @@ def __init__(self, db_file): c = db.cursor() # Get the schema version. - version = 0 # new database + version = 0 # new database if self._table_exists(c, 'SchemaVersion'): c.execute("select version from SchemaVersion") version = c.fetchone()[0] @@ -90,9 +96,12 @@ def __init__(self, db_file): elif version < self.current_schema: if version <= 1: self._upgrade_1_to_2(c) if version <= 2: self._upgrade_2_to_3(c) - c.execute("update SchemaVersion set version = ?", (self.current_schema,)) + c.execute("update SchemaVersion set version = ?", + (self.current_schema, )) else: - raise Exception("this NixOps version is too old to deal with schema version {0}".format(version)) + raise Exception( + "this NixOps version is too old to deal with schema version {0}". + format(version)) self.db = db @@ -114,7 +123,8 @@ def get_all_deployments(self): try: res.append(self.open_deployment(uuid=uuid)) except nixops.deployment.UnknownBackend as e: - sys.stderr.write("skipping deployment ‘{0}’: {1}\n".format(uuid, str(e))) + sys.stderr.write("skipping deployment ‘{0}’: {1}\n".format( + uuid, str(e))) return res def _find_deployment(self, uuid=None): @@ -122,12 +132,15 @@ def _find_deployment(self, uuid=None): if not uuid: c.execute("select uuid from Deployments") else: - c.execute("select uuid from Deployments d where uuid = ? or exists (select 1 from DeploymentAttrs where deployment = d.uuid and name = 'name' and value = ?)", (uuid, uuid)) + c.execute( + "select uuid from Deployments d where uuid = ? or exists (select 1 from DeploymentAttrs where deployment = d.uuid and name = 'name' and value = ?)", + (uuid, uuid)) res = c.fetchall() if len(res) == 0: if uuid: # try the prefix match - c.execute("select uuid from Deployments where uuid glob ?", (uuid + '*', )) + c.execute("select uuid from Deployments where uuid glob ?", + (uuid + '*', )) res = c.fetchall() if len(res) == 0: return None @@ -135,16 +148,22 @@ def _find_deployment(self, uuid=None): return None if len(res) > 1: if uuid: - raise Exception("state file contains multiple deployments with the same name, so you should specify one using its UUID") + raise Exception( + "state file contains multiple deployments with the same name, so you should specify one using its UUID" + ) else: - raise Exception("state file contains multiple deployments, so you should specify which one to use using ‘-d’, or set the environment variable NIXOPS_DEPLOYMENT") + raise Exception( + "state file contains multiple deployments, so you should specify which one to use using ‘-d’, or set the environment variable NIXOPS_DEPLOYMENT" + ) return nixops.deployment.Deployment(self, res[0][0], sys.stderr) def open_deployment(self, uuid=None): """Open an existing deployment.""" deployment = self._find_deployment(uuid=uuid) if deployment: return deployment - raise Exception("could not find specified deployment in state file ‘{0}’".format(self.db_file)) + raise Exception( + "could not find specified deployment in state file ‘{0}’".format( + self.db_file)) def create_deployment(self, uuid=None): """Create a new deployment.""" @@ -152,47 +171,48 @@ def create_deployment(self, uuid=None): import uuid uuid = str(uuid.uuid1()) with self.db: - self.db.execute("insert into Deployments(uuid) values (?)", (uuid,)) + self.db.execute("insert into Deployments(uuid) values (?)", + (uuid, )) return nixops.deployment.Deployment(self, uuid, sys.stderr) def _table_exists(self, c, table): - c.execute("select 1 from sqlite_master where name = ? and type='table'", (table,)); + c.execute( + "select 1 from sqlite_master where name = ? and type='table'", + (table, )) return c.fetchone() != None def _delete_deployment(self, deployment_uuid): """NOTE: This is UNSAFE, it's guarded in nixops/deployment.py. Do not call this function except from there!""" with self.db: - self.db.execute("delete from Deployments where uuid = '{}'".format(deployment_uuid)) + self.db.execute("delete from Deployments where uuid = ?", + (deployment_uuid, )) def clone_deployment(self, deployment_uuid): with self.db: new = self.create_deployment() - self.db.execute("insert into DeploymentAttrs (deployment, name, value) " + - "select '{}', name, value from DeploymentAttrs where deployment = '{}'" - .format(new.uuid, deployment_uuid) - ) + self.db.execute( + "insert into DeploymentAttrs (deployment, name, value) " + + "select ?, name, value from DeploymentAttrs where deployment = ?", + (new.uuid, deployment_uuid)) new.configs_path = None return new - def _create_schemaversion(self, c): - c.execute( - '''create table if not exists SchemaVersion( + c.execute('''create table if not exists SchemaVersion( version integer not null );''') - c.execute("insert into SchemaVersion(version) values (?)", (self.current_schema,)) + c.execute("insert into SchemaVersion(version) values (?)", + (self.current_schema, )) def _create_schema(self, c): self._create_schemaversion(c) - c.execute( - '''create table if not exists Deployments( + c.execute('''create table if not exists Deployments( uuid text primary key );''') - c.execute( - '''create table if not exists DeploymentAttrs( + c.execute('''create table if not exists DeploymentAttrs( deployment text not null, name text not null, value text not null, @@ -200,8 +220,7 @@ def _create_schema(self, c): foreign key(deployment) references Deployments(uuid) on delete cascade );''') - c.execute( - '''create table if not exists Resources( + c.execute('''create table if not exists Resources( id integer primary key autoincrement, deployment text not null, name text not null, @@ -209,8 +228,7 @@ def _create_schema(self, c): foreign key(deployment) references Deployments(uuid) on delete cascade );''') - c.execute( - '''create table if not exists ResourceAttrs( + c.execute('''create table if not exists ResourceAttrs( machine integer not null, name text not null, value text not null, @@ -229,71 +247,74 @@ def _upgrade_2_to_3(self, c): # ############################################################################################### # ## Deployment + def get_resources_for(self, deployment): """Get all the resources for a certain deployment""" with self.db: resources = {} - rows = self.db.execute("select id, name, type from Resources where deployment = '{}'".format(deployment.uuid)).fetchall() + rows = self.db.execute( + "select id, name, type from Resources where deployment = ?", + (deployment.uuid, )).fetchall() for (id, name, type) in rows: r = self._create_state(deployment, type, name, id) resources[name] = r return resources - def set_deployment_attrs(self, deployment_uuid, attrs): """Update deployment attributes in the state.""" with self.db: for name, value in attrs.iteritems(): if value == None: - self.db.execute("delete from DeploymentAttrs where deployment = '{}' and name = '{}'" - .format(deployment_uuid, name) - ) + self.db.execute( + "delete from DeploymentAttrs where deployment = ? and name = ?", + (deployment_uuid, name)) else: - if self.get_deployment_attr(deployment_uuid, name) == nixops.util.undefined: - self.db.execute("insert into DeploymentAttrs(deployment, name, value) values ('{}', '{}', '{}')" - .format(deployment_uuid, name, value) - ) + if self.get_deployment_attr(deployment_uuid, + name) == nixops.util.undefined: + self.db.execute( + "insert into DeploymentAttrs(deployment, name, value) values (?, ?, ?)", + (deployment_uuid, name, value)) else: - self.db.execute("update DeploymentAttrs set value='{}' where deployment='{}' and name='{}'" - .format(value, deployment_uuid, name) - ) - + self.db.execute( + "update DeploymentAttrs set value=? where deployment=? and name=?", + (value, deployment_uuid, name)) def del_deployment_attr(self, deployment_uuid, attr_name): with self.db: - self.db.execute("delete from DeploymentAttrs where deployment = '{}' and name = '{}'" - .format(deployment_uuid, attr_name) - ) - + self.db.execute( + "delete from DeploymentAttrs where deployment = ? and name = ?", + (deployment_uuid, attr_name)) def get_deployment_attr(self, deployment_uuid, name): """Get a deployment attribute from the state.""" with self.db: - rows = self.db.execute("select value from DeploymentAttrs where deployment = '{}' and name = {!r}" - .format(deployment_uuid, name)).fetchall() + rows = self.db.execute( + "select value from DeploymentAttrs where deployment = ? and name = ?", + (deployment_uuid, name)).fetchall() for row in rows: return row[0] return nixops.util.undefined - def get_all_deployment_attrs(self, deployment_uuid): with self.db: - rows = self.db.execute("select name, value from DeploymentAttrs where deployment = '{}'" - .format(deployment_uuid)).fetchall() + rows = self.db.execute( + "select name, value from DeploymentAttrs where deployment = ?", + (deployment_uuid, )).fetchall() res = {row[0]: row[1] for row in rows} return res - def get_deployment_lock(self, deployment): lock_dir = os.environ.get("HOME", "") + "/.nixops/locks" if not os.path.exists(lock_dir): os.makedirs(lock_dir, 0700) lock_file_path = lock_dir + "/" + deployment.uuid + class DeploymentLock(object): def __init__(self, logger, path): self._lock_file_path = path self._logger = logger self._lock_file = None + def __enter__(self): self._lock_file = open(self._lock_file_path, "w") fcntl.fcntl(self._lock_file, fcntl.F_SETFD, fcntl.FD_CLOEXEC) @@ -301,83 +322,88 @@ def __enter__(self): fcntl.flock(self._lock_file, fcntl.LOCK_EX | fcntl.LOCK_NB) except IOError: self._logger.log( - "waiting for exclusive deployment lock..." - ) + "waiting for exclusive deployment lock...") fcntl.flock(self._lock_file, fcntl.LOCK_EX) - def __exit__(self, exception_type, exception_value, exception_traceback): + + def __exit__(self, exception_type, exception_value, + exception_traceback): if self._lock_file: self._lock_file.close() + return DeploymentLock(deployment.logger, lock_file_path) # ############################################################################################### # ## Resources def create_resource(self, deployment, name, type): - count = self.db.execute("select count(id) from Resources where deployment = '{}' and name = '{}'" - .format(deployment.uuid, name)).fetchone()[0] + count = self.db.execute( + "select count(id) from Resources where deployment = ? and name = ?", + (deployment.uuid, name)).fetchone()[0] if count != 0: raise Exception("resource already exists in database!") - result = self.db.execute("insert into Resources(deployment, name, type) values ('{}', '{}', '{}')" - .format(deployment.uuid, name, type)) + result = self.db.execute( + "insert into Resources(deployment, name, type) values (?, ?, ?)", + (deployment.uuid, name, type)) id = result.lastrowid r = self._create_state(deployment, type, name, id) return r - def delete_resource(self, deployment_uuid, res_id): with self.db: - self.db.execute("delete from Resources where deployment = '{}' and id = '{}'" - .format(deployment_uuid, res_id)) - + self.db.execute( + "delete from Resources where deployment = ? and id = ?", + (deployment_uuid, res_id)) def _rename_resource(self, deployment_uuid, resource_id, new_name): """NOTE: Invariants are checked in nixops/deployment.py#rename""" with self.db: - self.db.execute("update Resources set name = '{}' where deployment = '{}' and id = '{}'" - .format(new_name, deployment_uuid, resource_id)) - + self.db.execute( + "update Resources set name = where deployment = and id = ", + (new_name, deployment_uuid, resource_id)) def set_resource_attrs(self, _deployment_uuid, resource_id, attrs): with self.db: for name, value in attrs.iteritems(): if value == None: - self.db.execute("delete from ResourceAttrs where machine = '{}' and name = '{}'" - .format(resource_id, name)) + self.db.execute( + "delete from ResourceAttrs where machine = ? and name = ?", + (resource_id, name)) else: - if self.get_resource_attr(_deployment_uuid, resource_id, name) == nixops.util.undefined: - self.db.execute("insert into ResourceAttrs(machine, name, value) values ('{}', '{}', '{}')" - .format(resource_id, name, value) - ) + if self.get_resource_attr(_deployment_uuid, resource_id, + name) == nixops.util.undefined: + self.db.execute( + "insert into ResourceAttrs(machine, name, value) values (?, ?, ?)", + (resource_id, name, value)) else: - self.db.execute("update ResourceAttrs set value='{}' where machine='{}' and name='{}'" - .format(value, resource_id, name) - ) - + self.db.execute( + "update ResourceAttrs set value=? where machine=? and name=?", + (value, resource_id, name)) def del_resource_attr(self, _deployment_uuid, resource_id, name): with self.db: - self.db.execute("delete from ResourceAttrs where machine = '{}' and name = '{}'" - .format(resource_id, name)) - + self.db.execute( + "delete from ResourceAttrs where machine = ? and name = ?", + (resource_id, name)) def get_resource_attr(self, _deployment_uuid, resource_id, name): with self.db: """Get a machine attribute from the state file.""" - rows = self.db.execute("select value from ResourceAttrs where machine = '{}' and name = '{}'" - .format(resource_id, name)).fetchall() + rows = self.db.execute( + "select value from ResourceAttrs where machine = ? and name = ?", + (resource_id, name)).fetchall() for row in rows: return row[0] return nixops.util.undefined - def get_all_resource_attrs(self, deployment_uuid, resource_id): with self.db: - rows = self.db.execute("select name, value from ResourceAttrs where machine = '{}'" - .format(resource_id)).fetchall() + rows = self.db.execute( + "select name, value from ResourceAttrs where machine = ?", + (resource_id, )).fetchall() res = {row[0]: row[1] for row in rows} return res @@ -393,3 +419,4 @@ def _create_state(self, depl, type, name, id): raise nixops.deployment.UnknownBackend("unknown resource type ‘{!r}’" .format(type)) + From dee2de29f108c65ed5cab987e51b0eb4ce896158 Mon Sep 17 00:00:00 2001 From: srghma Date: Sun, 10 Jun 2018 15:52:38 +0300 Subject: [PATCH 081/123] refactor: tests -> move --- .../{ => all/test_cloning_clones}/test_cloning_clones.py | 0 .../{ => all/test_deleting_deletes}/test_deleting_deletes.py | 0 .../functional/{ => all/test_deploys_nixos}/test_deploys_nixos.py | 0 .../test_deploys_spot_instance}/test_deploys_spot_instance.py | 0 .../{ => all/test_rebooting_reboots}/test_rebooting_reboots.py | 0 .../{ => all/test_rollback_rollsback}/test_rollback_rollsback.py | 0 .../test_send_keys_sends_keys}/test_send_keys_sends_keys.py | 0 .../{ => all/test_starting_starts}/test_starting_starts.py | 0 .../{ => all/test_stopping_stops}/test_stopping_stops.py | 0 tests/functional/{ => ec2/test_backups}/test_backups.py | 0 .../test_ec2_rds_dbinstance}/ec2-rds-dbinstance-with-sg.nix | 0 .../{ => ec2/test_ec2_rds_dbinstance}/ec2-rds-dbinstance.nix | 0 .../{ => ec2/test_ec2_rds_dbinstance}/test_ec2_rds_dbinstance.py | 0 ...neric_deployment_test.py => generic_sqlite_deployment_test.py} | 0 .../{ => other/test_invalid_indenrifier}/invalid-identifier.nix | 0 .../test_invalid_indenrifier}/test_invalid_identifier.py | 0 .../{ => other/test_query_deployments}/test_query_deployments.py | 0 tests/functional/{vpc.nix => other/test_vpc/test_vpc.nix} | 0 tests/functional/{vpc.py => other/test_vpc/test_vpc.py} | 0 .../functional/{ => single_machine}/single_machine_azure_base.nix | 0 tests/functional/{ => single_machine}/single_machine_ec2_base.nix | 0 tests/functional/{ => single_machine}/single_machine_ec2_ebs.nix | 0 .../functional/{ => single_machine}/single_machine_ec2_raid-0.nix | 0 .../{ => single_machine}/single_machine_ec2_spot_instance.nix | 0 .../{ => single_machine}/single_machine_elsewhere_key.nix | 0 tests/functional/{ => single_machine}/single_machine_gce_base.nix | 0 .../functional/{ => single_machine}/single_machine_has_hello.nix | 0 .../{ => single_machine}/single_machine_libvirtd_base.nix | 0 .../{ => single_machine}/single_machine_logical_base.nix | 0 tests/functional/{ => single_machine}/single_machine_rollback.nix | 0 .../functional/{ => single_machine}/single_machine_secret_key.nix | 0 tests/functional/{ => single_machine}/single_machine_test.py | 0 .../functional/{ => single_machine}/single_machine_vbox_base.nix | 0 .../{ => virtualbox/ test_encrypted_links}/encrypted-links.nix | 0 .../ test_encrypted_links}/test_encrypted_links.py | 0 35 files changed, 0 insertions(+), 0 deletions(-) rename tests/functional/{ => all/test_cloning_clones}/test_cloning_clones.py (100%) rename tests/functional/{ => all/test_deleting_deletes}/test_deleting_deletes.py (100%) rename tests/functional/{ => all/test_deploys_nixos}/test_deploys_nixos.py (100%) rename tests/functional/{ => all/test_deploys_spot_instance}/test_deploys_spot_instance.py (100%) rename tests/functional/{ => all/test_rebooting_reboots}/test_rebooting_reboots.py (100%) rename tests/functional/{ => all/test_rollback_rollsback}/test_rollback_rollsback.py (100%) rename tests/functional/{ => all/test_send_keys_sends_keys}/test_send_keys_sends_keys.py (100%) rename tests/functional/{ => all/test_starting_starts}/test_starting_starts.py (100%) rename tests/functional/{ => all/test_stopping_stops}/test_stopping_stops.py (100%) rename tests/functional/{ => ec2/test_backups}/test_backups.py (100%) rename tests/functional/{ => ec2/test_ec2_rds_dbinstance}/ec2-rds-dbinstance-with-sg.nix (100%) rename tests/functional/{ => ec2/test_ec2_rds_dbinstance}/ec2-rds-dbinstance.nix (100%) rename tests/functional/{ => ec2/test_ec2_rds_dbinstance}/test_ec2_rds_dbinstance.py (100%) rename tests/functional/{generic_deployment_test.py => generic_sqlite_deployment_test.py} (100%) rename tests/functional/{ => other/test_invalid_indenrifier}/invalid-identifier.nix (100%) rename tests/functional/{ => other/test_invalid_indenrifier}/test_invalid_identifier.py (100%) rename tests/functional/{ => other/test_query_deployments}/test_query_deployments.py (100%) rename tests/functional/{vpc.nix => other/test_vpc/test_vpc.nix} (100%) rename tests/functional/{vpc.py => other/test_vpc/test_vpc.py} (100%) rename tests/functional/{ => single_machine}/single_machine_azure_base.nix (100%) rename tests/functional/{ => single_machine}/single_machine_ec2_base.nix (100%) rename tests/functional/{ => single_machine}/single_machine_ec2_ebs.nix (100%) rename tests/functional/{ => single_machine}/single_machine_ec2_raid-0.nix (100%) rename tests/functional/{ => single_machine}/single_machine_ec2_spot_instance.nix (100%) rename tests/functional/{ => single_machine}/single_machine_elsewhere_key.nix (100%) rename tests/functional/{ => single_machine}/single_machine_gce_base.nix (100%) rename tests/functional/{ => single_machine}/single_machine_has_hello.nix (100%) rename tests/functional/{ => single_machine}/single_machine_libvirtd_base.nix (100%) rename tests/functional/{ => single_machine}/single_machine_logical_base.nix (100%) rename tests/functional/{ => single_machine}/single_machine_rollback.nix (100%) rename tests/functional/{ => single_machine}/single_machine_secret_key.nix (100%) rename tests/functional/{ => single_machine}/single_machine_test.py (100%) rename tests/functional/{ => single_machine}/single_machine_vbox_base.nix (100%) rename tests/functional/{ => virtualbox/ test_encrypted_links}/encrypted-links.nix (100%) rename tests/functional/{ => virtualbox/ test_encrypted_links}/test_encrypted_links.py (100%) diff --git a/tests/functional/test_cloning_clones.py b/tests/functional/all/test_cloning_clones/test_cloning_clones.py similarity index 100% rename from tests/functional/test_cloning_clones.py rename to tests/functional/all/test_cloning_clones/test_cloning_clones.py diff --git a/tests/functional/test_deleting_deletes.py b/tests/functional/all/test_deleting_deletes/test_deleting_deletes.py similarity index 100% rename from tests/functional/test_deleting_deletes.py rename to tests/functional/all/test_deleting_deletes/test_deleting_deletes.py diff --git a/tests/functional/test_deploys_nixos.py b/tests/functional/all/test_deploys_nixos/test_deploys_nixos.py similarity index 100% rename from tests/functional/test_deploys_nixos.py rename to tests/functional/all/test_deploys_nixos/test_deploys_nixos.py diff --git a/tests/functional/test_deploys_spot_instance.py b/tests/functional/all/test_deploys_spot_instance/test_deploys_spot_instance.py similarity index 100% rename from tests/functional/test_deploys_spot_instance.py rename to tests/functional/all/test_deploys_spot_instance/test_deploys_spot_instance.py diff --git a/tests/functional/test_rebooting_reboots.py b/tests/functional/all/test_rebooting_reboots/test_rebooting_reboots.py similarity index 100% rename from tests/functional/test_rebooting_reboots.py rename to tests/functional/all/test_rebooting_reboots/test_rebooting_reboots.py diff --git a/tests/functional/test_rollback_rollsback.py b/tests/functional/all/test_rollback_rollsback/test_rollback_rollsback.py similarity index 100% rename from tests/functional/test_rollback_rollsback.py rename to tests/functional/all/test_rollback_rollsback/test_rollback_rollsback.py diff --git a/tests/functional/test_send_keys_sends_keys.py b/tests/functional/all/test_send_keys_sends_keys/test_send_keys_sends_keys.py similarity index 100% rename from tests/functional/test_send_keys_sends_keys.py rename to tests/functional/all/test_send_keys_sends_keys/test_send_keys_sends_keys.py diff --git a/tests/functional/test_starting_starts.py b/tests/functional/all/test_starting_starts/test_starting_starts.py similarity index 100% rename from tests/functional/test_starting_starts.py rename to tests/functional/all/test_starting_starts/test_starting_starts.py diff --git a/tests/functional/test_stopping_stops.py b/tests/functional/all/test_stopping_stops/test_stopping_stops.py similarity index 100% rename from tests/functional/test_stopping_stops.py rename to tests/functional/all/test_stopping_stops/test_stopping_stops.py diff --git a/tests/functional/test_backups.py b/tests/functional/ec2/test_backups/test_backups.py similarity index 100% rename from tests/functional/test_backups.py rename to tests/functional/ec2/test_backups/test_backups.py diff --git a/tests/functional/ec2-rds-dbinstance-with-sg.nix b/tests/functional/ec2/test_ec2_rds_dbinstance/ec2-rds-dbinstance-with-sg.nix similarity index 100% rename from tests/functional/ec2-rds-dbinstance-with-sg.nix rename to tests/functional/ec2/test_ec2_rds_dbinstance/ec2-rds-dbinstance-with-sg.nix diff --git a/tests/functional/ec2-rds-dbinstance.nix b/tests/functional/ec2/test_ec2_rds_dbinstance/ec2-rds-dbinstance.nix similarity index 100% rename from tests/functional/ec2-rds-dbinstance.nix rename to tests/functional/ec2/test_ec2_rds_dbinstance/ec2-rds-dbinstance.nix diff --git a/tests/functional/test_ec2_rds_dbinstance.py b/tests/functional/ec2/test_ec2_rds_dbinstance/test_ec2_rds_dbinstance.py similarity index 100% rename from tests/functional/test_ec2_rds_dbinstance.py rename to tests/functional/ec2/test_ec2_rds_dbinstance/test_ec2_rds_dbinstance.py diff --git a/tests/functional/generic_deployment_test.py b/tests/functional/generic_sqlite_deployment_test.py similarity index 100% rename from tests/functional/generic_deployment_test.py rename to tests/functional/generic_sqlite_deployment_test.py diff --git a/tests/functional/invalid-identifier.nix b/tests/functional/other/test_invalid_indenrifier/invalid-identifier.nix similarity index 100% rename from tests/functional/invalid-identifier.nix rename to tests/functional/other/test_invalid_indenrifier/invalid-identifier.nix diff --git a/tests/functional/test_invalid_identifier.py b/tests/functional/other/test_invalid_indenrifier/test_invalid_identifier.py similarity index 100% rename from tests/functional/test_invalid_identifier.py rename to tests/functional/other/test_invalid_indenrifier/test_invalid_identifier.py diff --git a/tests/functional/test_query_deployments.py b/tests/functional/other/test_query_deployments/test_query_deployments.py similarity index 100% rename from tests/functional/test_query_deployments.py rename to tests/functional/other/test_query_deployments/test_query_deployments.py diff --git a/tests/functional/vpc.nix b/tests/functional/other/test_vpc/test_vpc.nix similarity index 100% rename from tests/functional/vpc.nix rename to tests/functional/other/test_vpc/test_vpc.nix diff --git a/tests/functional/vpc.py b/tests/functional/other/test_vpc/test_vpc.py similarity index 100% rename from tests/functional/vpc.py rename to tests/functional/other/test_vpc/test_vpc.py diff --git a/tests/functional/single_machine_azure_base.nix b/tests/functional/single_machine/single_machine_azure_base.nix similarity index 100% rename from tests/functional/single_machine_azure_base.nix rename to tests/functional/single_machine/single_machine_azure_base.nix diff --git a/tests/functional/single_machine_ec2_base.nix b/tests/functional/single_machine/single_machine_ec2_base.nix similarity index 100% rename from tests/functional/single_machine_ec2_base.nix rename to tests/functional/single_machine/single_machine_ec2_base.nix diff --git a/tests/functional/single_machine_ec2_ebs.nix b/tests/functional/single_machine/single_machine_ec2_ebs.nix similarity index 100% rename from tests/functional/single_machine_ec2_ebs.nix rename to tests/functional/single_machine/single_machine_ec2_ebs.nix diff --git a/tests/functional/single_machine_ec2_raid-0.nix b/tests/functional/single_machine/single_machine_ec2_raid-0.nix similarity index 100% rename from tests/functional/single_machine_ec2_raid-0.nix rename to tests/functional/single_machine/single_machine_ec2_raid-0.nix diff --git a/tests/functional/single_machine_ec2_spot_instance.nix b/tests/functional/single_machine/single_machine_ec2_spot_instance.nix similarity index 100% rename from tests/functional/single_machine_ec2_spot_instance.nix rename to tests/functional/single_machine/single_machine_ec2_spot_instance.nix diff --git a/tests/functional/single_machine_elsewhere_key.nix b/tests/functional/single_machine/single_machine_elsewhere_key.nix similarity index 100% rename from tests/functional/single_machine_elsewhere_key.nix rename to tests/functional/single_machine/single_machine_elsewhere_key.nix diff --git a/tests/functional/single_machine_gce_base.nix b/tests/functional/single_machine/single_machine_gce_base.nix similarity index 100% rename from tests/functional/single_machine_gce_base.nix rename to tests/functional/single_machine/single_machine_gce_base.nix diff --git a/tests/functional/single_machine_has_hello.nix b/tests/functional/single_machine/single_machine_has_hello.nix similarity index 100% rename from tests/functional/single_machine_has_hello.nix rename to tests/functional/single_machine/single_machine_has_hello.nix diff --git a/tests/functional/single_machine_libvirtd_base.nix b/tests/functional/single_machine/single_machine_libvirtd_base.nix similarity index 100% rename from tests/functional/single_machine_libvirtd_base.nix rename to tests/functional/single_machine/single_machine_libvirtd_base.nix diff --git a/tests/functional/single_machine_logical_base.nix b/tests/functional/single_machine/single_machine_logical_base.nix similarity index 100% rename from tests/functional/single_machine_logical_base.nix rename to tests/functional/single_machine/single_machine_logical_base.nix diff --git a/tests/functional/single_machine_rollback.nix b/tests/functional/single_machine/single_machine_rollback.nix similarity index 100% rename from tests/functional/single_machine_rollback.nix rename to tests/functional/single_machine/single_machine_rollback.nix diff --git a/tests/functional/single_machine_secret_key.nix b/tests/functional/single_machine/single_machine_secret_key.nix similarity index 100% rename from tests/functional/single_machine_secret_key.nix rename to tests/functional/single_machine/single_machine_secret_key.nix diff --git a/tests/functional/single_machine_test.py b/tests/functional/single_machine/single_machine_test.py similarity index 100% rename from tests/functional/single_machine_test.py rename to tests/functional/single_machine/single_machine_test.py diff --git a/tests/functional/single_machine_vbox_base.nix b/tests/functional/single_machine/single_machine_vbox_base.nix similarity index 100% rename from tests/functional/single_machine_vbox_base.nix rename to tests/functional/single_machine/single_machine_vbox_base.nix diff --git a/tests/functional/encrypted-links.nix b/tests/functional/virtualbox/ test_encrypted_links/encrypted-links.nix similarity index 100% rename from tests/functional/encrypted-links.nix rename to tests/functional/virtualbox/ test_encrypted_links/encrypted-links.nix diff --git a/tests/functional/test_encrypted_links.py b/tests/functional/virtualbox/ test_encrypted_links/test_encrypted_links.py similarity index 100% rename from tests/functional/test_encrypted_links.py rename to tests/functional/virtualbox/ test_encrypted_links/test_encrypted_links.py From b24e69a449eabe116860de8518b7b1f51d64d423 Mon Sep 17 00:00:00 2001 From: srghma Date: Sun, 10 Jun 2018 20:50:21 +0300 Subject: [PATCH 082/123] refactor: tests -> move files (WIP) --- .gitignore | 2 +- nixops/state/json_file.py | 10 +-- nixops/util.py | 2 + release.nix | 2 +- tests/__init__.py | 46 ------------ tests/functional/__init__.py | 24 ------- .../ec2/test_backups/test_backups.py | 45 ------------ .../generic_json_deployment_test.py | 12 ---- .../generic_sqlite_deployment_test.py | 12 ---- tests/functional/shared/__init__.py | 0 tests/functional/shared/create_deployment.py | 5 ++ .../shared/deployment_run_command.py | 4 ++ .../nix_expressions/azure_base.nix} | 0 .../nix_expressions/ec2_base.nix} | 0 .../nix_expressions/ec2_ebs.nix} | 0 .../nix_expressions/ec2_raid-0.nix} | 0 .../nix_expressions/ec2_spot_instance.nix} | 0 .../nix_expressions/elsewhere_key.nix} | 0 .../nix_expressions/gce_base.nix} | 0 .../nix_expressions/has_hello.nix} | 0 .../nix_expressions/libvirtd_base.nix} | 0 .../nix_expressions/logical_base.nix} | 0 .../nix_expressions/rollback.nix} | 0 .../nix_expressions/secret_key.nix} | 0 .../nix_expressions/vbox_base.nix} | 0 tests/functional/shared/using_state_file.py | 72 +++++++++++++++++++ .../single_machine/single_machine_test.py | 57 --------------- .../__init__.py} | 0 .../__init__.py} | 0 .../__init__.py} | 0 .../__init__.py} | 0 tests/functional/test_ec2_backups/__init__.py | 39 ++++++++++ .../__init__.py} | 0 .../ec2-rds-dbinstance-with-sg.nix | 0 .../ec2-rds-dbinstance.nix | 0 .../__init__.py} | 0 .../invalid-identifier.nix | 0 .../__init__.py} | 0 .../__init__.py} | 0 .../__init__.py} | 0 .../__init__.py} | 0 .../__init__.py} | 0 .../__init__.py} | 0 .../__init__.py} | 0 .../encrypted-links.nix | 0 .../test_vpc.py => test_vpc/__init__.py} | 0 .../{other => }/test_vpc/test_vpc.nix | 0 47 files changed, 130 insertions(+), 202 deletions(-) delete mode 100644 tests/functional/ec2/test_backups/test_backups.py delete mode 100644 tests/functional/generic_json_deployment_test.py delete mode 100644 tests/functional/generic_sqlite_deployment_test.py create mode 100644 tests/functional/shared/__init__.py create mode 100644 tests/functional/shared/create_deployment.py create mode 100644 tests/functional/shared/deployment_run_command.py rename tests/functional/{single_machine/single_machine_azure_base.nix => shared/nix_expressions/azure_base.nix} (100%) rename tests/functional/{single_machine/single_machine_ec2_base.nix => shared/nix_expressions/ec2_base.nix} (100%) rename tests/functional/{single_machine/single_machine_ec2_ebs.nix => shared/nix_expressions/ec2_ebs.nix} (100%) rename tests/functional/{single_machine/single_machine_ec2_raid-0.nix => shared/nix_expressions/ec2_raid-0.nix} (100%) rename tests/functional/{single_machine/single_machine_ec2_spot_instance.nix => shared/nix_expressions/ec2_spot_instance.nix} (100%) rename tests/functional/{single_machine/single_machine_elsewhere_key.nix => shared/nix_expressions/elsewhere_key.nix} (100%) rename tests/functional/{single_machine/single_machine_gce_base.nix => shared/nix_expressions/gce_base.nix} (100%) rename tests/functional/{single_machine/single_machine_has_hello.nix => shared/nix_expressions/has_hello.nix} (100%) rename tests/functional/{single_machine/single_machine_libvirtd_base.nix => shared/nix_expressions/libvirtd_base.nix} (100%) rename tests/functional/{single_machine/single_machine_logical_base.nix => shared/nix_expressions/logical_base.nix} (100%) rename tests/functional/{single_machine/single_machine_rollback.nix => shared/nix_expressions/rollback.nix} (100%) rename tests/functional/{single_machine/single_machine_secret_key.nix => shared/nix_expressions/secret_key.nix} (100%) rename tests/functional/{single_machine/single_machine_vbox_base.nix => shared/nix_expressions/vbox_base.nix} (100%) create mode 100644 tests/functional/shared/using_state_file.py delete mode 100644 tests/functional/single_machine/single_machine_test.py rename tests/functional/{all/test_cloning_clones/test_cloning_clones.py => test_cloning_clones/__init__.py} (100%) rename tests/functional/{all/test_deleting_deletes/test_deleting_deletes.py => test_deleting_deletes/__init__.py} (100%) rename tests/functional/{all/test_deploys_nixos/test_deploys_nixos.py => test_deploys_nixos/__init__.py} (100%) rename tests/functional/{all/test_deploys_spot_instance/test_deploys_spot_instance.py => test_deploys_spot_instance/__init__.py} (100%) create mode 100644 tests/functional/test_ec2_backups/__init__.py rename tests/functional/{ec2/test_ec2_rds_dbinstance/test_ec2_rds_dbinstance.py => test_ec2_rds_dbinstance/__init__.py} (100%) rename tests/functional/{ec2 => }/test_ec2_rds_dbinstance/ec2-rds-dbinstance-with-sg.nix (100%) rename tests/functional/{ec2 => }/test_ec2_rds_dbinstance/ec2-rds-dbinstance.nix (100%) rename tests/functional/{other/test_invalid_indenrifier/test_invalid_identifier.py => test_invalid_indentifier/__init__.py} (100%) rename tests/functional/{other/test_invalid_indenrifier => test_invalid_indentifier}/invalid-identifier.nix (100%) rename tests/functional/{other/test_query_deployments/test_query_deployments.py => test_query_deployments/__init__.py} (100%) rename tests/functional/{all/test_rebooting_reboots/test_rebooting_reboots.py => test_rebooting_reboots/__init__.py} (100%) rename tests/functional/{all/test_rollback_rollsback/test_rollback_rollsback.py => test_rollback_rollsback/__init__.py} (100%) rename tests/functional/{all/test_send_keys_sends_keys/test_send_keys_sends_keys.py => test_send_keys_sends_keys/__init__.py} (100%) rename tests/functional/{all/test_starting_starts/test_starting_starts.py => test_starting_starts/__init__.py} (100%) rename tests/functional/{all/test_stopping_stops/test_stopping_stops.py => test_stopping_stops/__init__.py} (100%) rename tests/functional/{virtualbox/ test_encrypted_links/test_encrypted_links.py => test_vbox_encrypted_links/__init__.py} (100%) rename tests/functional/{virtualbox/ test_encrypted_links => test_vbox_encrypted_links}/encrypted-links.nix (100%) rename tests/functional/{other/test_vpc/test_vpc.py => test_vpc/__init__.py} (100%) rename tests/functional/{other => }/test_vpc/test_vpc.nix (100%) diff --git a/.gitignore b/.gitignore index 1ee5c9759..c96657c2d 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,4 @@ html/ result *.sw[a-z] tags -tests/test.nixops* +tests/state_files diff --git a/nixops/state/json_file.py b/nixops/state/json_file.py index ef8ff87d7..8dec0b004 100644 --- a/nixops/state/json_file.py +++ b/nixops/state/json_file.py @@ -109,7 +109,7 @@ def _ensure_db_exists(self): def schema_version(self): version = self.read()["schemaVersion"] if version is None: - raise "illegal datafile" #TODO: proper exception + raise "illegal datafile" # TODO: proper exception else: return version @@ -124,12 +124,15 @@ def __init__(self, json_file): self.db = TransactionalJsonFile(json_file) - # Check that we're not using a to new DB schema version. + # Check that we're not using a new DB schema version. with self.db: version = self.db.schema_version() - if version > 0: + if version > 0: raise Exception("this NixOps version is too old to deal with JSON schema version {0}".format(version)) + def close(self): + pass + ############################################################################################### ## Deployment @@ -195,7 +198,6 @@ def create_deployment(self, uuid=None): def _delete_deployment(self, deployment_uuid): """NOTE: This is UNSAFE, it's guarded in nixops/deployment.py. Do not call this function except from there!""" - self.__db.execute("delete from Deployments where uuid = ?", (deployment_uuid,)) with self.db: state = self.db.read() state["deployments"].pop(deployment_uuid, None) diff --git a/nixops/util.py b/nixops/util.py index c0d399c01..8f85cfa41 100644 --- a/nixops/util.py +++ b/nixops/util.py @@ -384,3 +384,5 @@ def xml_expr_to_python(node): def parse_nixos_version(s): """Split a NixOS version string into a list of components.""" return s.split(".") + +root_dir = os.path.dirname(os.path.dirname(__file__)) diff --git a/release.nix b/release.nix index 53789c521..28bf53494 100644 --- a/release.nix +++ b/release.nix @@ -76,7 +76,7 @@ rec { src = "${tarball}/tarballs/*.tar.bz2"; - buildInputs = [ python2Packages.nose python2Packages.coverage ]; + buildInputs = with python2Packages; [ nose coverage parameterized ]; propagatedBuildInputs = with python2Packages; [ prettytable diff --git a/tests/__init__.py b/tests/__init__.py index 7b760ec24..e69de29bb 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,46 +0,0 @@ -# -*- coding: utf-8 -*- -import os -import sys -import threading -from os import path -import nixops.state - -_multiprocess_shared_ = True - -db_file = '%s/test.nixops' % (path.dirname(__file__)) -json_file = '%s/test.json' % (path.dirname(__file__)) - -def setup(): - state = nixops.state.open(db_file) - state.db.close() - -def destroy(state, uuid): - depl = state.open_deployment(uuid) - depl.logger.set_autoresponse("y") - try: - depl.clean_backups(keep=0) - except Exception: - pass - try: - depl.destroy_resources() - except Exception: - pass - depl.delete() - depl.logger.log("deployment ‘{0}’ destroyed".format(uuid)) - -def teardown(): - state = nixops.state.open(db_file) - uuids = state.query_deployments() - threads = [] - for uuid in uuids: - threads.append(threading.Thread(target=destroy,args=(state, uuid))) - for thread in threads: - thread.start() - for thread in threads: - thread.join() - uuids_left = state.query_deployments() - state.close() - if not uuids_left: - os.remove(db_file) - else: - sys.stderr.write("warning: not all deployments have been destroyed; some resources may still exist!\n") diff --git a/tests/functional/__init__.py b/tests/functional/__init__.py index 3c5d0c540..e69de29bb 100644 --- a/tests/functional/__init__.py +++ b/tests/functional/__init__.py @@ -1,24 +0,0 @@ -import os -from os import path -import nixops.state -from tests import db_file -from tests import json_file - -class DatabaseUsingTest(object): - _multiprocess_can_split_ = True - - def setup(self): - self.state = nixops.state.open(db_file) - - def teardown(self): - self.state.close() - - -class JSONUsingTest(object): - _multiprocess_can_split_ = True - - def setup(self): - self.state = nixops.state.open(json_file) - - def teardown(self): - self.state.close() diff --git a/tests/functional/ec2/test_backups/test_backups.py b/tests/functional/ec2/test_backups/test_backups.py deleted file mode 100644 index 279bb8c57..000000000 --- a/tests/functional/ec2/test_backups/test_backups.py +++ /dev/null @@ -1,45 +0,0 @@ -import time - -from os import path - -from nose import tools - -from tests.functional import generic_deployment_test - -parent_dir = path.dirname(__file__) - -logical_spec = '%s/single_machine_logical_base.nix' % (parent_dir) - -class TestBackups(generic_deployment_test.GenericDeploymentTest): - _multiprocess_can_split_ = True - - def setup(self): - super(TestBackups,self).setup() - self.depl.nix_exprs = [ logical_spec, - '%s/single_machine_ec2_ebs.nix' % (parent_dir), - '%s/single_machine_ec2_base.nix' % (parent_dir) - ] - - def backup_and_restore_path(self, path=""): - self.depl.deploy() - self.check_command("echo -n important-data > %s/back-me-up" % (path)) - backup_id = self.depl.backup() - backups = self.depl.get_backups() - while backups[backup_id]['status'] == "running": - time.sleep(10) - backups = self.depl.get_backups() - self.check_command("rm %s/back-me-up" % (path)) - self.depl.restore(backup_id=backup_id) - self.check_command("echo -n important-data | diff %s/back-me-up -" % (path)) - - def test_simple_restore(self): - self.backup_and_restore_path() - - def test_raid_restore(self): - self.depl.nix_exprs = self.depl.nix_exprs + [ '%s/single_machine_ec2_raid-0.nix' % (parent_dir) ] - self.backup_and_restore_path("/data") - - def check_command(self, command): - self.depl.evaluate() - machine = self.depl.machines.values()[0] - return machine.run_command(command) diff --git a/tests/functional/generic_json_deployment_test.py b/tests/functional/generic_json_deployment_test.py deleted file mode 100644 index ef6f3b04d..000000000 --- a/tests/functional/generic_json_deployment_test.py +++ /dev/null @@ -1,12 +0,0 @@ -import os -import subprocess - -from nose import SkipTest - -from tests.functional import JSONUsingTest - -class GenericJsonDeploymentTest(JSONUsingTest): - def setup(self): - super(GenericJsonDeploymentTest,self).setup() - self.depl = self.state.create_deployment() - self.depl.logger.set_autoresponse("y") diff --git a/tests/functional/generic_sqlite_deployment_test.py b/tests/functional/generic_sqlite_deployment_test.py deleted file mode 100644 index f40781423..000000000 --- a/tests/functional/generic_sqlite_deployment_test.py +++ /dev/null @@ -1,12 +0,0 @@ -import os -import subprocess - -from nose import SkipTest - -from tests.functional import DatabaseUsingTest - -class GenericDeploymentTest(DatabaseUsingTest): - def setup(self): - super(GenericDeploymentTest,self).setup() - self.depl = self.state.create_deployment() - self.depl.logger.set_autoresponse("y") diff --git a/tests/functional/shared/__init__.py b/tests/functional/shared/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/functional/shared/create_deployment.py b/tests/functional/shared/create_deployment.py new file mode 100644 index 000000000..67ac862eb --- /dev/null +++ b/tests/functional/shared/create_deployment.py @@ -0,0 +1,5 @@ +def create_deployment(state, nix_expressions): + deployment = state.create_deployment() + deployment.logger.set_autoresponse("y") + deployment.nix_exprs = nix_expressions + return deployment diff --git a/tests/functional/shared/deployment_run_command.py b/tests/functional/shared/deployment_run_command.py new file mode 100644 index 000000000..daeaa5812 --- /dev/null +++ b/tests/functional/shared/deployment_run_command.py @@ -0,0 +1,4 @@ +def deployment_run_command(deployment, command): + deployment.evaluate() + machine = deployment.machines.values()[0] + return machine.run_command(command) diff --git a/tests/functional/single_machine/single_machine_azure_base.nix b/tests/functional/shared/nix_expressions/azure_base.nix similarity index 100% rename from tests/functional/single_machine/single_machine_azure_base.nix rename to tests/functional/shared/nix_expressions/azure_base.nix diff --git a/tests/functional/single_machine/single_machine_ec2_base.nix b/tests/functional/shared/nix_expressions/ec2_base.nix similarity index 100% rename from tests/functional/single_machine/single_machine_ec2_base.nix rename to tests/functional/shared/nix_expressions/ec2_base.nix diff --git a/tests/functional/single_machine/single_machine_ec2_ebs.nix b/tests/functional/shared/nix_expressions/ec2_ebs.nix similarity index 100% rename from tests/functional/single_machine/single_machine_ec2_ebs.nix rename to tests/functional/shared/nix_expressions/ec2_ebs.nix diff --git a/tests/functional/single_machine/single_machine_ec2_raid-0.nix b/tests/functional/shared/nix_expressions/ec2_raid-0.nix similarity index 100% rename from tests/functional/single_machine/single_machine_ec2_raid-0.nix rename to tests/functional/shared/nix_expressions/ec2_raid-0.nix diff --git a/tests/functional/single_machine/single_machine_ec2_spot_instance.nix b/tests/functional/shared/nix_expressions/ec2_spot_instance.nix similarity index 100% rename from tests/functional/single_machine/single_machine_ec2_spot_instance.nix rename to tests/functional/shared/nix_expressions/ec2_spot_instance.nix diff --git a/tests/functional/single_machine/single_machine_elsewhere_key.nix b/tests/functional/shared/nix_expressions/elsewhere_key.nix similarity index 100% rename from tests/functional/single_machine/single_machine_elsewhere_key.nix rename to tests/functional/shared/nix_expressions/elsewhere_key.nix diff --git a/tests/functional/single_machine/single_machine_gce_base.nix b/tests/functional/shared/nix_expressions/gce_base.nix similarity index 100% rename from tests/functional/single_machine/single_machine_gce_base.nix rename to tests/functional/shared/nix_expressions/gce_base.nix diff --git a/tests/functional/single_machine/single_machine_has_hello.nix b/tests/functional/shared/nix_expressions/has_hello.nix similarity index 100% rename from tests/functional/single_machine/single_machine_has_hello.nix rename to tests/functional/shared/nix_expressions/has_hello.nix diff --git a/tests/functional/single_machine/single_machine_libvirtd_base.nix b/tests/functional/shared/nix_expressions/libvirtd_base.nix similarity index 100% rename from tests/functional/single_machine/single_machine_libvirtd_base.nix rename to tests/functional/shared/nix_expressions/libvirtd_base.nix diff --git a/tests/functional/single_machine/single_machine_logical_base.nix b/tests/functional/shared/nix_expressions/logical_base.nix similarity index 100% rename from tests/functional/single_machine/single_machine_logical_base.nix rename to tests/functional/shared/nix_expressions/logical_base.nix diff --git a/tests/functional/single_machine/single_machine_rollback.nix b/tests/functional/shared/nix_expressions/rollback.nix similarity index 100% rename from tests/functional/single_machine/single_machine_rollback.nix rename to tests/functional/shared/nix_expressions/rollback.nix diff --git a/tests/functional/single_machine/single_machine_secret_key.nix b/tests/functional/shared/nix_expressions/secret_key.nix similarity index 100% rename from tests/functional/single_machine/single_machine_secret_key.nix rename to tests/functional/shared/nix_expressions/secret_key.nix diff --git a/tests/functional/single_machine/single_machine_vbox_base.nix b/tests/functional/shared/nix_expressions/vbox_base.nix similarity index 100% rename from tests/functional/single_machine/single_machine_vbox_base.nix rename to tests/functional/shared/nix_expressions/vbox_base.nix diff --git a/tests/functional/shared/using_state_file.py b/tests/functional/shared/using_state_file.py new file mode 100644 index 000000000..d867530b7 --- /dev/null +++ b/tests/functional/shared/using_state_file.py @@ -0,0 +1,72 @@ +# -*- coding: utf-8 -*- + +from contextlib import contextmanager +import os +import sys +import threading +from distutils.dir_util import mkpath + +import nixops.state +from nixops.util import root_dir + +@contextmanager +def using_state_file(unique_name, state_extension): + state_file_path_ = state_file_path(unique_name, state_extension) + + create_file_parent_dirs_if_not_exists(state_file_path_) + + state = nixops.state.open(state_file_path_) + try: + yield state + finally: + state.close() + destroy_deployments_and_remove_state_file(state_file_path_) + + +def create_file_parent_dirs_if_not_exists(file_path): + mkpath(os.path.dirname(file_path)) + +def state_file_path(unique_name, state_extension): + unique_name_ = unique_name + '_' + state_extension + + return '{}/tests/state_files/{}/test.{}'.format( + root_dir, unique_name_, state_extension + ) + + +def destroy_deployments(state, uuid): + deployment = state.open_deployment(uuid) + deployment.logger.set_autoresponse("y") + try: + deployment.clean_backups(keep=0) + except Exception: + pass + try: + deployment.destroy_resources() + except Exception: + pass + deployment.delete() + deployment.logger.log("deployment ‘{0}’ destroyed".format(uuid)) + + +def destroy_deployments_and_remove_state_file(state_file_path): + state = nixops.state.open(state_file_path) + uuids = state.query_deployments() + threads = [] + for uuid in uuids: + threads.append( + threading.Thread(target=destroy_deployments, args=(state, uuid)) + ) + for thread in threads: + thread.start() + for thread in threads: + thread.join() + uuids_left = state.query_deployments() + state.close() + if not uuids_left: + os.remove(state_file_path) + else: + sys.stderr.write( + "warning: not all deployments have been destroyed; some resources may still exist!\n" + ) + diff --git a/tests/functional/single_machine/single_machine_test.py b/tests/functional/single_machine/single_machine_test.py deleted file mode 100644 index c3d795afd..000000000 --- a/tests/functional/single_machine/single_machine_test.py +++ /dev/null @@ -1,57 +0,0 @@ -from os import path - -from nose import tools -from nose.plugins.attrib import attr - -from tests.functional import generic_deployment_test - -parent_dir = path.dirname(__file__) - -logical_spec = '{0}/single_machine_logical_base.nix'.format(parent_dir) - -class SingleMachineTest(generic_deployment_test.GenericDeploymentTest): - _multiprocess_can_split_ = True - - def setup(self): - super(SingleMachineTest,self).setup() - self.depl.nix_exprs = [ logical_spec ] - - @attr("vbox") - def test_vbox(self): - self.depl.nix_exprs = self.depl.nix_exprs + [ - ('%s/single_machine_vbox_base.nix' % (parent_dir)) - ] - self.run_check() - - @attr("ec2") - def test_ec2(self): - self.depl.nix_exprs = self.depl.nix_exprs + [ - ('{0}/single_machine_ec2_base.nix'.format(parent_dir)) - ] - self.run_check() - - @attr("gce") - def test_gce(self): - self.depl.nix_exprs = self.depl.nix_exprs + [ - ('{0}/single_machine_gce_base.nix'.format(parent_dir)) - ] - self.run_check() - - @attr("azure") - def test_azure(self): - self.depl.nix_exprs = self.depl.nix_exprs + [ - ('{0}/single_machine_azure_base.nix'.format(parent_dir)) - ] - self.run_check() - - @attr("libvirtd") - def test_libvirtd(self): - self.depl.nix_exprs = self.depl.nix_exprs + [ - ('{0}/single_machine_libvirtd_base.nix'.format(parent_dir)) - ] - self.run_check() - - def check_command(self, command): - self.depl.evaluate() - machine = self.depl.machines.values()[0] - return machine.run_command(command) diff --git a/tests/functional/all/test_cloning_clones/test_cloning_clones.py b/tests/functional/test_cloning_clones/__init__.py similarity index 100% rename from tests/functional/all/test_cloning_clones/test_cloning_clones.py rename to tests/functional/test_cloning_clones/__init__.py diff --git a/tests/functional/all/test_deleting_deletes/test_deleting_deletes.py b/tests/functional/test_deleting_deletes/__init__.py similarity index 100% rename from tests/functional/all/test_deleting_deletes/test_deleting_deletes.py rename to tests/functional/test_deleting_deletes/__init__.py diff --git a/tests/functional/all/test_deploys_nixos/test_deploys_nixos.py b/tests/functional/test_deploys_nixos/__init__.py similarity index 100% rename from tests/functional/all/test_deploys_nixos/test_deploys_nixos.py rename to tests/functional/test_deploys_nixos/__init__.py diff --git a/tests/functional/all/test_deploys_spot_instance/test_deploys_spot_instance.py b/tests/functional/test_deploys_spot_instance/__init__.py similarity index 100% rename from tests/functional/all/test_deploys_spot_instance/test_deploys_spot_instance.py rename to tests/functional/test_deploys_spot_instance/__init__.py diff --git a/tests/functional/test_ec2_backups/__init__.py b/tests/functional/test_ec2_backups/__init__.py new file mode 100644 index 000000000..97f9dc429 --- /dev/null +++ b/tests/functional/test_ec2_backups/__init__.py @@ -0,0 +1,39 @@ +import time + +from nixops.util import root_dir +from itertools import product +from nose.tools import with_setup +from parameterized import parameterized + +from tests.functional.shared.deployment_run_command import deployment_run_command +from tests.functional.shared.create_deployment import create_deployment +from tests.functional.shared.using_state_file import using_state_file + +@parameterized(product( + # ['json', 'nixops'], + ['json'], + [ + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/ec2_ebs.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/ec2_base.nix'.format(root_dir) + ] + ], + [''] +)) +def test_ec2_backups(state_extension, nix_expressions, backup_path): + with using_state_file( + unique_name='test_ec2_backups', + state_extension=state_extension) as state: + deployment = create_deployment(state, nix_expressions) + + deployment.deploy() + deployment_run_command(deployment, "echo -n important-data > {}/back-me-up".format(backup_path)) + backup_id = deployment.backup() + backups = deployment.get_backups() + while backups[backup_id]['status'] == "running": + time.sleep(10) + backups = deployment.get_backups() + deployment_run_command(deployment, "rm {}/back-me-up".format(backup_path)) + deployment.restore(backup_id=backup_id) + deployment_run_command(deployment, "echo -n important-data | diff {}/back-me-up -".format(backup_path)) diff --git a/tests/functional/ec2/test_ec2_rds_dbinstance/test_ec2_rds_dbinstance.py b/tests/functional/test_ec2_rds_dbinstance/__init__.py similarity index 100% rename from tests/functional/ec2/test_ec2_rds_dbinstance/test_ec2_rds_dbinstance.py rename to tests/functional/test_ec2_rds_dbinstance/__init__.py diff --git a/tests/functional/ec2/test_ec2_rds_dbinstance/ec2-rds-dbinstance-with-sg.nix b/tests/functional/test_ec2_rds_dbinstance/ec2-rds-dbinstance-with-sg.nix similarity index 100% rename from tests/functional/ec2/test_ec2_rds_dbinstance/ec2-rds-dbinstance-with-sg.nix rename to tests/functional/test_ec2_rds_dbinstance/ec2-rds-dbinstance-with-sg.nix diff --git a/tests/functional/ec2/test_ec2_rds_dbinstance/ec2-rds-dbinstance.nix b/tests/functional/test_ec2_rds_dbinstance/ec2-rds-dbinstance.nix similarity index 100% rename from tests/functional/ec2/test_ec2_rds_dbinstance/ec2-rds-dbinstance.nix rename to tests/functional/test_ec2_rds_dbinstance/ec2-rds-dbinstance.nix diff --git a/tests/functional/other/test_invalid_indenrifier/test_invalid_identifier.py b/tests/functional/test_invalid_indentifier/__init__.py similarity index 100% rename from tests/functional/other/test_invalid_indenrifier/test_invalid_identifier.py rename to tests/functional/test_invalid_indentifier/__init__.py diff --git a/tests/functional/other/test_invalid_indenrifier/invalid-identifier.nix b/tests/functional/test_invalid_indentifier/invalid-identifier.nix similarity index 100% rename from tests/functional/other/test_invalid_indenrifier/invalid-identifier.nix rename to tests/functional/test_invalid_indentifier/invalid-identifier.nix diff --git a/tests/functional/other/test_query_deployments/test_query_deployments.py b/tests/functional/test_query_deployments/__init__.py similarity index 100% rename from tests/functional/other/test_query_deployments/test_query_deployments.py rename to tests/functional/test_query_deployments/__init__.py diff --git a/tests/functional/all/test_rebooting_reboots/test_rebooting_reboots.py b/tests/functional/test_rebooting_reboots/__init__.py similarity index 100% rename from tests/functional/all/test_rebooting_reboots/test_rebooting_reboots.py rename to tests/functional/test_rebooting_reboots/__init__.py diff --git a/tests/functional/all/test_rollback_rollsback/test_rollback_rollsback.py b/tests/functional/test_rollback_rollsback/__init__.py similarity index 100% rename from tests/functional/all/test_rollback_rollsback/test_rollback_rollsback.py rename to tests/functional/test_rollback_rollsback/__init__.py diff --git a/tests/functional/all/test_send_keys_sends_keys/test_send_keys_sends_keys.py b/tests/functional/test_send_keys_sends_keys/__init__.py similarity index 100% rename from tests/functional/all/test_send_keys_sends_keys/test_send_keys_sends_keys.py rename to tests/functional/test_send_keys_sends_keys/__init__.py diff --git a/tests/functional/all/test_starting_starts/test_starting_starts.py b/tests/functional/test_starting_starts/__init__.py similarity index 100% rename from tests/functional/all/test_starting_starts/test_starting_starts.py rename to tests/functional/test_starting_starts/__init__.py diff --git a/tests/functional/all/test_stopping_stops/test_stopping_stops.py b/tests/functional/test_stopping_stops/__init__.py similarity index 100% rename from tests/functional/all/test_stopping_stops/test_stopping_stops.py rename to tests/functional/test_stopping_stops/__init__.py diff --git a/tests/functional/virtualbox/ test_encrypted_links/test_encrypted_links.py b/tests/functional/test_vbox_encrypted_links/__init__.py similarity index 100% rename from tests/functional/virtualbox/ test_encrypted_links/test_encrypted_links.py rename to tests/functional/test_vbox_encrypted_links/__init__.py diff --git a/tests/functional/virtualbox/ test_encrypted_links/encrypted-links.nix b/tests/functional/test_vbox_encrypted_links/encrypted-links.nix similarity index 100% rename from tests/functional/virtualbox/ test_encrypted_links/encrypted-links.nix rename to tests/functional/test_vbox_encrypted_links/encrypted-links.nix diff --git a/tests/functional/other/test_vpc/test_vpc.py b/tests/functional/test_vpc/__init__.py similarity index 100% rename from tests/functional/other/test_vpc/test_vpc.py rename to tests/functional/test_vpc/__init__.py diff --git a/tests/functional/other/test_vpc/test_vpc.nix b/tests/functional/test_vpc/test_vpc.nix similarity index 100% rename from tests/functional/other/test_vpc/test_vpc.nix rename to tests/functional/test_vpc/test_vpc.nix From d92a9cb9ffc0ff06c245ef60330272f32a082271 Mon Sep 17 00:00:00 2001 From: srghma Date: Sun, 10 Jun 2018 21:41:26 +0300 Subject: [PATCH 083/123] refactor: tests -> test_ec2_backups stops without error, but "mux_client_request_session: read from master failed: Broken pipe" (WIP) --- nixops/state/json_file.py | 10 ++--- tests/functional/test_ec2_backups/__init__.py | 44 +++++++++++++------ 2 files changed, 36 insertions(+), 18 deletions(-) diff --git a/nixops/state/json_file.py b/nixops/state/json_file.py index 8dec0b004..96da7c680 100644 --- a/nixops/state/json_file.py +++ b/nixops/state/json_file.py @@ -35,7 +35,7 @@ def __init__(self, db_file): self._lock_file = open(lock_file_path, "w") fcntl.fcntl(self._lock_file, fcntl.F_SETFD, fcntl.FD_CLOEXEC) # to not keep the lock in child processes - self._db_file = db_file + self.db_file = db_file self.nesting = 0 self.lock = threading.RLock() @@ -45,7 +45,7 @@ def __init__(self, db_file): def read(self): if self.nesting == 0: - with open(self._db_file,"r") as f: + with open(self.db_file,"r") as f: return json.load(f) else: assert self.nesting > 0 @@ -88,21 +88,21 @@ def _commit(self): assert self.nesting == 0 # TODO: write to temp file, then mv - with open(self._db_file, "w") as f: + with open(self.db_file, "w") as f: json.dump(self._current_state, f,indent=2) self._backup_state = None self._current_state = None def _ensure_db_exists(self): - db_exists = os.path.exists(self._db_file) + db_exists = os.path.exists(self.db_file) if not db_exists: initial_db = { "schemaVersion": 0, "deployments": {} } - with open(self._db_file, "w", 0o600) as f: + with open(self.db_file, "w", 0o600) as f: json.dump(initial_db, f) f.close() diff --git a/tests/functional/test_ec2_backups/__init__.py b/tests/functional/test_ec2_backups/__init__.py index 97f9dc429..c9447b213 100644 --- a/tests/functional/test_ec2_backups/__init__.py +++ b/tests/functional/test_ec2_backups/__init__.py @@ -10,8 +10,7 @@ from tests.functional.shared.using_state_file import using_state_file @parameterized(product( - # ['json', 'nixops'], - ['json'], + ['json', 'nixops'], [ [ '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), @@ -19,21 +18,40 @@ '{}/tests/functional/shared/nix_expressions/ec2_base.nix'.format(root_dir) ] ], - [''] )) -def test_ec2_backups(state_extension, nix_expressions, backup_path): +def test_ec2_backups_simple(state_extension, nix_expressions): with using_state_file( unique_name='test_ec2_backups', state_extension=state_extension) as state: deployment = create_deployment(state, nix_expressions) + backup_and_restore_path(deployment) - deployment.deploy() - deployment_run_command(deployment, "echo -n important-data > {}/back-me-up".format(backup_path)) - backup_id = deployment.backup() +@parameterized(product( + ['json', 'nixops'], + [ + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/ec2_ebs.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/ec2_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/ec2_raid-0.nix'.format(root_dir) + ] + ], +)) +def test_ec2_backups_raid(state_extension, nix_expressions): + with using_state_file( + unique_name='test_ec2_backups', + state_extension=state_extension) as state: + deployment = create_deployment(state, nix_expressions) + backup_and_restore_path(deployment, '/data') + +def backup_and_restore_path(deployment, path=""): + deployment.deploy() + deployment_run_command(deployment, "echo -n important-data > {}/back-me-up".format(path)) + backup_id = deployment.backup() + backups = deployment.get_backups() + while backups[backup_id]['status'] == "running": + time.sleep(10) backups = deployment.get_backups() - while backups[backup_id]['status'] == "running": - time.sleep(10) - backups = deployment.get_backups() - deployment_run_command(deployment, "rm {}/back-me-up".format(backup_path)) - deployment.restore(backup_id=backup_id) - deployment_run_command(deployment, "echo -n important-data | diff {}/back-me-up -".format(backup_path)) + deployment_run_command(deployment, "rm {}/back-me-up".format(path)) + deployment.restore(backup_id=backup_id) + deployment_run_command(deployment, "echo -n important-data | diff {}/back-me-up -".format(path)) From 0f607c1577bb56b627d1cebc2a10677e35a07a59 Mon Sep 17 00:00:00 2001 From: srghma Date: Sun, 10 Jun 2018 22:13:35 +0300 Subject: [PATCH 084/123] refactor: tests -> test_ec2_spot_instance (WIP) --- .../test_deploys_spot_instance/__init__.py | 13 ---------- tests/functional/test_ec2_backups/__init__.py | 1 - .../test_ec2_spot_instance/__init__.py | 25 +++++++++++++++++++ 3 files changed, 25 insertions(+), 14 deletions(-) delete mode 100644 tests/functional/test_deploys_spot_instance/__init__.py create mode 100644 tests/functional/test_ec2_spot_instance/__init__.py diff --git a/tests/functional/test_deploys_spot_instance/__init__.py b/tests/functional/test_deploys_spot_instance/__init__.py deleted file mode 100644 index 81c80fd40..000000000 --- a/tests/functional/test_deploys_spot_instance/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -from nose import tools -from tests.functional import single_machine_test -from os import path - -parent_dir = path.dirname(__file__) - -class TestDeploysSpotInstance(single_machine_test.SingleMachineTest): - def run_check(self): - self.depl.nix_exprs = self.depl.nix_exprs + [ - '%s/single_machine_ec2_spot_instance.nix' % (parent_dir), - ] - self.depl.deploy() - self.check_command("test -f /etc/NIXOS") diff --git a/tests/functional/test_ec2_backups/__init__.py b/tests/functional/test_ec2_backups/__init__.py index c9447b213..ac0062af3 100644 --- a/tests/functional/test_ec2_backups/__init__.py +++ b/tests/functional/test_ec2_backups/__init__.py @@ -2,7 +2,6 @@ from nixops.util import root_dir from itertools import product -from nose.tools import with_setup from parameterized import parameterized from tests.functional.shared.deployment_run_command import deployment_run_command diff --git a/tests/functional/test_ec2_spot_instance/__init__.py b/tests/functional/test_ec2_spot_instance/__init__.py new file mode 100644 index 000000000..ddf9bec3b --- /dev/null +++ b/tests/functional/test_ec2_spot_instance/__init__.py @@ -0,0 +1,25 @@ +from nixops.util import root_dir +from itertools import product +from parameterized import parameterized + +from tests.functional.shared.deployment_run_command import deployment_run_command +from tests.functional.shared.create_deployment import create_deployment +from tests.functional.shared.using_state_file import using_state_file + +@parameterized(product( + ['json', 'nixops'], + [ + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/ec2_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/ec2_spot_instance.nix'.format(root_dir) + ] + ], +)) + +def test_ec2_spot_instance(state_extension, nix_expressions): + with using_state_file( + unique_name='test_ec2_spot_instance', + state_extension=state_extension) as state: + deployment = create_deployment(state, nix_expressions) + deployment_run_command(deployment, "test -f /etc/NIXOS") From d8533547c960f1a8079c2f267be32d08854c8a52 Mon Sep 17 00:00:00 2001 From: srghma Date: Sun, 10 Jun 2018 22:27:02 +0300 Subject: [PATCH 085/123] refactor: tests -> test_deploys_nixos --- .../functional/test_deploys_nixos/__init__.py | 49 ++++++++++++++++--- 1 file changed, 43 insertions(+), 6 deletions(-) diff --git a/tests/functional/test_deploys_nixos/__init__.py b/tests/functional/test_deploys_nixos/__init__.py index 52dd16542..15ff4881c 100644 --- a/tests/functional/test_deploys_nixos/__init__.py +++ b/tests/functional/test_deploys_nixos/__init__.py @@ -1,8 +1,45 @@ -from nose import tools +from nixops.util import root_dir +from itertools import product +from parameterized import parameterized -from tests.functional import single_machine_test +from tests.functional.shared.deployment_run_command import deployment_run_command +from tests.functional.shared.create_deployment import create_deployment +from tests.functional.shared.using_state_file import using_state_file -class TestDeploysNixos(single_machine_test.SingleMachineTest): - def run_check(self): - self.depl.deploy() - self.check_command("test -f /etc/NIXOS") +@parameterized(product( + ['json', 'nixops'], + [ + # vbox + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/vbox_base.nix'.format(root_dir), + ], + # ec2 + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/ec2_base.nix'.format(root_dir), + ], + # gce + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/gce_base.nix'.format(root_dir), + ], + # azure + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/azure_base.nix'.format(root_dir), + ], + # libvirtd + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/libvirtd_base.nix'.format(root_dir), + ] + ], +)) + +def test_deploys_nixos(state_extension, nix_expressions): + with using_state_file( + unique_name='test_deploys_nixos', + state_extension=state_extension) as state: + deployment = create_deployment(state, nix_expressions) + deployment_run_command(deployment, "test -f /etc/NIXOS") From c0a21c98bcac832a19aca0269ed014613eaac6af Mon Sep 17 00:00:00 2001 From: srghma Date: Sun, 10 Jun 2018 22:30:02 +0300 Subject: [PATCH 086/123] refactor: tests -> using_state_file -> revert implementation to create state file without unique parent dir (this is not the time to parallelize tests) --- tests/functional/shared/using_state_file.py | 12 +++++------- tests/functional/test_deploys_nixos/__init__.py | 4 +--- tests/functional/test_ec2_backups/__init__.py | 8 ++------ tests/functional/test_ec2_spot_instance/__init__.py | 4 +--- 4 files changed, 9 insertions(+), 19 deletions(-) diff --git a/tests/functional/shared/using_state_file.py b/tests/functional/shared/using_state_file.py index d867530b7..3dfe92b6d 100644 --- a/tests/functional/shared/using_state_file.py +++ b/tests/functional/shared/using_state_file.py @@ -10,8 +10,8 @@ from nixops.util import root_dir @contextmanager -def using_state_file(unique_name, state_extension): - state_file_path_ = state_file_path(unique_name, state_extension) +def using_state_file(state_extension): + state_file_path_ = state_file_path(state_extension) create_file_parent_dirs_if_not_exists(state_file_path_) @@ -26,11 +26,9 @@ def using_state_file(unique_name, state_extension): def create_file_parent_dirs_if_not_exists(file_path): mkpath(os.path.dirname(file_path)) -def state_file_path(unique_name, state_extension): - unique_name_ = unique_name + '_' + state_extension - - return '{}/tests/state_files/{}/test.{}'.format( - root_dir, unique_name_, state_extension +def state_file_path(state_extension): + return '{}/tests/state_files/test.{}'.format( + root_dir, state_extension ) diff --git a/tests/functional/test_deploys_nixos/__init__.py b/tests/functional/test_deploys_nixos/__init__.py index 15ff4881c..7649fb653 100644 --- a/tests/functional/test_deploys_nixos/__init__.py +++ b/tests/functional/test_deploys_nixos/__init__.py @@ -38,8 +38,6 @@ )) def test_deploys_nixos(state_extension, nix_expressions): - with using_state_file( - unique_name='test_deploys_nixos', - state_extension=state_extension) as state: + with using_state_file(state_extension) as state: deployment = create_deployment(state, nix_expressions) deployment_run_command(deployment, "test -f /etc/NIXOS") diff --git a/tests/functional/test_ec2_backups/__init__.py b/tests/functional/test_ec2_backups/__init__.py index ac0062af3..222ffe3c8 100644 --- a/tests/functional/test_ec2_backups/__init__.py +++ b/tests/functional/test_ec2_backups/__init__.py @@ -19,9 +19,7 @@ ], )) def test_ec2_backups_simple(state_extension, nix_expressions): - with using_state_file( - unique_name='test_ec2_backups', - state_extension=state_extension) as state: + with using_state_file(state_extension) as state: deployment = create_deployment(state, nix_expressions) backup_and_restore_path(deployment) @@ -37,9 +35,7 @@ def test_ec2_backups_simple(state_extension, nix_expressions): ], )) def test_ec2_backups_raid(state_extension, nix_expressions): - with using_state_file( - unique_name='test_ec2_backups', - state_extension=state_extension) as state: + with using_state_file(state_extension) as state: deployment = create_deployment(state, nix_expressions) backup_and_restore_path(deployment, '/data') diff --git a/tests/functional/test_ec2_spot_instance/__init__.py b/tests/functional/test_ec2_spot_instance/__init__.py index ddf9bec3b..beec40066 100644 --- a/tests/functional/test_ec2_spot_instance/__init__.py +++ b/tests/functional/test_ec2_spot_instance/__init__.py @@ -18,8 +18,6 @@ )) def test_ec2_spot_instance(state_extension, nix_expressions): - with using_state_file( - unique_name='test_ec2_spot_instance', - state_extension=state_extension) as state: + with using_state_file(state_extension) as state: deployment = create_deployment(state, nix_expressions) deployment_run_command(deployment, "test -f /etc/NIXOS") From 047e2372e90fd3edc8a343511f927070076c3806 Mon Sep 17 00:00:00 2001 From: srghma Date: Mon, 11 Jun 2018 00:00:48 +0300 Subject: [PATCH 087/123] refactor: tests -> refactor other tests, not checked (WIP) --- .../shared/deployment_run_command.py | 10 +- .../test_cloning_clones/__init__.py | 52 +++- .../test_deleting_deletes/__init__.py | 51 +++- .../functional/test_deploys_nixos/__init__.py | 2 +- .../test_ec2_rds_dbinstance/__init__.py | 41 +-- .../test_ec2_spot_instance/__init__.py | 3 +- .../test_invalid_indentifier/__init__.py | 35 +-- .../test_query_deployments/__init__.py | 23 +- .../test_rebooting_reboots/__init__.py | 55 +++- .../test_rollback_rollsback/__init__.py | 76 ++++-- .../test_send_keys_sends_keys/__init__.py | 71 ++++-- .../test_starting_starts/__init__.py | 51 +++- .../test_stopping_stops/__init__.py | 49 +++- .../test_vbox_encrypted_links/__init__.py | 56 +++-- tests/functional/test_vpc/__init__.py | 236 ++++++------------ tests/functional/test_vpc/resources.py | 109 ++++++++ 16 files changed, 620 insertions(+), 300 deletions(-) create mode 100644 tests/functional/test_vpc/resources.py diff --git a/tests/functional/shared/deployment_run_command.py b/tests/functional/shared/deployment_run_command.py index daeaa5812..728711a09 100644 --- a/tests/functional/shared/deployment_run_command.py +++ b/tests/functional/shared/deployment_run_command.py @@ -1,4 +1,10 @@ -def deployment_run_command(deployment, command): +from nixops.util import ansi_highlight + +def deployment_run_command(deployment, command, michine_index=0): deployment.evaluate() - machine = deployment.machines.values()[0] + machine = deployment.machines.values()[michine_index] + + debug_message = ansi_highlight('tests> ') + command + import sys; print >> sys.__stdout__, debug_message + return machine.run_command(command) diff --git a/tests/functional/test_cloning_clones/__init__.py b/tests/functional/test_cloning_clones/__init__.py index 1c90b763b..f2545eade 100644 --- a/tests/functional/test_cloning_clones/__init__.py +++ b/tests/functional/test_cloning_clones/__init__.py @@ -1,10 +1,48 @@ from nose import tools +from nixops.util import root_dir +from itertools import product +from parameterized import parameterized -from tests.functional import single_machine_test +from tests.functional.shared.deployment_run_command import deployment_run_command +from tests.functional.shared.create_deployment import create_deployment +from tests.functional.shared.using_state_file import using_state_file -class TestCloningClones(single_machine_test.SingleMachineTest): - def run_check(self): - depl = self.depl.clone() - tools.assert_equal(depl.nix_exprs, self.depl.nix_exprs) - tools.assert_equal(depl.nix_path, self.depl.nix_path) - tools.assert_equal(depl.args, self.depl.args) +@parameterized(product( + ['json', 'nixops'], + [ + # vbox + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/vbox_base.nix'.format(root_dir), + ], + # ec2 + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/ec2_base.nix'.format(root_dir), + ], + # gce + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/gce_base.nix'.format(root_dir), + ], + # azure + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/azure_base.nix'.format(root_dir), + ], + # libvirtd + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/libvirtd_base.nix'.format(root_dir), + ] + ], +)) + +def test_cloning_clones(state_extension, nix_expressions): + with using_state_file(state_extension) as state: + deployment = create_deployment(state, nix_expressions) + + deployment2 = deployment.clone() + tools.assert_equal(deployment.nix_exprs, deployment2.nix_exprs) + tools.assert_equal(deployment.nix_path, deployment2.nix_path) + tools.assert_equal(deployment.args, deployment2.args) diff --git a/tests/functional/test_deleting_deletes/__init__.py b/tests/functional/test_deleting_deletes/__init__.py index 83ca9859f..285cfd1fb 100644 --- a/tests/functional/test_deleting_deletes/__init__.py +++ b/tests/functional/test_deleting_deletes/__init__.py @@ -1,10 +1,47 @@ from nose import tools -from nixops import deployment +from nixops.util import root_dir +from itertools import product +from parameterized import parameterized -from tests.functional import single_machine_test +from tests.functional.shared.deployment_run_command import deployment_run_command +from tests.functional.shared.create_deployment import create_deployment +from tests.functional.shared.using_state_file import using_state_file -class TestDeletingDeletes(single_machine_test.SingleMachineTest): - def run_check(self): - uuid = self.depl.uuid - self.depl.delete() - tools.assert_raises(Exception, self.state.open_deployment, (uuid,)) +@parameterized(product( + ['json', 'nixops'], + [ + # vbox + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/vbox_base.nix'.format(root_dir), + ], + # ec2 + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/ec2_base.nix'.format(root_dir), + ], + # gce + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/gce_base.nix'.format(root_dir), + ], + # azure + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/azure_base.nix'.format(root_dir), + ], + # libvirtd + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/libvirtd_base.nix'.format(root_dir), + ] + ], +)) + +def test_deleting_deletes(state_extension, nix_expressions): + with using_state_file(state_extension) as state: + deployment = create_deployment(state, nix_expressions) + + uuid = deployment.uuid + deployment.delete() + tools.assert_raises(Exception, state.open_deployment, (uuid,)) diff --git a/tests/functional/test_deploys_nixos/__init__.py b/tests/functional/test_deploys_nixos/__init__.py index 7649fb653..de47feeff 100644 --- a/tests/functional/test_deploys_nixos/__init__.py +++ b/tests/functional/test_deploys_nixos/__init__.py @@ -36,8 +36,8 @@ ] ], )) - def test_deploys_nixos(state_extension, nix_expressions): with using_state_file(state_extension) as state: deployment = create_deployment(state, nix_expressions) + deployment.deploy() deployment_run_command(deployment, "test -f /etc/NIXOS") diff --git a/tests/functional/test_ec2_rds_dbinstance/__init__.py b/tests/functional/test_ec2_rds_dbinstance/__init__.py index 5596d326a..f0e727204 100644 --- a/tests/functional/test_ec2_rds_dbinstance/__init__.py +++ b/tests/functional/test_ec2_rds_dbinstance/__init__.py @@ -1,24 +1,27 @@ -from os import path +import time -from nose import tools +from itertools import product +from parameterized import parameterized -from tests.functional import generic_deployment_test +from tests.functional.shared.deployment_run_command import deployment_run_command +from tests.functional.shared.create_deployment import create_deployment +from tests.functional.shared.using_state_file import using_state_file parent_dir = path.dirname(__file__) -logical_spec = '%s/ec2-rds-dbinstance.nix' % (parent_dir) -sg_spec = '%s/ec2-rds-dbinstance-with-sg.nix' % (parent_dir) - -class TestEc2RdsDbinstanceTest(generic_deployment_test.GenericDeploymentTest): - _multiprocess_can_split_ = True - - def setup(self): - super(TestEc2RdsDbinstanceTest, self).setup() - self.depl.nix_exprs = [ logical_spec ] - - def test_deploy(self): - self.depl.deploy() - - def test_deploy_with_sg(self): - self.depl.nix_exprs = [ sg_spec ] - self.depl.deploy() +@parameterized(product( + ['json', 'nixops'], + [ + [ + '{}/ec2-rds-dbinstance.nix'.format(parent_dir), + ], + [ + '{}/ec2-rds-dbinstance.nix'.format(parent_dir), + '{}/ec2-rds-dbinstance-with-sg.nix'.format(parent_dir), + ] + ], +)) +def test_ec2_rds_dbinstance(state_extension, nix_expressions): + with using_state_file(state_extension) as state: + deployment = create_deployment(state, nix_expressions) + deployment.deploy() diff --git a/tests/functional/test_ec2_spot_instance/__init__.py b/tests/functional/test_ec2_spot_instance/__init__.py index beec40066..086856d3a 100644 --- a/tests/functional/test_ec2_spot_instance/__init__.py +++ b/tests/functional/test_ec2_spot_instance/__init__.py @@ -14,10 +14,11 @@ '{}/tests/functional/shared/nix_expressions/ec2_base.nix'.format(root_dir), '{}/tests/functional/shared/nix_expressions/ec2_spot_instance.nix'.format(root_dir) ] - ], + ] )) def test_ec2_spot_instance(state_extension, nix_expressions): with using_state_file(state_extension) as state: deployment = create_deployment(state, nix_expressions) + deployment.deploy() deployment_run_command(deployment, "test -f /etc/NIXOS") diff --git a/tests/functional/test_invalid_indentifier/__init__.py b/tests/functional/test_invalid_indentifier/__init__.py index 8e534be9c..2f7e6932a 100644 --- a/tests/functional/test_invalid_indentifier/__init__.py +++ b/tests/functional/test_invalid_indentifier/__init__.py @@ -1,20 +1,23 @@ -from os import path - from nose import tools -from nose.tools import raises -from tests.functional import generic_deployment_test - -parent_dir = path.dirname(__file__) +from itertools import product +from parameterized import parameterized -logical_spec = '%s/invalid-identifier.nix' % (parent_dir) +from tests.functional.shared.deployment_run_command import deployment_run_command +from tests.functional.shared.create_deployment import create_deployment +from tests.functional.shared.using_state_file import using_state_file -class TestInvalidIdentifier(generic_deployment_test.GenericDeploymentTest): - - def setup(self): - super(TestInvalidIdentifier,self).setup() - self.depl.nix_exprs = [ logical_spec ] - - @raises(Exception) - def test_invalid_identifier_fails_evaluation(self): - self.depl.evaluate() +parent_dir = path.dirname(__file__) +@parameterized(product( + ['json', 'nixops'], + [ + [ + '{}/invalid-identifier.nix'.format(parent_dir), + ] + ] +)) +def test_invalid_indentifier(state_extension, nix_expressions): + with using_state_file(state_extension) as state: + deployment = create_deployment(state, nix_expressions) + with tools.assert_raises(Exception): + deployment.evaluate() diff --git a/tests/functional/test_query_deployments/__init__.py b/tests/functional/test_query_deployments/__init__.py index c045586dc..5e60b2d7b 100644 --- a/tests/functional/test_query_deployments/__init__.py +++ b/tests/functional/test_query_deployments/__init__.py @@ -1,12 +1,21 @@ from nose import tools +from parameterized import parameterized -from tests.functional import DatabaseUsingTest +from tests.functional.shared.deployment_run_command import deployment_run_command +from tests.functional.shared.create_deployment import create_deployment +from tests.functional.shared.using_state_file import using_state_file + +@parameterized( + ['json', 'nixops'] +) +def test_query_deployments(state_extension): + with using_state_file(state_extension) as state: + deployments = [] -class TestQueryDeployments(DatabaseUsingTest): - def test_shows_all_deployments(self): - depls = [] for i in range(10): - depls.append(self.state.create_deployment()) - uuids = self.state.query_deployments() - for depl in depls: + deployments.append(state.create_deployment()) + + uuids = state.query_deployments() + + for depl in deployments: tools.assert_true(any([ depl.uuid == uuid for uuid in uuids ])) diff --git a/tests/functional/test_rebooting_reboots/__init__.py b/tests/functional/test_rebooting_reboots/__init__.py index ed7e62638..75b6227bf 100644 --- a/tests/functional/test_rebooting_reboots/__init__.py +++ b/tests/functional/test_rebooting_reboots/__init__.py @@ -1,16 +1,51 @@ from nose import tools - -from tests.functional import single_machine_test - from nixops.ssh_util import SSHCommandFailed +from nixops.util import root_dir +from itertools import product +from parameterized import parameterized + +from tests.functional.shared.deployment_run_command import deployment_run_command +from tests.functional.shared.create_deployment import create_deployment +from tests.functional.shared.using_state_file import using_state_file -class TestRebootingReboots(single_machine_test.SingleMachineTest): - def run_check(self): - self.depl.deploy() - self.check_command("touch /run/not-rebooted") - self.depl.reboot_machines(wait=True) - m = self.depl.active.values()[0] +@parameterized(product( + ['json', 'nixops'], + [ + # vbox + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/vbox_base.nix'.format(root_dir), + ], + # ec2 + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/ec2_base.nix'.format(root_dir), + ], + # gce + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/gce_base.nix'.format(root_dir), + ], + # azure + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/azure_base.nix'.format(root_dir), + ], + # libvirtd + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/libvirtd_base.nix'.format(root_dir), + ] + ], +)) +def test_rebooting_reboots(state_extension, nix_expressions): + with using_state_file(state_extension) as state: + deployment = create_deployment(state, nix_expressions) + deployment.deploy() + deployment_run_command(deployment, "touch /run/not-rebooted") + deployment.reboot_machines(wait=True) + m = deployment.active.values()[0] m.check() tools.assert_equal(m.state, m.UP) with tools.assert_raises(SSHCommandFailed): - self.check_command("test -f /run/not-rebooted") + deployment_run_command(deployment, "test -f /run/not-rebooted") diff --git a/tests/functional/test_rollback_rollsback/__init__.py b/tests/functional/test_rollback_rollsback/__init__.py index de6c0bb49..981fa4323 100644 --- a/tests/functional/test_rollback_rollsback/__init__.py +++ b/tests/functional/test_rollback_rollsback/__init__.py @@ -1,30 +1,62 @@ -from os import path from nose import tools - -from tests.functional import single_machine_test - from nixops.ssh_util import SSHCommandFailed +from nixops.util import root_dir +from itertools import product +from parameterized import parameterized + +from tests.functional.shared.deployment_run_command import deployment_run_command +from tests.functional.shared.create_deployment import create_deployment +from tests.functional.shared.using_state_file import using_state_file + +has_hello_spec = '{}/tests/functional/shared/nix_expressions/has_hello.nix'.format(root_dir) +rollback_spec = '{}/tests/functional/shared/nix_expressions/rollback.nix'.format(root_dir) + +@parameterized(product( + ['json', 'nixops'], + [ + # vbox + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/vbox_base.nix'.format(root_dir), + ], + # ec2 + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/ec2_base.nix'.format(root_dir), + ], + # gce + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/gce_base.nix'.format(root_dir), + ], + # azure + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/azure_base.nix'.format(root_dir), + ], + # libvirtd + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/libvirtd_base.nix'.format(root_dir), + ] + ], +)) +def test_rollback_rollsback(state_extension, nix_expressions): + with using_state_file(state_extension) as state: + deployment = create_deployment(state, nix_expressions) + + deployment.nix_exprs = deployment.nix_exprs + [ rollback_spec ] + deployment.deploy() -parent_dir = path.dirname(__file__) - -has_hello_spec = '%s/single_machine_has_hello.nix' % (parent_dir) + with tools.assert_raises(SSHCommandFailed): + deployment_run_command(deployment, "hello") -rollback_spec = '%s/single_machine_rollback.nix' % (parent_dir) + deployment.nix_exprs = deployment.nix_exprs + [ has_hello_spec ] + deployment.deploy() -class TestRollbackRollsback(single_machine_test.SingleMachineTest): - _multiprocess_can_split_ = True + deployment_run_command(deployment, "hello") - def setup(self): - super(TestRollbackRollsback,self).setup() - self.depl.nix_exprs = self.depl.nix_exprs + [ rollback_spec ] + deployment.rollback(generation=1) - def run_check(self): - self.depl.deploy() - with tools.assert_raises(SSHCommandFailed): - self.check_command("hello") - self.depl.nix_exprs = self.depl.nix_exprs + [ has_hello_spec ] - self.depl.deploy() - self.check_command("hello") - self.depl.rollback(generation=1) with tools.assert_raises(SSHCommandFailed): - self.check_command("hello") + deployment_run_command(deployment, "hello") diff --git a/tests/functional/test_send_keys_sends_keys/__init__.py b/tests/functional/test_send_keys_sends_keys/__init__.py index 6d1f86dec..1cb1f3b51 100644 --- a/tests/functional/test_send_keys_sends_keys/__init__.py +++ b/tests/functional/test_send_keys_sends_keys/__init__.py @@ -1,26 +1,57 @@ -from os import path from nose import tools +from nixops.ssh_util import SSHCommandFailed +from nixops.util import root_dir +from itertools import product +from parameterized import parameterized -from tests.functional import single_machine_test +from tests.functional.shared.deployment_run_command import deployment_run_command +from tests.functional.shared.create_deployment import create_deployment +from tests.functional.shared.using_state_file import using_state_file -parent_dir = path.dirname(__file__) +secret_key_spec = '{}/tests/functional/shared/nix_expressions/single_machine_secret_key.nix'.format(root_dir) +elsewhere_key_spec = '{}/tests/functional/shared/nix_expressions/single_machine_elsewhere_key.nix'.format(root_dir) -secret_key_spec = '%s/single_machine_secret_key.nix' % (parent_dir) -elsewhere_key_spec = '%s/single_machine_elsewhere_key.nix' % (parent_dir) +@parameterized(product( + ['json', 'nixops'], + [ + # vbox + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/vbox_base.nix'.format(root_dir), + ], + # ec2 + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/ec2_base.nix'.format(root_dir), + ], + # gce + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/gce_base.nix'.format(root_dir), + ], + # azure + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/azure_base.nix'.format(root_dir), + ], + # libvirtd + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/libvirtd_base.nix'.format(root_dir), + ] + ], +)) +def test_send_keys_sends_keys(state_extension, nix_expressions): + with using_state_file(state_extension) as state: + deployment = create_deployment(state, nix_expressions) -class TestSendKeysSendsKeys(single_machine_test.SingleMachineTest): - _multiprocess_can_split_ = True + deployment.nix_exprs = deployment.nix_exprs + [ secret_key_spec, elsewhere_key_spec ] + deployment.deploy() - def setup(self): - super(TestSendKeysSendsKeys,self).setup() - self.depl.nix_exprs = self.depl.nix_exprs + [ secret_key_spec, elsewhere_key_spec ] - - def run_check(self): - self.depl.deploy() - self.check_command("test -f /run/keys/secret.key") - self.check_command("rm -f /run/keys/secret.key") - self.check_command("test -f /new/directory/elsewhere.key") - self.check_command("rm -f /new/directory/elsewhere.key") - self.depl.send_keys() - self.check_command("test -f /run/keys/secret.key") - self.check_command("test -f /new/directory/elsewhere.key") + deployment_run_command(deployment, "test -f /run/keys/secret.key") + deployment_run_command(deployment, "rm -f /run/keys/secret.key") + deployment_run_command(deployment, "test -f /new/directory/elsewhere.key") + deployment_run_command(deployment, "rm -f /new/directory/elsewhere.key") + deployment.send_keys() + deployment_run_command(deployment, "test -f /run/keys/secret.key") + deployment_run_command(deployment, "test -f /new/directory/elsewhere.key") diff --git a/tests/functional/test_starting_starts/__init__.py b/tests/functional/test_starting_starts/__init__.py index d445562e2..629c0c6c4 100644 --- a/tests/functional/test_starting_starts/__init__.py +++ b/tests/functional/test_starting_starts/__init__.py @@ -1,12 +1,49 @@ from nose import tools +from nixops.ssh_util import SSHCommandFailed +from nixops.util import root_dir +from itertools import product +from parameterized import parameterized -from tests.functional import single_machine_test +from tests.functional.shared.deployment_run_command import deployment_run_command +from tests.functional.shared.create_deployment import create_deployment +from tests.functional.shared.using_state_file import using_state_file -class TestStartingStarts(single_machine_test.SingleMachineTest): - def run_check(self): - self.depl.deploy() - self.depl.stop_machines() - self.depl.start_machines() - m = self.depl.active.values()[0] +@parameterized(product( + ['json', 'nixops'], + [ + # vbox + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/vbox_base.nix'.format(root_dir), + ], + # ec2 + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/ec2_base.nix'.format(root_dir), + ], + # gce + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/gce_base.nix'.format(root_dir), + ], + # azure + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/azure_base.nix'.format(root_dir), + ], + # libvirtd + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/libvirtd_base.nix'.format(root_dir), + ] + ], +)) +def test_starting_starts(state_extension, nix_expressions): + with using_state_file(state_extension) as state: + deployment = create_deployment(state, nix_expressions) + deployment.deploy() + deployment.stop_machines() + deployment.start_machines() + m = deployment.active.values()[0] m.check() tools.assert_equal(m.state, m.UP) diff --git a/tests/functional/test_stopping_stops/__init__.py b/tests/functional/test_stopping_stops/__init__.py index 4841d22b6..3c79654e6 100644 --- a/tests/functional/test_stopping_stops/__init__.py +++ b/tests/functional/test_stopping_stops/__init__.py @@ -1,10 +1,47 @@ from nose import tools +from nixops.ssh_util import SSHCommandFailed +from nixops.util import root_dir +from itertools import product +from parameterized import parameterized -from tests.functional import single_machine_test +from tests.functional.shared.deployment_run_command import deployment_run_command +from tests.functional.shared.create_deployment import create_deployment +from tests.functional.shared.using_state_file import using_state_file -class TestStoppingStops(single_machine_test.SingleMachineTest): - def run_check(self): - self.depl.deploy() - self.depl.stop_machines() - m = self.depl.active.values()[0] +@parameterized(product( + ['json', 'nixops'], + [ + # vbox + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/vbox_base.nix'.format(root_dir), + ], + # ec2 + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/ec2_base.nix'.format(root_dir), + ], + # gce + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/gce_base.nix'.format(root_dir), + ], + # azure + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/azure_base.nix'.format(root_dir), + ], + # libvirtd + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/libvirtd_base.nix'.format(root_dir), + ] + ], +)) +def test_stopping_stops(state_extension, nix_expressions): + with using_state_file(state_extension) as state: + deployment = create_deployment(state, nix_expressions) + deployment.deploy() + deployment.stop_machines() + m = deployment.active.values()[0] tools.assert_equal(m.state, m.STOPPED) diff --git a/tests/functional/test_vbox_encrypted_links/__init__.py b/tests/functional/test_vbox_encrypted_links/__init__.py index 25c7a753b..bf5080118 100644 --- a/tests/functional/test_vbox_encrypted_links/__init__.py +++ b/tests/functional/test_vbox_encrypted_links/__init__.py @@ -1,3 +1,13 @@ +from nose import tools +from nixops.ssh_util import SSHCommandFailed +from nixops.util import root_dir +from itertools import product +from parameterized import parameterized + +from tests.functional.shared.deployment_run_command import deployment_run_command +from tests.functional.shared.create_deployment import create_deployment +from tests.functional.shared.using_state_file import using_state_file + from os import path from nose import tools, SkipTest from tests.functional import generic_deployment_test @@ -10,33 +20,39 @@ parent_dir = path.dirname(__file__) -logical_spec = '%s/encrypted-links.nix' % (parent_dir) +logical_spec = '{}/encrypted-links.nix'.format(parent_dir) -class TestEncryptedLinks(generic_deployment_test.GenericDeploymentTest): +@parameterized( + ['json', 'nixops'] +) +def test_vbox_encrypted_links(state_extension): + if subprocess.call(["VBoxManage", "--version"], + stdout=devnull, + stderr=devnull) != 0: + raise SkipTest("VirtualBox is not available") - def setup(self): - super(TestEncryptedLinks,self).setup() - self.depl.nix_exprs = [ logical_spec ] + with using_state_file(state_extension) as state: + deployment = create_deployment(state, [logical_spec]) - def test_deploy(self): - if subprocess.call(["VBoxManage", "--version"], - stdout=devnull, - stderr=devnull) != 0: - raise SkipTest("VirtualBox is not available") - - self.depl.debug = True - self.depl.deploy() + deployment.debug = True + deployment.deploy() # !!! Shouldn't need this, instead the encrypted links target # should wait until the link is active... time.sleep(1) - self.ping("machine1", "machine2") - self.ping("machine2", "machine1") - self.depl.machines["machine1"].run_command("systemctl stop encrypted-links.target") + ping(deployment, "machine1", "machine2") + ping(deployment, "machine2", "machine1") + + deployment.machines["machine1"].run_command("systemctl stop encrypted-links.target") + with tools.assert_raises(SSHCommandFailed): - self.ping("machine1", "machine2") + ping(deployment, "machine1", "machine2") + with tools.assert_raises(SSHCommandFailed): - self.ping("machine2", "machine1") + ping(deployment, "machine2", "machine1") + + +# Helpers - def ping(self, machine1, machine2): - self.depl.machines[machine1].run_command("ping -c1 {0}-encrypted".format(machine2)) +def ping(deployment, machine1, machine2): + deployment.machines[machine1].run_command("ping -c1 {0}-encrypted".format(machine2)) diff --git a/tests/functional/test_vpc/__init__.py b/tests/functional/test_vpc/__init__.py index 25e778d6f..1fa5dcd7a 100644 --- a/tests/functional/test_vpc/__init__.py +++ b/tests/functional/test_vpc/__init__.py @@ -5,179 +5,105 @@ from nose.plugins.attrib import attr import boto3 -from nixops.nix_expr import RawValue, Function, Call, py2nix import nixops.util -from tests.functional import generic_deployment_test parent_dir = path.dirname(__file__) base_spec = "{}/vpc.nix".format(parent_dir) -class TestVPC(generic_deployment_test.GenericDeploymentTest): +@parameterized(['json', 'nixops']) +def test_deploy_vpc(state_extension, nix_expressions): + with using_state_file(state_extension) as state: + deployment = create_deployment(state, [base_spec]) - def setup(self): - super(TestVPC,self).setup() - self.depl.nix_exprs = [ base_spec ] - self.exprs_dir = nixops.util.SelfDeletingDir(tempfile.mkdtemp("nixos-tests")) - - - def test_deploy_vpc(self): - self.depl.deploy() - vpc_resource = self.depl.get_typed_resource("vpc-test", "vpc") + deployment.deploy() + vpc_resource = deployment.get_typed_resource("vpc-test", "vpc") vpc = vpc_resource.get_client().describe_vpcs(VpcIds=[vpc_resource._state['vpcId']]) tools.ok_(len(vpc['Vpcs']) > 0, "VPC not found!") tools.eq_(vpc['Vpcs'][0]['CidrBlock'], "10.0.0.0/16", "CIDR block mismatch") - def test_deploy_vpc_machine(self): - self.compose_expressions([CFG_SUBNET, CFG_INTERNET_ROUTE, CFG_VPC_MACHINE]) - self.depl.deploy(plan_only=True) - self.depl.deploy() - - def test_enable_dns_support(self): - self.compose_expressions([CFG_DNS_SUPPORT]) - self.depl.deploy(plan_only=True) - self.depl.deploy() - - def test_enable_ipv6(self): - self.compose_expressions([CFG_IPV6]) - self.depl.deploy(plan_only=True) - self.depl.deploy() - vpc_resource = self.depl.get_typed_resource("vpc-test", "vpc") +@parameterized(['json', 'nixops']) +def test_deploy_vpc_machine(state_extension, nix_expressions): + with using_state_file(state_extension) as state: + nix_expressions = compose_expressions([CFG_SUBNET, CFG_INTERNET_ROUTE, CFG_VPC_MACHINE]) + nix_expressions_ = [base_spec] + nix_expressions + + deployment = create_deployment(state, nix_expressions_) + + deployment.deploy(plan_only=True) + deployment.deploy() + +@parameterized(['json', 'nixops']) +def test_enable_dns_support(state_extension, nix_expressions): + with using_state_file(state_extension) as state: + nix_expressions = compose_expressions([CFG_DNS_SUPPORT]) + nix_expressions_ = [base_spec] + nix_expressions + + deployment = create_deployment(state, nix_expressions_) + + deployment.deploy(plan_only=True) + deployment.deploy() + +@parameterized(['json', 'nixops']) +def test_enable_ipv6(): + with using_state_file(state_extension) as state: + nix_expressions = compose_expressions([CFG_IPV6]) + nix_expressions_ = [base_spec] + nix_expressions + + deployment = create_deployment(state, nix_expressions_) + + deployment.deploy(plan_only=True) + deployment.deploy() + + vpc_resource = deployment.get_typed_resource("vpc-test", "vpc") vpc = vpc_resource.get_client().describe_vpcs(VpcIds=[vpc_resource._state['vpcId']]) ipv6_block = vpc['Vpcs'][0]['Ipv6CidrBlockAssociationSet'] tools.ok_(len(ipv6_block) > 0, "There is no Ipv6 block") tools.ok_(ipv6_block[0].get('Ipv6CidrBlock', None) != None, "No Ipv6 cidr block in the response") - def test_deploy_subnets(self): +@parameterized(['json', 'nixops']) +def test_deploy_subnets(): + with using_state_file(state_extension) as state: # FIXME might need to factor out resources into separate test # classes depending on the number of tests needed. - self.compose_expressions([CFG_SUBNET]) - self.depl.deploy(plan_only=True) - self.depl.deploy() - subnet_resource = self.depl.get_typed_resource("subnet-test", "vpc-subnet") + nix_expressions = compose_expressions([CFG_SUBNET]) + nix_expressions_ = [base_spec] + nix_expressions + + deployment = create_deployment(state, nix_expressions_) + + deployment.deploy(plan_only=True) + deployment.deploy() + subnet_resource = deployment.get_typed_resource("subnet-test", "vpc-subnet") subnet = subnet_resource.get_client().describe_subnets(SubnetIds=[subnet_resource._state['subnetId']]) tools.ok_(len(subnet['Subnets']) > 0, "VPC subnet not found!") - def test_deploy_nat_gtw(self): - self.compose_expressions([CFG_SUBNET, CFG_NAT_GTW]) - self.depl.deploy(plan_only=True) - self.depl.deploy() - - def compose_expressions(self, configurations): - extra_exprs = list(map(self.generate_config, configurations)) - self.depl.nix_exprs = [base_spec] + extra_exprs - - def generate_config(self, config): - basename, expr = config - expr_path = "{0}/{1}".format(self.exprs_dir, basename) - with open(expr_path, "w") as cfg: - cfg.write(expr) - return expr_path - -CFG_VPC_MACHINE = ("network.nix", """ - { - machine = - {config, resources, pkgs, lib, ...}: - { - deployment.targetEnv = "ec2"; - deployment.hasFastConnection = true; - deployment.ec2.associatePublicIpAddress = true; - deployment.ec2.region = "us-east-1"; - deployment.ec2.instanceType = "c3.large"; - deployment.ec2.subnetId = resources.vpcSubnets.subnet-test; - deployment.ec2.keyPair = resources.ec2KeyPairs.keypair.name; - deployment.ec2.securityGroups = []; - deployment.ec2.securityGroupIds = [ resources.ec2SecurityGroups.public-ssh.name ]; - }; - - resources.ec2KeyPairs.keypair = { region = "us-east-1"; }; - resources.ec2SecurityGroups.public-ssh = - { resources, ... }: - { - region = "us-east-1"; - vpcId = resources.vpc.vpc-test; - rules = [{ toPort = 22; fromPort = 22; sourceIp = "0.0.0.0/0"; }]; - }; - } - """) - -CFG_INTERNET_ROUTE = ("igw_route.nix", """ - let - region = "us-east-1"; - in - { - resources = { - - vpcRouteTables.route-table = - { resources, ... }: - { inherit region; vpcId = resources.vpc.vpc-test; }; - - vpcRouteTableAssociations.association-test = - { resources, ... }: - { - inherit region; - subnetId = resources.vpcSubnets.subnet-test; - routeTableId = resources.vpcRouteTables.route-table; - }; - - vpcRoutes.igw-route = - { resources, ... }: - { - inherit region; - routeTableId = resources.vpcRouteTables.route-table; - destinationCidrBlock = "0.0.0.0/0"; - gatewayId = resources.vpcInternetGateways.igw-test; - }; - - vpcInternetGateways.igw-test = - { resources, ... }: - { - inherit region; - vpcId = resources.vpc.vpc-test; - }; - }; - } - """) - -CFG_DNS_SUPPORT = ("enable_dns_support.nix", py2nix({ - ('resources', 'vpc', 'vpc-test', 'enableDnsSupport'): True - })) - -CFG_IPV6 = ("ipv6.nix", py2nix({ - ('resources', 'vpc', 'vpc-test', 'amazonProvidedIpv6CidrBlock'): True - })) - -CFG_NAT_GTW = ("nat_gtw.nix", """ - { - resources.elasticIPs.nat-eip = - { - region = "us-east-1"; - vpc = true; - }; - - resources.vpcNatGateways.nat = - { resources, ... }: - { - region = "us-east-1"; - allocationId = resources.elasticIPs.nat-eip; - subnetId = resources.vpcSubnets.subnet-test; - }; - } - """) - -CFG_SUBNET = ("subnet.nix", """ - { - resources.vpcSubnets.subnet-test = - { resources, ... }: - { - region = "us-east-1"; - zone = "us-east-1a"; - vpcId = resources.vpc.vpc-test; - cidrBlock = "10.0.0.0/19"; - mapPublicIpOnLaunch = true; - tags = { - Source = "NixOps Tests"; - }; - }; - } - """) +@parameterized(['json', 'nixops']) +def test_deploy_nat_gtw(): + with using_state_file(state_extension) as state: + nix_expressions = compose_expressions([CFG_SUBNET, CFG_NAT_GTW]) + nix_expressions_ = [base_spec] + nix_expressions + + deployment = create_deployment(state, nix_expressions_) + + deployment.deploy(plan_only=True) + deployment.deploy() + +# Helpers + +def create_exprs_dir(): + return nixops.util.SelfDeletingDir(tempfile.mkdtemp("nixos-tests")) + +def compose_expressions(configurations): + exprs_dir = create_exprs_dir() + + extra_exprs = list(map(lambda x: generate_config(exprs_dir, x), configurations)) + + nix_exprs = [base_spec] + extra_exprs + return nix_exprs + +def generate_config(exprs_dir, config): + basename, expr = config + expr_path = "{0}/{1}".format(exprs_dir, basename) + with open(expr_path, "w") as cfg: + cfg.write(expr) + return expr_path diff --git a/tests/functional/test_vpc/resources.py b/tests/functional/test_vpc/resources.py new file mode 100644 index 000000000..8e7dd8495 --- /dev/null +++ b/tests/functional/test_vpc/resources.py @@ -0,0 +1,109 @@ +from nixops.nix_expr import py2nix + +CFG_VPC_MACHINE = ("network.nix", """ + { + machine = + {config, resources, pkgs, lib, ...}: + { + deployment.targetEnv = "ec2"; + deployment.hasFastConnection = true; + deployment.ec2.associatePublicIpAddress = true; + deployment.ec2.region = "us-east-1"; + deployment.ec2.instanceType = "c3.large"; + deployment.ec2.subnetId = resources.vpcSubnets.subnet-test; + deployment.ec2.keyPair = resources.ec2KeyPairs.keypair.name; + deployment.ec2.securityGroups = []; + deployment.ec2.securityGroupIds = [ resources.ec2SecurityGroups.public-ssh.name ]; + }; + + resources.ec2KeyPairs.keypair = { region = "us-east-1"; }; + resources.ec2SecurityGroups.public-ssh = + { resources, ... }: + { + region = "us-east-1"; + vpcId = resources.vpc.vpc-test; + rules = [{ toPort = 22; fromPort = 22; sourceIp = "0.0.0.0/0"; }]; + }; + } + """) + +CFG_INTERNET_ROUTE = ("igw_route.nix", """ + let + region = "us-east-1"; + in + { + resources = { + + vpcRouteTables.route-table = + { resources, ... }: + { inherit region; vpcId = resources.vpc.vpc-test; }; + + vpcRouteTableAssociations.association-test = + { resources, ... }: + { + inherit region; + subnetId = resources.vpcSubnets.subnet-test; + routeTableId = resources.vpcRouteTables.route-table; + }; + + vpcRoutes.igw-route = + { resources, ... }: + { + inherit region; + routeTableId = resources.vpcRouteTables.route-table; + destinationCidrBlock = "0.0.0.0/0"; + gatewayId = resources.vpcInternetGateways.igw-test; + }; + + vpcInternetGateways.igw-test = + { resources, ... }: + { + inherit region; + vpcId = resources.vpc.vpc-test; + }; + }; + } + """) + +CFG_DNS_SUPPORT = ("enable_dns_support.nix", py2nix({ + ('resources', 'vpc', 'vpc-test', 'enableDnsSupport'): True +})) + +CFG_IPV6 = ("ipv6.nix", py2nix({ + ('resources', 'vpc', 'vpc-test', 'amazonProvidedIpv6CidrBlock'): True +})) + +CFG_NAT_GTW = ("nat_gtw.nix", """ + { + resources.elasticIPs.nat-eip = + { + region = "us-east-1"; + vpc = true; + }; + + resources.vpcNatGateways.nat = + { resources, ... }: + { + region = "us-east-1"; + allocationId = resources.elasticIPs.nat-eip; + subnetId = resources.vpcSubnets.subnet-test; + }; + } + """) + +CFG_SUBNET = ("subnet.nix", """ + { + resources.vpcSubnets.subnet-test = + { resources, ... }: + { + region = "us-east-1"; + zone = "us-east-1a"; + vpcId = resources.vpc.vpc-test; + cidrBlock = "10.0.0.0/19"; + mapPublicIpOnLaunch = true; + tags = { + Source = "NixOps Tests"; + }; + }; + } + """) From 04f99807c986088908b1c8beafdc22a4eb0243af Mon Sep 17 00:00:00 2001 From: srghma Date: Mon, 11 Jun 2018 21:30:39 +0300 Subject: [PATCH 088/123] refactor: tests -> make vbox test unoptional (this should be centralized) --- tests/functional/test_vbox_encrypted_links/__init__.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tests/functional/test_vbox_encrypted_links/__init__.py b/tests/functional/test_vbox_encrypted_links/__init__.py index bf5080118..1298887f2 100644 --- a/tests/functional/test_vbox_encrypted_links/__init__.py +++ b/tests/functional/test_vbox_encrypted_links/__init__.py @@ -26,11 +26,6 @@ ['json', 'nixops'] ) def test_vbox_encrypted_links(state_extension): - if subprocess.call(["VBoxManage", "--version"], - stdout=devnull, - stderr=devnull) != 0: - raise SkipTest("VirtualBox is not available") - with using_state_file(state_extension) as state: deployment = create_deployment(state, [logical_spec]) From 11aa0e3931eac88317b28b15edc2a37e94574aa9 Mon Sep 17 00:00:00 2001 From: srghma Date: Thu, 14 Jun 2018 20:48:51 +0300 Subject: [PATCH 089/123] fix: tests --- .../functional/test_ec2_rds_dbinstance/__init__.py | 1 + .../test_vbox_encrypted_links/__init__.py | 13 ++++--------- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/tests/functional/test_ec2_rds_dbinstance/__init__.py b/tests/functional/test_ec2_rds_dbinstance/__init__.py index f0e727204..ac9188c3e 100644 --- a/tests/functional/test_ec2_rds_dbinstance/__init__.py +++ b/tests/functional/test_ec2_rds_dbinstance/__init__.py @@ -1,4 +1,5 @@ import time +from os import path from itertools import product from parameterized import parameterized diff --git a/tests/functional/test_vbox_encrypted_links/__init__.py b/tests/functional/test_vbox_encrypted_links/__init__.py index 1298887f2..3761b4574 100644 --- a/tests/functional/test_vbox_encrypted_links/__init__.py +++ b/tests/functional/test_vbox_encrypted_links/__init__.py @@ -3,21 +3,16 @@ from nixops.util import root_dir from itertools import product from parameterized import parameterized - -from tests.functional.shared.deployment_run_command import deployment_run_command -from tests.functional.shared.create_deployment import create_deployment -from tests.functional.shared.using_state_file import using_state_file - from os import path -from nose import tools, SkipTest -from tests.functional import generic_deployment_test -from nixops.ssh_util import SSHCommandFailed -from nixops.util import devnull import sys import time import signal import subprocess +from tests.functional.shared.deployment_run_command import deployment_run_command +from tests.functional.shared.create_deployment import create_deployment +from tests.functional.shared.using_state_file import using_state_file + parent_dir = path.dirname(__file__) logical_spec = '{}/encrypted-links.nix'.format(parent_dir) From cf063be667dabfdc7ef987a50e9e1fc3a8968644 Mon Sep 17 00:00:00 2001 From: srghma Date: Thu, 14 Jun 2018 23:04:54 +0300 Subject: [PATCH 090/123] fix(#970): tests.functional.test_ec2_rds_dbinstance -> remove default group, pass empty array when no security groups --- nix/ec2-rds-dbinstance.nix | 2 +- nix/ec2-rds-dbsecurity-group.nix | 2 +- nixops/resources/__init__.py | 2 +- tests/functional/test_ec2_rds_dbinstance/__init__.py | 1 - 4 files changed, 3 insertions(+), 4 deletions(-) diff --git a/nix/ec2-rds-dbinstance.nix b/nix/ec2-rds-dbinstance.nix index b95b740ff..438650558 100644 --- a/nix/ec2-rds-dbinstance.nix +++ b/nix/ec2-rds-dbinstance.nix @@ -82,7 +82,7 @@ with import ./lib.nix lib; }; securityGroups = mkOption { - default = [ "default" ]; + default = [ ]; type = types.listOf (types.either types.str (resource "ec2-rds-security-group")); apply = map (x: if builtins.isString x then x else "res-" + x._name); description = '' diff --git a/nix/ec2-rds-dbsecurity-group.nix b/nix/ec2-rds-dbsecurity-group.nix index dd9d1674f..f0d960e54 100644 --- a/nix/ec2-rds-dbsecurity-group.nix +++ b/nix/ec2-rds-dbsecurity-group.nix @@ -18,7 +18,7 @@ with import ./lib.nix lib; type = types.str; description = '' Description of the RDS DB security group. - ''; + ''; }; region = mkOption { diff --git a/nixops/resources/__init__.py b/nixops/resources/__init__.py index d53f7f5fa..f86a2f535 100644 --- a/nixops/resources/__init__.py +++ b/nixops/resources/__init__.py @@ -36,7 +36,7 @@ class ResourceState(object): @classmethod def get_type(cls): - """A resource type identifier that must match the corresponding ResourceDefinition classs""" + """A resource type identifier that must match the corresponding ResourceDefinition class""" raise NotImplementedError("get_type") # Valid values for self.state. Not all of these make sense for diff --git a/tests/functional/test_ec2_rds_dbinstance/__init__.py b/tests/functional/test_ec2_rds_dbinstance/__init__.py index ac9188c3e..ab2b1f7e1 100644 --- a/tests/functional/test_ec2_rds_dbinstance/__init__.py +++ b/tests/functional/test_ec2_rds_dbinstance/__init__.py @@ -17,7 +17,6 @@ '{}/ec2-rds-dbinstance.nix'.format(parent_dir), ], [ - '{}/ec2-rds-dbinstance.nix'.format(parent_dir), '{}/ec2-rds-dbinstance-with-sg.nix'.format(parent_dir), ] ], From e2a64a2b1a30c667b86361c70e66b75c0a867618 Mon Sep 17 00:00:00 2001 From: srghma Date: Fri, 15 Jun 2018 21:30:55 +0300 Subject: [PATCH 091/123] feat: tests -> using_state_file -> delete state file if exists before creating new --- tests/functional/shared/using_state_file.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/functional/shared/using_state_file.py b/tests/functional/shared/using_state_file.py index 3dfe92b6d..dbf43568b 100644 --- a/tests/functional/shared/using_state_file.py +++ b/tests/functional/shared/using_state_file.py @@ -15,7 +15,11 @@ def using_state_file(state_extension): create_file_parent_dirs_if_not_exists(state_file_path_) + if os.path.exists(state_file_path_): + destroy_deployments_and_remove_state_file(state_file_path_) + state = nixops.state.open(state_file_path_) + try: yield state finally: From 8511f5923dca76693d99415d6b161fbb2a766e86 Mon Sep 17 00:00:00 2001 From: srghma Date: Fri, 15 Jun 2018 23:19:52 +0300 Subject: [PATCH 092/123] feat: tests -> using_state_file -> unique state file for each test --- .../shared/unique_state_file_path.py | 8 +++++ tests/functional/shared/using_state_file.py | 21 ++++------- .../test_ec2_rds_dbinstance/__init__.py | 35 ++++++++++++++----- 3 files changed, 40 insertions(+), 24 deletions(-) create mode 100644 tests/functional/shared/unique_state_file_path.py diff --git a/tests/functional/shared/unique_state_file_path.py b/tests/functional/shared/unique_state_file_path.py new file mode 100644 index 000000000..ee568fbb5 --- /dev/null +++ b/tests/functional/shared/unique_state_file_path.py @@ -0,0 +1,8 @@ +from nixops.util import root_dir + +def unique_state_file_path(array_of_keys_file_name_depends_on, extension): + unique_file_name = '_'.join(array_of_keys_file_name_depends_on) + + return '{}/tests/state_files/{}.{}'.format( + root_dir, unique_file_name, extension + ) diff --git a/tests/functional/shared/using_state_file.py b/tests/functional/shared/using_state_file.py index dbf43568b..730478242 100644 --- a/tests/functional/shared/using_state_file.py +++ b/tests/functional/shared/using_state_file.py @@ -7,35 +7,26 @@ from distutils.dir_util import mkpath import nixops.state -from nixops.util import root_dir @contextmanager -def using_state_file(state_extension): - state_file_path_ = state_file_path(state_extension) +def using_state_file(state_file_path): + create_file_parent_dirs_if_not_exists(state_file_path) - create_file_parent_dirs_if_not_exists(state_file_path_) + if os.path.exists(state_file_path): + destroy_deployments_and_remove_state_file(state_file_path) - if os.path.exists(state_file_path_): - destroy_deployments_and_remove_state_file(state_file_path_) - - state = nixops.state.open(state_file_path_) + state = nixops.state.open(state_file_path) try: yield state finally: state.close() - destroy_deployments_and_remove_state_file(state_file_path_) + destroy_deployments_and_remove_state_file(state_file_path) def create_file_parent_dirs_if_not_exists(file_path): mkpath(os.path.dirname(file_path)) -def state_file_path(state_extension): - return '{}/tests/state_files/test.{}'.format( - root_dir, state_extension - ) - - def destroy_deployments(state, uuid): deployment = state.open_deployment(uuid) deployment.logger.set_autoresponse("y") diff --git a/tests/functional/test_ec2_rds_dbinstance/__init__.py b/tests/functional/test_ec2_rds_dbinstance/__init__.py index ab2b1f7e1..e50ac4c65 100644 --- a/tests/functional/test_ec2_rds_dbinstance/__init__.py +++ b/tests/functional/test_ec2_rds_dbinstance/__init__.py @@ -7,21 +7,38 @@ from tests.functional.shared.deployment_run_command import deployment_run_command from tests.functional.shared.create_deployment import create_deployment from tests.functional.shared.using_state_file import using_state_file +from tests.functional.shared.unique_state_file_path import unique_state_file_path parent_dir = path.dirname(__file__) @parameterized(product( - ['json', 'nixops'], [ - [ - '{}/ec2-rds-dbinstance.nix'.format(parent_dir), - ], - [ - '{}/ec2-rds-dbinstance-with-sg.nix'.format(parent_dir), - ] + 'json', + 'nixops' + ], + [ + ( + 'simple', + [ + '{}/ec2-rds-dbinstance.nix'.format(parent_dir), + ] + ), + ( + 'sg', + [ + '{}/ec2-rds-dbinstance-with-sg.nix'.format(parent_dir), + ] + ) ], )) -def test_ec2_rds_dbinstance(state_extension, nix_expressions): - with using_state_file(state_extension) as state: +def test_ec2_rds_dbinstance(state_extension, nix_expressions_tuple): + nix_expressions_id, nix_expressions = nix_expressions_tuple + + with using_state_file( + unique_state_file_path( + ['test_ec2_rds_dbinstance', nix_expressions_id], + state_extension + ) + ) as state: deployment = create_deployment(state, nix_expressions) deployment.deploy() From 0bd5210d20bee71cac29a64bb87ef2fcc4708561 Mon Sep 17 00:00:00 2001 From: srghma Date: Sat, 16 Jun 2018 12:00:18 +0300 Subject: [PATCH 093/123] refactor: tests -> use using_unique_state_file in some tests --- .../shared/using_unique_state_file.py | 12 +++ .../test_cloning_clones/__init__.py | 76 ++++++++++------- .../test_deleting_deletes/__init__.py | 76 ++++++++++------- .../functional/test_deploys_nixos/__init__.py | 84 ++++++++++++------- tests/functional/test_ec2_backups/__init__.py | 22 +++-- .../test_ec2_rds_dbinstance/__init__.py | 11 +-- .../test_ec2_spot_instance/__init__.py | 24 ------ 7 files changed, 182 insertions(+), 123 deletions(-) create mode 100644 tests/functional/shared/using_unique_state_file.py delete mode 100644 tests/functional/test_ec2_spot_instance/__init__.py diff --git a/tests/functional/shared/using_unique_state_file.py b/tests/functional/shared/using_unique_state_file.py new file mode 100644 index 000000000..653a7b02b --- /dev/null +++ b/tests/functional/shared/using_unique_state_file.py @@ -0,0 +1,12 @@ +from tests.functional.shared.using_state_file import using_state_file +from tests.functional.shared.unique_state_file_path import unique_state_file_path + +def using_unique_state_file( + array_of_keys_file_name_depends_on, + extension): + return using_state_file( + unique_state_file_path( + array_of_keys_file_name_depends_on, + extension + ) + ) diff --git a/tests/functional/test_cloning_clones/__init__.py b/tests/functional/test_cloning_clones/__init__.py index f2545eade..570b3435e 100644 --- a/tests/functional/test_cloning_clones/__init__.py +++ b/tests/functional/test_cloning_clones/__init__.py @@ -5,41 +5,59 @@ from tests.functional.shared.deployment_run_command import deployment_run_command from tests.functional.shared.create_deployment import create_deployment -from tests.functional.shared.using_state_file import using_state_file +from tests.functional.shared.using_unique_state_file import using_unique_state_file @parameterized(product( - ['json', 'nixops'], [ - # vbox - [ - '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), - '{}/tests/functional/shared/nix_expressions/vbox_base.nix'.format(root_dir), - ], - # ec2 - [ - '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), - '{}/tests/functional/shared/nix_expressions/ec2_base.nix'.format(root_dir), - ], - # gce - [ - '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), - '{}/tests/functional/shared/nix_expressions/gce_base.nix'.format(root_dir), - ], - # azure - [ - '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), - '{}/tests/functional/shared/nix_expressions/azure_base.nix'.format(root_dir), - ], - # libvirtd - [ - '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), - '{}/tests/functional/shared/nix_expressions/libvirtd_base.nix'.format(root_dir), - ] + 'json', + 'nixops' + ], + [ + ( + 'vbox', + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/vbox_base.nix'.format(root_dir), + ] + ), + ( + 'ec2', + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/ec2_base.nix'.format(root_dir), + ], + ), + ( + 'gce', + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/gce_base.nix'.format(root_dir), + ], + ), + ( + 'azure', + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/azure_base.nix'.format(root_dir), + ], + ), + ( + 'libvirtd', + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/libvirtd_base.nix'.format(root_dir), + ] + ) ], )) -def test_cloning_clones(state_extension, nix_expressions): - with using_state_file(state_extension) as state: +def test_cloning_clones(state_extension, nix_expressions_tuple): + nix_expressions_id, nix_expressions = nix_expressions_tuple + + with using_unique_state_file( + [test_cloning_clones.__name__, nix_expressions_id], + state_extension + ) as state: deployment = create_deployment(state, nix_expressions) deployment2 = deployment.clone() diff --git a/tests/functional/test_deleting_deletes/__init__.py b/tests/functional/test_deleting_deletes/__init__.py index 285cfd1fb..1c840a38a 100644 --- a/tests/functional/test_deleting_deletes/__init__.py +++ b/tests/functional/test_deleting_deletes/__init__.py @@ -5,41 +5,59 @@ from tests.functional.shared.deployment_run_command import deployment_run_command from tests.functional.shared.create_deployment import create_deployment -from tests.functional.shared.using_state_file import using_state_file +from tests.functional.shared.using_unique_state_file import using_unique_state_file @parameterized(product( - ['json', 'nixops'], [ - # vbox - [ - '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), - '{}/tests/functional/shared/nix_expressions/vbox_base.nix'.format(root_dir), - ], - # ec2 - [ - '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), - '{}/tests/functional/shared/nix_expressions/ec2_base.nix'.format(root_dir), - ], - # gce - [ - '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), - '{}/tests/functional/shared/nix_expressions/gce_base.nix'.format(root_dir), - ], - # azure - [ - '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), - '{}/tests/functional/shared/nix_expressions/azure_base.nix'.format(root_dir), - ], - # libvirtd - [ - '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), - '{}/tests/functional/shared/nix_expressions/libvirtd_base.nix'.format(root_dir), - ] + 'json', + 'nixops' + ], + [ + ( + 'vbox', + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/vbox_base.nix'.format(root_dir), + ] + ), + ( + 'ec2', + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/ec2_base.nix'.format(root_dir), + ], + ), + ( + 'gce', + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/gce_base.nix'.format(root_dir), + ], + ), + ( + 'azure', + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/azure_base.nix'.format(root_dir), + ], + ), + ( + 'libvirtd', + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/libvirtd_base.nix'.format(root_dir), + ] + ) ], )) -def test_deleting_deletes(state_extension, nix_expressions): - with using_state_file(state_extension) as state: +def test_deleting_deletes(state_extension, nix_expressions_tuple): + nix_expressions_id, nix_expressions = nix_expressions_tuple + + with using_unique_state_file( + [test_deleting_deletes.__name__, nix_expressions_id], + state_extension + ) as state: deployment = create_deployment(state, nix_expressions) uuid = deployment.uuid diff --git a/tests/functional/test_deploys_nixos/__init__.py b/tests/functional/test_deploys_nixos/__init__.py index de47feeff..d88253c3b 100644 --- a/tests/functional/test_deploys_nixos/__init__.py +++ b/tests/functional/test_deploys_nixos/__init__.py @@ -4,40 +4,66 @@ from tests.functional.shared.deployment_run_command import deployment_run_command from tests.functional.shared.create_deployment import create_deployment -from tests.functional.shared.using_state_file import using_state_file +from tests.functional.shared.using_unique_state_file import using_unique_state_file @parameterized(product( - ['json', 'nixops'], [ - # vbox - [ - '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), - '{}/tests/functional/shared/nix_expressions/vbox_base.nix'.format(root_dir), - ], - # ec2 - [ - '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), - '{}/tests/functional/shared/nix_expressions/ec2_base.nix'.format(root_dir), - ], - # gce - [ - '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), - '{}/tests/functional/shared/nix_expressions/gce_base.nix'.format(root_dir), - ], - # azure - [ - '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), - '{}/tests/functional/shared/nix_expressions/azure_base.nix'.format(root_dir), - ], - # libvirtd - [ - '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), - '{}/tests/functional/shared/nix_expressions/libvirtd_base.nix'.format(root_dir), - ] + 'json', + 'nixops' + ], + [ + ( + 'vbox', + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/vbox_base.nix'.format(root_dir), + ] + ), + ( + 'ec2', + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/ec2_base.nix'.format(root_dir), + ], + ), + ( + 'ec2_spot', + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/ec2_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/ec2_spot_instance.nix'.format(root_dir) + ], + ), + ( + 'gce', + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/gce_base.nix'.format(root_dir), + ], + ), + ( + 'azure', + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/azure_base.nix'.format(root_dir), + ], + ), + ( + 'libvirtd', + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/libvirtd_base.nix'.format(root_dir), + ] + ) ], )) -def test_deploys_nixos(state_extension, nix_expressions): - with using_state_file(state_extension) as state: +def test_deploys_nixos(state_extension, nix_expressions_tuple): + nix_expressions_id, nix_expressions = nix_expressions_tuple + + with using_unique_state_file( + [test_deploys_nixos.__name__, nix_expressions_id], + state_extension + ) as state: deployment = create_deployment(state, nix_expressions) deployment.deploy() deployment_run_command(deployment, "test -f /etc/NIXOS") diff --git a/tests/functional/test_ec2_backups/__init__.py b/tests/functional/test_ec2_backups/__init__.py index 222ffe3c8..d0a1f5906 100644 --- a/tests/functional/test_ec2_backups/__init__.py +++ b/tests/functional/test_ec2_backups/__init__.py @@ -6,10 +6,13 @@ from tests.functional.shared.deployment_run_command import deployment_run_command from tests.functional.shared.create_deployment import create_deployment -from tests.functional.shared.using_state_file import using_state_file +from tests.functional.shared.using_unique_state_file import using_unique_state_file @parameterized(product( - ['json', 'nixops'], + [ + 'json', + 'nixops' + ], [ [ '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), @@ -19,12 +22,18 @@ ], )) def test_ec2_backups_simple(state_extension, nix_expressions): - with using_state_file(state_extension) as state: + with using_unique_state_file( + [test_ec2_backups_simple.__name__], + state_extension + ) as state: deployment = create_deployment(state, nix_expressions) backup_and_restore_path(deployment) @parameterized(product( - ['json', 'nixops'], + [ + 'json', + 'nixops' + ], [ [ '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), @@ -35,7 +44,10 @@ def test_ec2_backups_simple(state_extension, nix_expressions): ], )) def test_ec2_backups_raid(state_extension, nix_expressions): - with using_state_file(state_extension) as state: + with using_unique_state_file( + [test_ec2_backups_raid.__name__], + state_extension + ) as state: deployment = create_deployment(state, nix_expressions) backup_and_restore_path(deployment, '/data') diff --git a/tests/functional/test_ec2_rds_dbinstance/__init__.py b/tests/functional/test_ec2_rds_dbinstance/__init__.py index e50ac4c65..92af900ca 100644 --- a/tests/functional/test_ec2_rds_dbinstance/__init__.py +++ b/tests/functional/test_ec2_rds_dbinstance/__init__.py @@ -6,8 +6,7 @@ from tests.functional.shared.deployment_run_command import deployment_run_command from tests.functional.shared.create_deployment import create_deployment -from tests.functional.shared.using_state_file import using_state_file -from tests.functional.shared.unique_state_file_path import unique_state_file_path +from tests.functional.shared.using_unique_state_file import using_unique_state_file parent_dir = path.dirname(__file__) @@ -34,11 +33,9 @@ def test_ec2_rds_dbinstance(state_extension, nix_expressions_tuple): nix_expressions_id, nix_expressions = nix_expressions_tuple - with using_state_file( - unique_state_file_path( - ['test_ec2_rds_dbinstance', nix_expressions_id], - state_extension - ) + with using_unique_state_file( + [test_ec2_rds_dbinstance.__name__, nix_expressions_id], + state_extension ) as state: deployment = create_deployment(state, nix_expressions) deployment.deploy() diff --git a/tests/functional/test_ec2_spot_instance/__init__.py b/tests/functional/test_ec2_spot_instance/__init__.py deleted file mode 100644 index 086856d3a..000000000 --- a/tests/functional/test_ec2_spot_instance/__init__.py +++ /dev/null @@ -1,24 +0,0 @@ -from nixops.util import root_dir -from itertools import product -from parameterized import parameterized - -from tests.functional.shared.deployment_run_command import deployment_run_command -from tests.functional.shared.create_deployment import create_deployment -from tests.functional.shared.using_state_file import using_state_file - -@parameterized(product( - ['json', 'nixops'], - [ - [ - '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), - '{}/tests/functional/shared/nix_expressions/ec2_base.nix'.format(root_dir), - '{}/tests/functional/shared/nix_expressions/ec2_spot_instance.nix'.format(root_dir) - ] - ] -)) - -def test_ec2_spot_instance(state_extension, nix_expressions): - with using_state_file(state_extension) as state: - deployment = create_deployment(state, nix_expressions) - deployment.deploy() - deployment_run_command(deployment, "test -f /etc/NIXOS") From a681561f1b791fa2e791fcccf26a4db016c3aa5a Mon Sep 17 00:00:00 2001 From: srghma Date: Sat, 16 Jun 2018 12:11:45 +0300 Subject: [PATCH 094/123] fix: json, when clone_deployment -> local variable "uuid" referenced before assignment --- nixops/state/json_file.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/nixops/state/json_file.py b/nixops/state/json_file.py index 96da7c680..3a906ad5c 100644 --- a/nixops/state/json_file.py +++ b/nixops/state/json_file.py @@ -205,20 +205,20 @@ def _delete_deployment(self, deployment_uuid): def clone_deployment(self, deployment_uuid): with self.db: - if not uuid: - import uuid - new_uuid = str(uuid.uuid1()) + import uuid + new_deployment_uuid = str(uuid.uuid1()) + state = self.db.read() cloned_attributes = copy.deepcopy(state["deployments"][deployment_uuid]["attributes"]) - state["deployments"][new_uuid] = { + state["deployments"][new_deployment_uuid] = { "attributes": cloned_attributes, "resources": {} } self.db.set(state) - return self._find_deployment(new_uuid) + return self._find_deployment(new_deployment_uuid) def get_resources_for(self, deployment): """Get all the resources for a certain deployment""" From 5f4cd5de081169ad682a51aa138da63c6423265f Mon Sep 17 00:00:00 2001 From: srghma Date: Sat, 16 Jun 2018 14:10:16 +0300 Subject: [PATCH 095/123] fix: tests --- .../functional/test_invalid_indentifier/__init__.py | 13 ++++++++++--- tests/functional/test_query_deployments/__init__.py | 13 ++++++++++--- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/tests/functional/test_invalid_indentifier/__init__.py b/tests/functional/test_invalid_indentifier/__init__.py index 2f7e6932a..8ad813300 100644 --- a/tests/functional/test_invalid_indentifier/__init__.py +++ b/tests/functional/test_invalid_indentifier/__init__.py @@ -1,15 +1,19 @@ from nose import tools from itertools import product +from os import path from parameterized import parameterized from tests.functional.shared.deployment_run_command import deployment_run_command from tests.functional.shared.create_deployment import create_deployment -from tests.functional.shared.using_state_file import using_state_file +from tests.functional.shared.using_unique_state_file import using_unique_state_file parent_dir = path.dirname(__file__) @parameterized(product( - ['json', 'nixops'], + [ + 'json', + 'nixops' + ], [ [ '{}/invalid-identifier.nix'.format(parent_dir), @@ -17,7 +21,10 @@ ] )) def test_invalid_indentifier(state_extension, nix_expressions): - with using_state_file(state_extension) as state: + with using_unique_state_file( + [test_invalid_indentifier.__name__], + state_extension + ) as state: deployment = create_deployment(state, nix_expressions) with tools.assert_raises(Exception): deployment.evaluate() diff --git a/tests/functional/test_query_deployments/__init__.py b/tests/functional/test_query_deployments/__init__.py index 5e60b2d7b..50d9ff474 100644 --- a/tests/functional/test_query_deployments/__init__.py +++ b/tests/functional/test_query_deployments/__init__.py @@ -3,13 +3,20 @@ from tests.functional.shared.deployment_run_command import deployment_run_command from tests.functional.shared.create_deployment import create_deployment -from tests.functional.shared.using_state_file import using_state_file +from tests.functional.shared.using_unique_state_file import using_unique_state_file @parameterized( - ['json', 'nixops'] + [ + 'json', + 'nixops' + ] ) def test_query_deployments(state_extension): - with using_state_file(state_extension) as state: + with using_unique_state_file( + [test_query_deployments.__name__], + state_extension + ) as state: + deployments = [] for i in range(10): From f453d6e21619a79678fd52cde147f978591d5cde Mon Sep 17 00:00:00 2001 From: srghma Date: Sat, 16 Jun 2018 15:15:30 +0300 Subject: [PATCH 096/123] fix: tests.functional.test_query_deployments -> No JSON object could be decoded --- nixops/state/json_file.py | 9 +++++---- tests/functional/shared/using_state_file.py | 3 +++ tests/functional/test_query_deployments/__init__.py | 2 +- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/nixops/state/json_file.py b/nixops/state/json_file.py index 3a906ad5c..81d3cc59e 100644 --- a/nixops/state/json_file.py +++ b/nixops/state/json_file.py @@ -30,7 +30,6 @@ class TransactionalJsonFile: # Implementation notes: # if self.nesting > 0, then no write will propagate. def __init__(self, db_file): - lock_file_path = re.sub("\.json$", ".lock", db_file) self._lock_file = open(lock_file_path, "w") fcntl.fcntl(self._lock_file, fcntl.F_SETFD, fcntl.FD_CLOEXEC) # to not keep the lock in child processes @@ -154,6 +153,7 @@ def get_all_deployments(self): def _find_deployment(self, uuid=None): all_deployments = self.db.read()["deployments"] + found = [] # if nothing exists no reason to check for things @@ -181,9 +181,10 @@ def _find_deployment(self, uuid=None): def open_deployment(self, uuid=None): """Open an existing deployment.""" - deployment = self._find_deployment(uuid=uuid) - if deployment: return deployment - raise Exception("could not find specified deployment in state file ‘{0}’".format(self.db_file)) + with self.db: + deployment = self._find_deployment(uuid=uuid) + if deployment: return deployment + raise Exception("could not find specified deployment in state file ‘{0}’".format(self.db_file)) def create_deployment(self, uuid=None): """Create a new deployment.""" diff --git a/tests/functional/shared/using_state_file.py b/tests/functional/shared/using_state_file.py index 730478242..a25c672ad 100644 --- a/tests/functional/shared/using_state_file.py +++ b/tests/functional/shared/using_state_file.py @@ -33,6 +33,9 @@ def destroy_deployments(state, uuid): try: deployment.clean_backups(keep=0) except Exception: + # TODO: clean_backups() takes at least 3 arguments (2 given) + # import sys; import pprint; pprint.pprint(('clean_backups', e), stream=sys.__stdout__) + pass try: deployment.destroy_resources() diff --git a/tests/functional/test_query_deployments/__init__.py b/tests/functional/test_query_deployments/__init__.py index 50d9ff474..148486080 100644 --- a/tests/functional/test_query_deployments/__init__.py +++ b/tests/functional/test_query_deployments/__init__.py @@ -8,7 +8,7 @@ @parameterized( [ 'json', - 'nixops' + # 'nixops' ] ) def test_query_deployments(state_extension): From 3c6433d183c21af25e073fa0dd5a29a1d93f3045 Mon Sep 17 00:00:00 2001 From: srghma Date: Sat, 16 Jun 2018 17:53:52 +0300 Subject: [PATCH 097/123] fix: tests (WIP) --- tests/functional/shared/create_deployment.py | 2 +- .../test_cloning_clones/__init__.py | 14 +-- .../test_deleting_deletes/__init__.py | 14 +-- .../functional/test_deploys_nixos/__init__.py | 14 +-- .../test_query_deployments/__init__.py | 2 +- .../test_rebooting_reboots/__init__.py | 76 +++++++++------ .../test_rollback_rollsback/__init__.py | 92 +++++++++++------- .../has_hello.nix | 0 .../rollback.nix | 0 .../test_send_keys_sends_keys/__init__.py | 89 ++++++++++------- .../elsewhere_key.nix | 0 .../secret_key.nix | 0 .../test_starting_starts/__init__.py | 76 +++++++++------ .../test_stopping_stops/__init__.py | 76 +++++++++------ .../test_vbox_encrypted_links/__init__.py | 22 +++-- .../test_vbox_encrypted_links/helpers.py | 2 + tests/functional/test_vpc/__init__.py | 96 +++++++++++-------- tests/functional/test_vpc/helpers.py | 20 ++++ 18 files changed, 369 insertions(+), 226 deletions(-) rename tests/functional/{shared/nix_expressions => test_rollback_rollsback}/has_hello.nix (100%) rename tests/functional/{shared/nix_expressions => test_rollback_rollsback}/rollback.nix (100%) rename tests/functional/{shared/nix_expressions => test_send_keys_sends_keys}/elsewhere_key.nix (100%) rename tests/functional/{shared/nix_expressions => test_send_keys_sends_keys}/secret_key.nix (100%) create mode 100644 tests/functional/test_vbox_encrypted_links/helpers.py create mode 100644 tests/functional/test_vpc/helpers.py diff --git a/tests/functional/shared/create_deployment.py b/tests/functional/shared/create_deployment.py index 67ac862eb..5b17932b0 100644 --- a/tests/functional/shared/create_deployment.py +++ b/tests/functional/shared/create_deployment.py @@ -1,4 +1,4 @@ -def create_deployment(state, nix_expressions): +def create_deployment(state, nix_expressions=[]): deployment = state.create_deployment() deployment.logger.set_autoresponse("y") deployment.nix_exprs = nix_expressions diff --git a/tests/functional/test_cloning_clones/__init__.py b/tests/functional/test_cloning_clones/__init__.py index 570b3435e..d588f1cea 100644 --- a/tests/functional/test_cloning_clones/__init__.py +++ b/tests/functional/test_cloning_clones/__init__.py @@ -20,6 +20,13 @@ '{}/tests/functional/shared/nix_expressions/vbox_base.nix'.format(root_dir), ] ), + ( + 'libvirtd', + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/libvirtd_base.nix'.format(root_dir), + ] + ), ( 'ec2', [ @@ -40,13 +47,6 @@ '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), '{}/tests/functional/shared/nix_expressions/azure_base.nix'.format(root_dir), ], - ), - ( - 'libvirtd', - [ - '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), - '{}/tests/functional/shared/nix_expressions/libvirtd_base.nix'.format(root_dir), - ] ) ], )) diff --git a/tests/functional/test_deleting_deletes/__init__.py b/tests/functional/test_deleting_deletes/__init__.py index 1c840a38a..80aab4527 100644 --- a/tests/functional/test_deleting_deletes/__init__.py +++ b/tests/functional/test_deleting_deletes/__init__.py @@ -20,6 +20,13 @@ '{}/tests/functional/shared/nix_expressions/vbox_base.nix'.format(root_dir), ] ), + ( + 'libvirtd', + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/libvirtd_base.nix'.format(root_dir), + ] + ), ( 'ec2', [ @@ -40,13 +47,6 @@ '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), '{}/tests/functional/shared/nix_expressions/azure_base.nix'.format(root_dir), ], - ), - ( - 'libvirtd', - [ - '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), - '{}/tests/functional/shared/nix_expressions/libvirtd_base.nix'.format(root_dir), - ] ) ], )) diff --git a/tests/functional/test_deploys_nixos/__init__.py b/tests/functional/test_deploys_nixos/__init__.py index d88253c3b..37ebe917c 100644 --- a/tests/functional/test_deploys_nixos/__init__.py +++ b/tests/functional/test_deploys_nixos/__init__.py @@ -19,6 +19,13 @@ '{}/tests/functional/shared/nix_expressions/vbox_base.nix'.format(root_dir), ] ), + ( + 'libvirtd', + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/libvirtd_base.nix'.format(root_dir), + ] + ), ( 'ec2', [ @@ -47,13 +54,6 @@ '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), '{}/tests/functional/shared/nix_expressions/azure_base.nix'.format(root_dir), ], - ), - ( - 'libvirtd', - [ - '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), - '{}/tests/functional/shared/nix_expressions/libvirtd_base.nix'.format(root_dir), - ] ) ], )) diff --git a/tests/functional/test_query_deployments/__init__.py b/tests/functional/test_query_deployments/__init__.py index 148486080..50d9ff474 100644 --- a/tests/functional/test_query_deployments/__init__.py +++ b/tests/functional/test_query_deployments/__init__.py @@ -8,7 +8,7 @@ @parameterized( [ 'json', - # 'nixops' + 'nixops' ] ) def test_query_deployments(state_extension): diff --git a/tests/functional/test_rebooting_reboots/__init__.py b/tests/functional/test_rebooting_reboots/__init__.py index 75b6227bf..ae542cfdb 100644 --- a/tests/functional/test_rebooting_reboots/__init__.py +++ b/tests/functional/test_rebooting_reboots/__init__.py @@ -6,40 +6,58 @@ from tests.functional.shared.deployment_run_command import deployment_run_command from tests.functional.shared.create_deployment import create_deployment -from tests.functional.shared.using_state_file import using_state_file +from tests.functional.shared.using_unique_state_file import using_unique_state_file @parameterized(product( - ['json', 'nixops'], [ - # vbox - [ - '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), - '{}/tests/functional/shared/nix_expressions/vbox_base.nix'.format(root_dir), - ], - # ec2 - [ - '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), - '{}/tests/functional/shared/nix_expressions/ec2_base.nix'.format(root_dir), - ], - # gce - [ - '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), - '{}/tests/functional/shared/nix_expressions/gce_base.nix'.format(root_dir), - ], - # azure - [ - '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), - '{}/tests/functional/shared/nix_expressions/azure_base.nix'.format(root_dir), - ], - # libvirtd - [ - '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), - '{}/tests/functional/shared/nix_expressions/libvirtd_base.nix'.format(root_dir), - ] + 'json', + 'nixops' + ], + [ + ( + 'vbox', + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/vbox_base.nix'.format(root_dir), + ] + ), + ( + 'libvirtd', + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/libvirtd_base.nix'.format(root_dir), + ] + ), + ( + 'ec2', + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/ec2_base.nix'.format(root_dir), + ], + ), + ( + 'gce', + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/gce_base.nix'.format(root_dir), + ], + ), + ( + 'azure', + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/azure_base.nix'.format(root_dir), + ], + ) ], )) -def test_rebooting_reboots(state_extension, nix_expressions): - with using_state_file(state_extension) as state: +def test_rebooting_reboots(state_extension, nix_expressions_tuple): + nix_expressions_id, nix_expressions = nix_expressions_tuple + + with using_unique_state_file( + [test_rebooting_reboots.__name__, nix_expressions_id], + state_extension + ) as state: deployment = create_deployment(state, nix_expressions) deployment.deploy() deployment_run_command(deployment, "touch /run/not-rebooted") diff --git a/tests/functional/test_rollback_rollsback/__init__.py b/tests/functional/test_rollback_rollsback/__init__.py index 981fa4323..1df836213 100644 --- a/tests/functional/test_rollback_rollsback/__init__.py +++ b/tests/functional/test_rollback_rollsback/__init__.py @@ -3,55 +3,79 @@ from nixops.util import root_dir from itertools import product from parameterized import parameterized +from os import path from tests.functional.shared.deployment_run_command import deployment_run_command from tests.functional.shared.create_deployment import create_deployment -from tests.functional.shared.using_state_file import using_state_file +from tests.functional.shared.using_unique_state_file import using_unique_state_file -has_hello_spec = '{}/tests/functional/shared/nix_expressions/has_hello.nix'.format(root_dir) -rollback_spec = '{}/tests/functional/shared/nix_expressions/rollback.nix'.format(root_dir) +parent_dir = path.dirname(__file__) + +has_hello_spec = '{}/has_hello.nix'.format(parent_dir) +rollback_spec = '{}/rollback.nix'.format(parent_dir) @parameterized(product( - ['json', 'nixops'], [ - # vbox - [ - '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), - '{}/tests/functional/shared/nix_expressions/vbox_base.nix'.format(root_dir), - ], - # ec2 - [ - '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), - '{}/tests/functional/shared/nix_expressions/ec2_base.nix'.format(root_dir), - ], - # gce - [ - '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), - '{}/tests/functional/shared/nix_expressions/gce_base.nix'.format(root_dir), - ], - # azure - [ - '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), - '{}/tests/functional/shared/nix_expressions/azure_base.nix'.format(root_dir), - ], - # libvirtd - [ - '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), - '{}/tests/functional/shared/nix_expressions/libvirtd_base.nix'.format(root_dir), - ] + 'json', + 'nixops' + ], + [ + ( + 'vbox', + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/vbox_base.nix'.format(root_dir), + ] + ), + ( + 'libvirtd', + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/libvirtd_base.nix'.format(root_dir), + ] + ), + ( + 'ec2', + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/ec2_base.nix'.format(root_dir), + ], + ), + ( + 'gce', + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/gce_base.nix'.format(root_dir), + ], + ), + ( + 'azure', + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/azure_base.nix'.format(root_dir), + ], + ) ], )) -def test_rollback_rollsback(state_extension, nix_expressions): - with using_state_file(state_extension) as state: - deployment = create_deployment(state, nix_expressions) +def test_rollback_rollsback(state_extension, nix_expressions_tuple): + nix_expressions_id, nix_expressions = nix_expressions_tuple - deployment.nix_exprs = deployment.nix_exprs + [ rollback_spec ] + with using_unique_state_file( + [test_rollback_rollsback.__name__, nix_expressions_id], + state_extension + ) as state: + nix_expressions_ = nix_expressions + [ rollback_spec ] + + deployment = create_deployment(state) + deployment.nix_exprs = nix_expressions_ deployment.deploy() with tools.assert_raises(SSHCommandFailed): deployment_run_command(deployment, "hello") - deployment.nix_exprs = deployment.nix_exprs + [ has_hello_spec ] + nix_expressions__ = nix_expressions + [ rollback_spec, has_hello_spec ] + + deployment.nix_exprs = nix_expressions__ deployment.deploy() deployment_run_command(deployment, "hello") diff --git a/tests/functional/shared/nix_expressions/has_hello.nix b/tests/functional/test_rollback_rollsback/has_hello.nix similarity index 100% rename from tests/functional/shared/nix_expressions/has_hello.nix rename to tests/functional/test_rollback_rollsback/has_hello.nix diff --git a/tests/functional/shared/nix_expressions/rollback.nix b/tests/functional/test_rollback_rollsback/rollback.nix similarity index 100% rename from tests/functional/shared/nix_expressions/rollback.nix rename to tests/functional/test_rollback_rollsback/rollback.nix diff --git a/tests/functional/test_send_keys_sends_keys/__init__.py b/tests/functional/test_send_keys_sends_keys/__init__.py index 1cb1f3b51..ae81124eb 100644 --- a/tests/functional/test_send_keys_sends_keys/__init__.py +++ b/tests/functional/test_send_keys_sends_keys/__init__.py @@ -3,55 +3,78 @@ from nixops.util import root_dir from itertools import product from parameterized import parameterized +from os import path from tests.functional.shared.deployment_run_command import deployment_run_command from tests.functional.shared.create_deployment import create_deployment -from tests.functional.shared.using_state_file import using_state_file +from tests.functional.shared.using_unique_state_file import using_unique_state_file -secret_key_spec = '{}/tests/functional/shared/nix_expressions/single_machine_secret_key.nix'.format(root_dir) -elsewhere_key_spec = '{}/tests/functional/shared/nix_expressions/single_machine_elsewhere_key.nix'.format(root_dir) +parent_dir = path.dirname(__file__) + +secret_key_spec = '{}/secret_key.nix'.format(parent_dir) +elsewhere_key_spec = '{}/elsewhere_key.nix'.format(parent_dir) @parameterized(product( - ['json', 'nixops'], [ - # vbox - [ - '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), - '{}/tests/functional/shared/nix_expressions/vbox_base.nix'.format(root_dir), - ], - # ec2 - [ - '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), - '{}/tests/functional/shared/nix_expressions/ec2_base.nix'.format(root_dir), - ], - # gce - [ - '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), - '{}/tests/functional/shared/nix_expressions/gce_base.nix'.format(root_dir), - ], - # azure - [ - '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), - '{}/tests/functional/shared/nix_expressions/azure_base.nix'.format(root_dir), - ], - # libvirtd - [ - '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), - '{}/tests/functional/shared/nix_expressions/libvirtd_base.nix'.format(root_dir), - ] + 'json', + 'nixops' + ], + [ + ( + 'vbox', + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/vbox_base.nix'.format(root_dir), + ] + ), + ( + 'libvirtd', + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/libvirtd_base.nix'.format(root_dir), + ] + ), + ( + 'ec2', + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/ec2_base.nix'.format(root_dir), + ], + ), + ( + 'gce', + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/gce_base.nix'.format(root_dir), + ], + ), + ( + 'azure', + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/azure_base.nix'.format(root_dir), + ], + ) ], )) -def test_send_keys_sends_keys(state_extension, nix_expressions): - with using_state_file(state_extension) as state: - deployment = create_deployment(state, nix_expressions) +def test_send_keys_sends_keys(state_extension, nix_expressions_tuple): + nix_expressions_id, nix_expressions = nix_expressions_tuple - deployment.nix_exprs = deployment.nix_exprs + [ secret_key_spec, elsewhere_key_spec ] + with using_unique_state_file( + [test_send_keys_sends_keys.__name__, nix_expressions_id], + state_extension + ) as state: + nix_expressions_ = nix_expressions + [ secret_key_spec, elsewhere_key_spec ] + + deployment = create_deployment(state, nix_expressions_) deployment.deploy() deployment_run_command(deployment, "test -f /run/keys/secret.key") deployment_run_command(deployment, "rm -f /run/keys/secret.key") deployment_run_command(deployment, "test -f /new/directory/elsewhere.key") deployment_run_command(deployment, "rm -f /new/directory/elsewhere.key") + deployment.send_keys() + deployment_run_command(deployment, "test -f /run/keys/secret.key") deployment_run_command(deployment, "test -f /new/directory/elsewhere.key") diff --git a/tests/functional/shared/nix_expressions/elsewhere_key.nix b/tests/functional/test_send_keys_sends_keys/elsewhere_key.nix similarity index 100% rename from tests/functional/shared/nix_expressions/elsewhere_key.nix rename to tests/functional/test_send_keys_sends_keys/elsewhere_key.nix diff --git a/tests/functional/shared/nix_expressions/secret_key.nix b/tests/functional/test_send_keys_sends_keys/secret_key.nix similarity index 100% rename from tests/functional/shared/nix_expressions/secret_key.nix rename to tests/functional/test_send_keys_sends_keys/secret_key.nix diff --git a/tests/functional/test_starting_starts/__init__.py b/tests/functional/test_starting_starts/__init__.py index 629c0c6c4..cf58b8d5f 100644 --- a/tests/functional/test_starting_starts/__init__.py +++ b/tests/functional/test_starting_starts/__init__.py @@ -6,40 +6,58 @@ from tests.functional.shared.deployment_run_command import deployment_run_command from tests.functional.shared.create_deployment import create_deployment -from tests.functional.shared.using_state_file import using_state_file +from tests.functional.shared.using_unique_state_file import using_unique_state_file @parameterized(product( - ['json', 'nixops'], [ - # vbox - [ - '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), - '{}/tests/functional/shared/nix_expressions/vbox_base.nix'.format(root_dir), - ], - # ec2 - [ - '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), - '{}/tests/functional/shared/nix_expressions/ec2_base.nix'.format(root_dir), - ], - # gce - [ - '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), - '{}/tests/functional/shared/nix_expressions/gce_base.nix'.format(root_dir), - ], - # azure - [ - '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), - '{}/tests/functional/shared/nix_expressions/azure_base.nix'.format(root_dir), - ], - # libvirtd - [ - '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), - '{}/tests/functional/shared/nix_expressions/libvirtd_base.nix'.format(root_dir), - ] + 'json', + 'nixops' + ], + [ + ( + 'vbox', + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/vbox_base.nix'.format(root_dir), + ] + ), + ( + 'libvirtd', + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/libvirtd_base.nix'.format(root_dir), + ] + ), + ( + 'ec2', + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/ec2_base.nix'.format(root_dir), + ], + ), + ( + 'gce', + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/gce_base.nix'.format(root_dir), + ], + ), + ( + 'azure', + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/azure_base.nix'.format(root_dir), + ], + ) ], )) -def test_starting_starts(state_extension, nix_expressions): - with using_state_file(state_extension) as state: +def test_starting_starts(state_extension, nix_expressions_tuple): + nix_expressions_id, nix_expressions = nix_expressions_tuple + + with using_unique_state_file( + [test_starting_starts.__name__, nix_expressions_id], + state_extension + ) as state: deployment = create_deployment(state, nix_expressions) deployment.deploy() deployment.stop_machines() diff --git a/tests/functional/test_stopping_stops/__init__.py b/tests/functional/test_stopping_stops/__init__.py index 3c79654e6..fd2e7156d 100644 --- a/tests/functional/test_stopping_stops/__init__.py +++ b/tests/functional/test_stopping_stops/__init__.py @@ -6,40 +6,58 @@ from tests.functional.shared.deployment_run_command import deployment_run_command from tests.functional.shared.create_deployment import create_deployment -from tests.functional.shared.using_state_file import using_state_file +from tests.functional.shared.using_unique_state_file import using_unique_state_file @parameterized(product( - ['json', 'nixops'], [ - # vbox - [ - '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), - '{}/tests/functional/shared/nix_expressions/vbox_base.nix'.format(root_dir), - ], - # ec2 - [ - '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), - '{}/tests/functional/shared/nix_expressions/ec2_base.nix'.format(root_dir), - ], - # gce - [ - '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), - '{}/tests/functional/shared/nix_expressions/gce_base.nix'.format(root_dir), - ], - # azure - [ - '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), - '{}/tests/functional/shared/nix_expressions/azure_base.nix'.format(root_dir), - ], - # libvirtd - [ - '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), - '{}/tests/functional/shared/nix_expressions/libvirtd_base.nix'.format(root_dir), - ] + 'json', + 'nixops' + ], + [ + ( + 'vbox', + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/vbox_base.nix'.format(root_dir), + ] + ), + ( + 'libvirtd', + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/libvirtd_base.nix'.format(root_dir), + ] + ), + ( + 'ec2', + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/ec2_base.nix'.format(root_dir), + ], + ), + ( + 'gce', + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/gce_base.nix'.format(root_dir), + ], + ), + ( + 'azure', + [ + '{}/tests/functional/shared/nix_expressions/logical_base.nix'.format(root_dir), + '{}/tests/functional/shared/nix_expressions/azure_base.nix'.format(root_dir), + ], + ) ], )) -def test_stopping_stops(state_extension, nix_expressions): - with using_state_file(state_extension) as state: +def test_stopping_stops(state_extension, nix_expressions_tuple): + nix_expressions_id, nix_expressions = nix_expressions_tuple + + with using_unique_state_file( + [test_stopping_stops.__name__, nix_expressions_id], + state_extension + ) as state: deployment = create_deployment(state, nix_expressions) deployment.deploy() deployment.stop_machines() diff --git a/tests/functional/test_vbox_encrypted_links/__init__.py b/tests/functional/test_vbox_encrypted_links/__init__.py index 3761b4574..6e7818599 100644 --- a/tests/functional/test_vbox_encrypted_links/__init__.py +++ b/tests/functional/test_vbox_encrypted_links/__init__.py @@ -11,17 +11,23 @@ from tests.functional.shared.deployment_run_command import deployment_run_command from tests.functional.shared.create_deployment import create_deployment -from tests.functional.shared.using_state_file import using_state_file +from tests.functional.shared.using_unique_state_file import using_unique_state_file + +from tests.functional.test_vbox_encrypted_links.helpers import ping parent_dir = path.dirname(__file__) logical_spec = '{}/encrypted-links.nix'.format(parent_dir) -@parameterized( - ['json', 'nixops'] -) +@parameterized([ + 'json', + 'nixops' +]) def test_vbox_encrypted_links(state_extension): - with using_state_file(state_extension) as state: + with using_unique_state_file( + [test_vbox_encrypted_links.__name__], + state_extension + ) as state: deployment = create_deployment(state, [logical_spec]) deployment.debug = True @@ -30,6 +36,7 @@ def test_vbox_encrypted_links(state_extension): # !!! Shouldn't need this, instead the encrypted links target # should wait until the link is active... time.sleep(1) + ping(deployment, "machine1", "machine2") ping(deployment, "machine2", "machine1") @@ -41,8 +48,3 @@ def test_vbox_encrypted_links(state_extension): with tools.assert_raises(SSHCommandFailed): ping(deployment, "machine2", "machine1") - -# Helpers - -def ping(deployment, machine1, machine2): - deployment.machines[machine1].run_command("ping -c1 {0}-encrypted".format(machine2)) diff --git a/tests/functional/test_vbox_encrypted_links/helpers.py b/tests/functional/test_vbox_encrypted_links/helpers.py new file mode 100644 index 000000000..38c82cb16 --- /dev/null +++ b/tests/functional/test_vbox_encrypted_links/helpers.py @@ -0,0 +1,2 @@ +def ping(deployment, machine1, machine2): + deployment.machines[machine1].run_command("ping -c1 {0}-encrypted".format(machine2)) diff --git a/tests/functional/test_vpc/__init__.py b/tests/functional/test_vpc/__init__.py index 1fa5dcd7a..a9d24154c 100644 --- a/tests/functional/test_vpc/__init__.py +++ b/tests/functional/test_vpc/__init__.py @@ -1,30 +1,44 @@ from os import path -import tempfile +from parameterized import parameterized from nose import tools from nose.plugins.attrib import attr import boto3 -import nixops.util +from tests.functional.shared.using_unique_state_file import using_unique_state_file +from tests.functional.shared.create_deployment import create_deployment parent_dir = path.dirname(__file__) base_spec = "{}/vpc.nix".format(parent_dir) -@parameterized(['json', 'nixops']) -def test_deploy_vpc(state_extension, nix_expressions): - with using_state_file(state_extension) as state: +@parameterized([ + 'json', + 'nixops' +]) +def test_deploy_vpc(state_extension): + with using_unique_state_file( + [test_deploy_vpc.__name__], + state_extension + ) as state: deployment = create_deployment(state, [base_spec]) deployment.deploy() + vpc_resource = deployment.get_typed_resource("vpc-test", "vpc") vpc = vpc_resource.get_client().describe_vpcs(VpcIds=[vpc_resource._state['vpcId']]) tools.ok_(len(vpc['Vpcs']) > 0, "VPC not found!") tools.eq_(vpc['Vpcs'][0]['CidrBlock'], "10.0.0.0/16", "CIDR block mismatch") -@parameterized(['json', 'nixops']) -def test_deploy_vpc_machine(state_extension, nix_expressions): - with using_state_file(state_extension) as state: +@parameterized([ + 'json', + 'nixops' +]) +def test_deploy_vpc_machine(state_extension): + with using_unique_state_file( + [test_deploy_vpc_machine.__name__], + state_extension + ) as state: nix_expressions = compose_expressions([CFG_SUBNET, CFG_INTERNET_ROUTE, CFG_VPC_MACHINE]) nix_expressions_ = [base_spec] + nix_expressions @@ -33,9 +47,15 @@ def test_deploy_vpc_machine(state_extension, nix_expressions): deployment.deploy(plan_only=True) deployment.deploy() -@parameterized(['json', 'nixops']) -def test_enable_dns_support(state_extension, nix_expressions): - with using_state_file(state_extension) as state: +@parameterized([ + 'json', + 'nixops' +]) +def test_enable_dns_support(state_extension): + with using_unique_state_file( + [test_enable_dns_support.__name__], + state_extension + ) as state: nix_expressions = compose_expressions([CFG_DNS_SUPPORT]) nix_expressions_ = [base_spec] + nix_expressions @@ -44,9 +64,15 @@ def test_enable_dns_support(state_extension, nix_expressions): deployment.deploy(plan_only=True) deployment.deploy() -@parameterized(['json', 'nixops']) +@parameterized([ + 'json', + 'nixops' +]) def test_enable_ipv6(): - with using_state_file(state_extension) as state: + with using_unique_state_file( + [test_enable_ipv6.__name__], + state_extension + ) as state: nix_expressions = compose_expressions([CFG_IPV6]) nix_expressions_ = [base_spec] + nix_expressions @@ -61,9 +87,15 @@ def test_enable_ipv6(): tools.ok_(len(ipv6_block) > 0, "There is no Ipv6 block") tools.ok_(ipv6_block[0].get('Ipv6CidrBlock', None) != None, "No Ipv6 cidr block in the response") -@parameterized(['json', 'nixops']) -def test_deploy_subnets(): - with using_state_file(state_extension) as state: +@parameterized([ + 'json', + 'nixops' +]) +def test_deploy_subnets(state_extension): + with using_unique_state_file( + [test_deploy_subnets.__name__], + state_extension + ) as state: # FIXME might need to factor out resources into separate test # classes depending on the number of tests needed. nix_expressions = compose_expressions([CFG_SUBNET]) @@ -77,9 +109,15 @@ def test_deploy_subnets(): subnet = subnet_resource.get_client().describe_subnets(SubnetIds=[subnet_resource._state['subnetId']]) tools.ok_(len(subnet['Subnets']) > 0, "VPC subnet not found!") -@parameterized(['json', 'nixops']) -def test_deploy_nat_gtw(): - with using_state_file(state_extension) as state: +@parameterized([ + 'json', + 'nixops' +]) +def test_deploy_nat_gtw(state_extension): + with using_unique_state_file( + [test_deploy_nat_gtw.__name__], + state_extension + ) as state: nix_expressions = compose_expressions([CFG_SUBNET, CFG_NAT_GTW]) nix_expressions_ = [base_spec] + nix_expressions @@ -87,23 +125,3 @@ def test_deploy_nat_gtw(): deployment.deploy(plan_only=True) deployment.deploy() - -# Helpers - -def create_exprs_dir(): - return nixops.util.SelfDeletingDir(tempfile.mkdtemp("nixos-tests")) - -def compose_expressions(configurations): - exprs_dir = create_exprs_dir() - - extra_exprs = list(map(lambda x: generate_config(exprs_dir, x), configurations)) - - nix_exprs = [base_spec] + extra_exprs - return nix_exprs - -def generate_config(exprs_dir, config): - basename, expr = config - expr_path = "{0}/{1}".format(exprs_dir, basename) - with open(expr_path, "w") as cfg: - cfg.write(expr) - return expr_path diff --git a/tests/functional/test_vpc/helpers.py b/tests/functional/test_vpc/helpers.py new file mode 100644 index 000000000..41043c938 --- /dev/null +++ b/tests/functional/test_vpc/helpers.py @@ -0,0 +1,20 @@ +import nixops.util +import tempfile + +def create_exprs_dir(): + return nixops.util.SelfDeletingDir(tempfile.mkdtemp("nixos-tests")) + +def compose_expressions(configurations): + exprs_dir = create_exprs_dir() + + extra_exprs = list(map(lambda x: generate_config(exprs_dir, x), configurations)) + + nix_exprs = [base_spec] + extra_exprs + return nix_exprs + +def generate_config(exprs_dir, config): + basename, expr = config + expr_path = "{0}/{1}".format(exprs_dir, basename) + with open(expr_path, "w") as cfg: + cfg.write(expr) + return expr_path From 458771e18fe30ebcbd1fd6f8cc73ee37436e79ab Mon Sep 17 00:00:00 2001 From: srghma Date: Sat, 16 Jun 2018 17:58:04 +0300 Subject: [PATCH 098/123] fix: tests -> shared -> destroy_deployments --- tests/functional/shared/using_state_file.py | 5 +---- tests/functional/test_vbox_encrypted_links/__init__.py | 1 - 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/functional/shared/using_state_file.py b/tests/functional/shared/using_state_file.py index a25c672ad..7e50e951e 100644 --- a/tests/functional/shared/using_state_file.py +++ b/tests/functional/shared/using_state_file.py @@ -31,11 +31,8 @@ def destroy_deployments(state, uuid): deployment = state.open_deployment(uuid) deployment.logger.set_autoresponse("y") try: - deployment.clean_backups(keep=0) + deployment.clean_backups(keep=False, keep_days=False) except Exception: - # TODO: clean_backups() takes at least 3 arguments (2 given) - # import sys; import pprint; pprint.pprint(('clean_backups', e), stream=sys.__stdout__) - pass try: deployment.destroy_resources() diff --git a/tests/functional/test_vbox_encrypted_links/__init__.py b/tests/functional/test_vbox_encrypted_links/__init__.py index 6e7818599..9471dfab6 100644 --- a/tests/functional/test_vbox_encrypted_links/__init__.py +++ b/tests/functional/test_vbox_encrypted_links/__init__.py @@ -47,4 +47,3 @@ def test_vbox_encrypted_links(state_extension): with tools.assert_raises(SSHCommandFailed): ping(deployment, "machine2", "machine1") - From 877b7fba6a858958d37244c66eef13fe925b9272 Mon Sep 17 00:00:00 2001 From: srghma Date: Sat, 16 Jun 2018 18:15:13 +0300 Subject: [PATCH 099/123] feat: clean_test_state_files.py --- clean_test_state_files.py | 15 +++++++ ...stroy_deployments_and_remove_state_file.py | 39 ++++++++++++++++++ .../shared/state_files_directory.py | 3 ++ .../shared/unique_state_file_path.py | 6 +-- tests/functional/shared/using_state_file.py | 40 ------------------- 5 files changed, 59 insertions(+), 44 deletions(-) create mode 100755 clean_test_state_files.py create mode 100644 tests/functional/shared/destroy_deployments_and_remove_state_file.py create mode 100644 tests/functional/shared/state_files_directory.py diff --git a/clean_test_state_files.py b/clean_test_state_files.py new file mode 100755 index 000000000..c301bce21 --- /dev/null +++ b/clean_test_state_files.py @@ -0,0 +1,15 @@ +#! /usr/bin/env python2 +# -*- coding: utf-8 -*- + +from nixops.util import root_dir +from tests.functional.shared.destroy_deployments_and_remove_state_file import destroy_deployments_and_remove_state_file +from tests.functional.shared.state_files_directory import state_files_directory +import os + +for file in os.listdir(state_files_directory): + if file.endswith(".nixops") or file.endswith(".json"): + file_path = os.path.join(state_files_directory, file) + + print("Destroying {}".format(file_path)) + + destroy_deployments_and_remove_state_file(file_path) diff --git a/tests/functional/shared/destroy_deployments_and_remove_state_file.py b/tests/functional/shared/destroy_deployments_and_remove_state_file.py new file mode 100644 index 000000000..5a5ccccda --- /dev/null +++ b/tests/functional/shared/destroy_deployments_and_remove_state_file.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- + +import nixops.state +import threading +import os + +def destroy_deployments_and_remove_state_file(state_file_path): + state = nixops.state.open(state_file_path) + uuids = state.query_deployments() + threads = [] + for uuid in uuids: + threads.append( + threading.Thread(target=destroy_deployments, args=(state, uuid)) + ) + for thread in threads: + thread.start() + for thread in threads: + thread.join() + uuids_left = state.query_deployments() + state.close() + if not uuids_left: + os.remove(state_file_path) + else: + message = "warning: not all deployments have been destroyed; some resources may still exist!\n" + sys.stderr.write(message) + +def destroy_deployments(state, uuid): + deployment = state.open_deployment(uuid) + deployment.logger.set_autoresponse("y") + try: + deployment.clean_backups(keep=False, keep_days=False) + except Exception: + pass + try: + deployment.destroy_resources() + except Exception: + pass + deployment.delete() + deployment.logger.log("deployment ‘{0}’ destroyed".format(uuid)) diff --git a/tests/functional/shared/state_files_directory.py b/tests/functional/shared/state_files_directory.py new file mode 100644 index 000000000..57a00330a --- /dev/null +++ b/tests/functional/shared/state_files_directory.py @@ -0,0 +1,3 @@ +from nixops.util import root_dir + +state_files_directory = '{}/tests/state_files'.format(root_dir) diff --git a/tests/functional/shared/unique_state_file_path.py b/tests/functional/shared/unique_state_file_path.py index ee568fbb5..38b35e684 100644 --- a/tests/functional/shared/unique_state_file_path.py +++ b/tests/functional/shared/unique_state_file_path.py @@ -1,8 +1,6 @@ -from nixops.util import root_dir +from tests.functional.shared.state_files_directory import state_files_directory def unique_state_file_path(array_of_keys_file_name_depends_on, extension): unique_file_name = '_'.join(array_of_keys_file_name_depends_on) - return '{}/tests/state_files/{}.{}'.format( - root_dir, unique_file_name, extension - ) + return '{}/{}.{}'.format(state_files_directory, unique_file_name, extension) diff --git a/tests/functional/shared/using_state_file.py b/tests/functional/shared/using_state_file.py index 7e50e951e..e4e719222 100644 --- a/tests/functional/shared/using_state_file.py +++ b/tests/functional/shared/using_state_file.py @@ -1,9 +1,6 @@ -# -*- coding: utf-8 -*- - from contextlib import contextmanager import os import sys -import threading from distutils.dir_util import mkpath import nixops.state @@ -26,40 +23,3 @@ def using_state_file(state_file_path): def create_file_parent_dirs_if_not_exists(file_path): mkpath(os.path.dirname(file_path)) - -def destroy_deployments(state, uuid): - deployment = state.open_deployment(uuid) - deployment.logger.set_autoresponse("y") - try: - deployment.clean_backups(keep=False, keep_days=False) - except Exception: - pass - try: - deployment.destroy_resources() - except Exception: - pass - deployment.delete() - deployment.logger.log("deployment ‘{0}’ destroyed".format(uuid)) - - -def destroy_deployments_and_remove_state_file(state_file_path): - state = nixops.state.open(state_file_path) - uuids = state.query_deployments() - threads = [] - for uuid in uuids: - threads.append( - threading.Thread(target=destroy_deployments, args=(state, uuid)) - ) - for thread in threads: - thread.start() - for thread in threads: - thread.join() - uuids_left = state.query_deployments() - state.close() - if not uuids_left: - os.remove(state_file_path) - else: - sys.stderr.write( - "warning: not all deployments have been destroyed; some resources may still exist!\n" - ) - From 59c33cda1a4f1f0aa03da12c0e850de32e5c5b13 Mon Sep 17 00:00:00 2001 From: srghma Date: Sat, 16 Jun 2018 20:09:40 +0300 Subject: [PATCH 100/123] fix: tests.functional.test_vpc:test_deploy_vpc -> works with nixops state file --- tests/functional/shared/using_state_file.py | 1 + tests/functional/test_vpc/{test_vpc.nix => vpc.nix} | 0 2 files changed, 1 insertion(+) rename tests/functional/test_vpc/{test_vpc.nix => vpc.nix} (100%) diff --git a/tests/functional/shared/using_state_file.py b/tests/functional/shared/using_state_file.py index e4e719222..938f69856 100644 --- a/tests/functional/shared/using_state_file.py +++ b/tests/functional/shared/using_state_file.py @@ -4,6 +4,7 @@ from distutils.dir_util import mkpath import nixops.state +from tests.functional.shared.destroy_deployments_and_remove_state_file import destroy_deployments_and_remove_state_file @contextmanager def using_state_file(state_file_path): diff --git a/tests/functional/test_vpc/test_vpc.nix b/tests/functional/test_vpc/vpc.nix similarity index 100% rename from tests/functional/test_vpc/test_vpc.nix rename to tests/functional/test_vpc/vpc.nix From e1b9158f67dc7673e9d84eaf9e3a5b73c21cb31a Mon Sep 17 00:00:00 2001 From: srghma Date: Sat, 16 Jun 2018 21:28:10 +0300 Subject: [PATCH 101/123] fix: tests.functional.test_vpc:test_deploy_vpc with json state -> TypeError expected string or buffer --- nixops/state/state_helper.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/nixops/state/state_helper.py b/nixops/state/state_helper.py index 12bfa225e..f97bfaf26 100644 --- a/nixops/state/state_helper.py +++ b/nixops/state/state_helper.py @@ -25,12 +25,18 @@ def __setitem__(self, key, value): def __getitem__(self, key): value = self._state.get_resource_attr(self.uuid, self.id, key) - if value != nixops.util.undefined: + + if value == nixops.util.undefined: + raise KeyError("couldn't find key {} in the state file".format(key)) + + if isinstance(value, str): try: return json.loads(value) except ValueError: return value - raise KeyError("couldn't find key {} in the state file".format(key)) + + return value + def __delitem__(self, key): self._state.del_resource_attr(self.uuid, self.id, key) From b30dbfecfae5cf71b48184ae0b7b76984f8dcdc2 Mon Sep 17 00:00:00 2001 From: srghma Date: Sun, 17 Jun 2018 12:16:20 +0300 Subject: [PATCH 102/123] refactor: state -> _create_state repeats --- nixops/state/json_file.py | 16 +++------------ nixops/state/sqlite_connector.py | 20 +++---------------- nixops/state/state_helper.py | 15 +++++++++++++- ...stroy_deployments_and_remove_state_file.py | 14 ++++++++----- tests/functional/shared/using_state_file.py | 1 + tests/functional/test_vpc/__init__.py | 5 ++++- tests/functional/test_vpc/helpers.py | 9 ++++----- 7 files changed, 38 insertions(+), 42 deletions(-) diff --git a/nixops/state/json_file.py b/nixops/state/json_file.py index 81d3cc59e..5e091223f 100644 --- a/nixops/state/json_file.py +++ b/nixops/state/json_file.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- import nixops.deployment -from nixops.state.state_helper import _subclasses +from nixops.state.state_helper import _create_resource_state import os import os.path import sys @@ -228,7 +228,7 @@ def get_resources_for(self, deployment): state = self.db.read() state_resources = state["deployments"][deployment.uuid]["resources"] for res_id, res in state_resources.items(): - r = self._create_state(deployment, res["type"], res["name"], res_id) + r = _create_resource_state(deployment, res["type"], res["name"], res_id) resources[res["name"]] = r self.db.set(state) return resources @@ -304,7 +304,7 @@ def create_resource(self, deployment, name, type): "attributes" : {} } self.db.set(state) - r = self._create_state(deployment, type, name, id) + r = _create_resource_state(deployment, type, name, id) return r def delete_resource(self, deployment_uuid, res_id): @@ -354,13 +354,3 @@ def get_all_resource_attrs(self, deployment_uuid, resource_id): state = self.db.read() resource_attrs = state["deployments"][deployment_uuid]["resources"][resource_id]["attributes"] return copy.deepcopy(resource_attrs) - - ### STATE - def _create_state(self, depl, type, name, id): - """Create a resource state object of the desired type.""" - - for cls in _subclasses(nixops.resources.ResourceState): - if type == cls.get_type(): - return cls(depl, name, id) - - raise nixops.deployment.UnknownBackend("unknown resource type ‘{0}’".format(type)) diff --git a/nixops/state/sqlite_connector.py b/nixops/state/sqlite_connector.py index 8c9042d8b..d42e0a5d5 100644 --- a/nixops/state/sqlite_connector.py +++ b/nixops/state/sqlite_connector.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- import nixops.deployment -from nixops.state.state_helper import _subclasses +from nixops.state.state_helper import _create_resource_state import os import os.path import urlparse @@ -257,7 +257,7 @@ def get_resources_for(self, deployment): "select id, name, type from Resources where deployment = ?", (deployment.uuid, )).fetchall() for (id, name, type) in rows: - r = self._create_state(deployment, type, name, id) + r = _create_resource_state(deployment, type, name, id) resources[name] = r return resources @@ -348,7 +348,7 @@ def create_resource(self, deployment, name, type): (deployment.uuid, name, type)) id = result.lastrowid - r = self._create_state(deployment, type, name, id) + r = _create_resource_state(deployment, type, name, id) return r def delete_resource(self, deployment_uuid, res_id): @@ -406,17 +406,3 @@ def get_all_resource_attrs(self, deployment_uuid, resource_id): (resource_id, )).fetchall() res = {row[0]: row[1] for row in rows} return res - - -# ### STATE - - def _create_state(self, depl, type, name, id): - """Create a resource state object of the desired type.""" - - for cls in _subclasses(nixops.resources.ResourceState): - if type == cls.get_type(): - return cls(depl, name, id) - - raise nixops.deployment.UnknownBackend("unknown resource type ‘{!r}’" - .format(type)) - diff --git a/nixops/state/state_helper.py b/nixops/state/state_helper.py index f97bfaf26..c608f7981 100644 --- a/nixops/state/state_helper.py +++ b/nixops/state/state_helper.py @@ -1,3 +1,5 @@ +# -*- coding: utf-8 -*- + import json import collections import nixops.util @@ -7,6 +9,12 @@ def _subclasses(cls): sub = cls.__subclasses__() return [cls] if not sub else [g for s in sub for g in _subclasses(s)] +def _create_resource_state(depl, type, name, id): + """Create a resource state object of the desired type.""" + for cls in _subclasses(nixops.resources.ResourceState): + if type == cls.get_type(): + return cls(depl, name, id) + raise nixops.deployment.UnknownBackend("unknown resource type ‘{}’".format(type)) class StateDict(collections.MutableMapping): """ @@ -21,7 +29,12 @@ def __init__(self, depl, id): self.id = id def __setitem__(self, key, value): - self._state.set_resource_attrs(self.uuid, self.id, {key:value}) + value_ = value + + if not isinstance(value, str): + value_ = json.dumps(value) + + self._state.set_resource_attrs(self.uuid, self.id, { key: value_ }) def __getitem__(self, key): value = self._state.get_resource_attr(self.uuid, self.id, key) diff --git a/tests/functional/shared/destroy_deployments_and_remove_state_file.py b/tests/functional/shared/destroy_deployments_and_remove_state_file.py index 5a5ccccda..e822d450c 100644 --- a/tests/functional/shared/destroy_deployments_and_remove_state_file.py +++ b/tests/functional/shared/destroy_deployments_and_remove_state_file.py @@ -3,6 +3,7 @@ import nixops.state import threading import os +import sys def destroy_deployments_and_remove_state_file(state_file_path): state = nixops.state.open(state_file_path) @@ -27,13 +28,16 @@ def destroy_deployments_and_remove_state_file(state_file_path): def destroy_deployments(state, uuid): deployment = state.open_deployment(uuid) deployment.logger.set_autoresponse("y") + try: deployment.clean_backups(keep=False, keep_days=False) - except Exception: - pass + except Exception as e: + deployment.logger.error("on clean backups for deployment ‘{}’: {}".format(uuid, e)) + try: deployment.destroy_resources() - except Exception: - pass + except Exception as e: + deployment.logger.error("on destroy resources for deployment ‘{}’: {}".format(uuid, e)) + deployment.delete() - deployment.logger.log("deployment ‘{0}’ destroyed".format(uuid)) + deployment.logger.log("deployment ‘{}’ destroyed".format(uuid)) diff --git a/tests/functional/shared/using_state_file.py b/tests/functional/shared/using_state_file.py index 938f69856..5a9f84b06 100644 --- a/tests/functional/shared/using_state_file.py +++ b/tests/functional/shared/using_state_file.py @@ -11,6 +11,7 @@ def using_state_file(state_file_path): create_file_parent_dirs_if_not_exists(state_file_path) if os.path.exists(state_file_path): + # TODO: throw error here if can't delete destroy_deployments_and_remove_state_file(state_file_path) state = nixops.state.open(state_file_path) diff --git a/tests/functional/test_vpc/__init__.py b/tests/functional/test_vpc/__init__.py index a9d24154c..0fc4c4386 100644 --- a/tests/functional/test_vpc/__init__.py +++ b/tests/functional/test_vpc/__init__.py @@ -8,6 +8,9 @@ from tests.functional.shared.using_unique_state_file import using_unique_state_file from tests.functional.shared.create_deployment import create_deployment +from tests.functional.test_vpc.helpers import compose_expressions +from tests.functional.test_vpc.resources import CFG_VPC_MACHINE, CFG_INTERNET_ROUTE, CFG_DNS_SUPPORT, CFG_IPV6, CFG_NAT_GTW, CFG_SUBNET + parent_dir = path.dirname(__file__) base_spec = "{}/vpc.nix".format(parent_dir) @@ -32,7 +35,7 @@ def test_deploy_vpc(state_extension): @parameterized([ 'json', - 'nixops' + # 'nixops' ]) def test_deploy_vpc_machine(state_extension): with using_unique_state_file( diff --git a/tests/functional/test_vpc/helpers.py b/tests/functional/test_vpc/helpers.py index 41043c938..5c99cd763 100644 --- a/tests/functional/test_vpc/helpers.py +++ b/tests/functional/test_vpc/helpers.py @@ -1,17 +1,16 @@ import nixops.util import tempfile -def create_exprs_dir(): - return nixops.util.SelfDeletingDir(tempfile.mkdtemp("nixos-tests")) - def compose_expressions(configurations): exprs_dir = create_exprs_dir() - extra_exprs = list(map(lambda x: generate_config(exprs_dir, x), configurations)) + nix_exprs = list(map(lambda x: generate_config(exprs_dir, x), configurations)) - nix_exprs = [base_spec] + extra_exprs return nix_exprs +def create_exprs_dir(): + return nixops.util.SelfDeletingDir(tempfile.mkdtemp("nixos-tests")) + def generate_config(exprs_dir, config): basename, expr = config expr_path = "{0}/{1}".format(exprs_dir, basename) From 0b6c27cf07af79b24784eddabd90ed2ced2259df Mon Sep 17 00:00:00 2001 From: srghma Date: Sun, 17 Jun 2018 12:22:40 +0300 Subject: [PATCH 103/123] refactor: test, destroy_deployments_and_remove_state_file -> print error --- .../shared/destroy_deployments_and_remove_state_file.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/functional/shared/destroy_deployments_and_remove_state_file.py b/tests/functional/shared/destroy_deployments_and_remove_state_file.py index e822d450c..d91c3a344 100644 --- a/tests/functional/shared/destroy_deployments_and_remove_state_file.py +++ b/tests/functional/shared/destroy_deployments_and_remove_state_file.py @@ -4,6 +4,7 @@ import threading import os import sys +import traceback def destroy_deployments_and_remove_state_file(state_file_path): state = nixops.state.open(state_file_path) @@ -33,11 +34,12 @@ def destroy_deployments(state, uuid): deployment.clean_backups(keep=False, keep_days=False) except Exception as e: deployment.logger.error("on clean backups for deployment ‘{}’: {}".format(uuid, e)) - + traceback.print_exc(file=sys.__stdout__) try: deployment.destroy_resources() except Exception as e: deployment.logger.error("on destroy resources for deployment ‘{}’: {}".format(uuid, e)) + traceback.print_exc(file=sys.__stdout__) deployment.delete() deployment.logger.log("deployment ‘{}’ destroyed".format(uuid)) From 898107cb047bd59b512af524d15bf8daf71ec61e Mon Sep 17 00:00:00 2001 From: srghma Date: Sun, 17 Jun 2018 12:49:31 +0300 Subject: [PATCH 104/123] fix: deployment.clean_backups method -> UnboundLocalError: local variable tbr referenced before assignment --- nixops/deployment.py | 12 +++++++----- .../destroy_deployments_and_remove_state_file.py | 2 +- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/nixops/deployment.py b/nixops/deployment.py index ea27d0dd9..581406887 100644 --- a/nixops/deployment.py +++ b/nixops/deployment.py @@ -739,21 +739,23 @@ def get_backups(self, include=[], exclude=[]): return backups - def clean_backups(self, keep, keep_days, keep_physical = False): + def clean_backups(self, keep = None, keep_days = None, keep_physical = False): _backups = self.get_backups() + backup_ids = [b for b in _backups.keys()] backup_ids.sort() + to_be_removed = backup_ids if keep: - index = len(backup_ids)-keep - tbr = backup_ids[:index] + index = len(backup_ids) - keep + to_be_removed = backup_ids[:index] if keep_days: cutoff = (datetime.now()- timedelta(days=keep_days)).strftime("%Y%m%d%H%M%S") print cutoff - tbr = [bid for bid in backup_ids if bid < cutoff] + to_be_removed = [bid for bid in backup_ids if bid < cutoff] - for backup_id in tbr: + for backup_id in to_be_removed: print 'Removing backup {0}'.format(backup_id) self.remove_backup(backup_id, keep_physical) diff --git a/tests/functional/shared/destroy_deployments_and_remove_state_file.py b/tests/functional/shared/destroy_deployments_and_remove_state_file.py index d91c3a344..bcc157442 100644 --- a/tests/functional/shared/destroy_deployments_and_remove_state_file.py +++ b/tests/functional/shared/destroy_deployments_and_remove_state_file.py @@ -31,7 +31,7 @@ def destroy_deployments(state, uuid): deployment.logger.set_autoresponse("y") try: - deployment.clean_backups(keep=False, keep_days=False) + deployment.clean_backups() except Exception as e: deployment.logger.error("on clean backups for deployment ‘{}’: {}".format(uuid, e)) traceback.print_exc(file=sys.__stdout__) From e554c1313c87954d28c1ea6cbda2cb3f4b155ea1 Mon Sep 17 00:00:00 2001 From: srghma Date: Sun, 17 Jun 2018 12:53:08 +0300 Subject: [PATCH 105/123] fix: tests.functional.test_vpc:test_deploy_vpc -> create_exprs_dir -> dont remove directory with nix expressions on program exit, because in case of error it makes impossible to destroy resources with clean_test_state_files.py util --- tests/functional/test_vpc/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/test_vpc/helpers.py b/tests/functional/test_vpc/helpers.py index 5c99cd763..579615712 100644 --- a/tests/functional/test_vpc/helpers.py +++ b/tests/functional/test_vpc/helpers.py @@ -9,7 +9,7 @@ def compose_expressions(configurations): return nix_exprs def create_exprs_dir(): - return nixops.util.SelfDeletingDir(tempfile.mkdtemp("nixos-tests")) + return tempfile.mkdtemp("nixos-tests") def generate_config(exprs_dir, config): basename, expr = config From db84a8f2f1a73fa32c3e05f5db43b0f06afb7ce5 Mon Sep 17 00:00:00 2001 From: srghma Date: Sun, 17 Jun 2018 13:44:51 +0300 Subject: [PATCH 106/123] fix: state_dict -> An error occurred (InvalidRouteTableId.Malformed) when calling the DeleteRoute operation: Invalid id: ""rtb-d9ca1aa6"" (expecting "rtb-...") --- nixops/state/state_helper.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nixops/state/state_helper.py b/nixops/state/state_helper.py index c608f7981..a4f896775 100644 --- a/nixops/state/state_helper.py +++ b/nixops/state/state_helper.py @@ -31,7 +31,7 @@ def __init__(self, depl, id): def __setitem__(self, key, value): value_ = value - if not isinstance(value, str): + if not isinstance(value, basestring): value_ = json.dumps(value) self._state.set_resource_attrs(self.uuid, self.id, { key: value_ }) @@ -42,7 +42,7 @@ def __getitem__(self, key): if value == nixops.util.undefined: raise KeyError("couldn't find key {} in the state file".format(key)) - if isinstance(value, str): + if isinstance(value, basestring): try: return json.loads(value) except ValueError: From 2e547bd19163019f6a8c7f1fe80a3650688131b7 Mon Sep 17 00:00:00 2001 From: srghma Date: Sun, 17 Jun 2018 13:47:58 +0300 Subject: [PATCH 107/123] fix: use isinstance(XXX, basestring) because isinstance(XXX, str) returns false for unicode strings in python2 --- nixops/deployment.py | 2 +- nixops/diff.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/nixops/deployment.py b/nixops/deployment.py index 581406887..3d529dfe6 100644 --- a/nixops/deployment.py +++ b/nixops/deployment.py @@ -225,7 +225,7 @@ def set_argstr(self, name, value): def unset_arg(self, name): """Unset a persistent argument to the deployment specification.""" - assert isinstance(name, str) + assert isinstance(name, basestring) args = self.args args.pop(name, None) self.args = args diff --git a/nixops/diff.py b/nixops/diff.py index d6b7bb733..0b977674f 100644 --- a/nixops/diff.py +++ b/nixops/diff.py @@ -68,7 +68,7 @@ def topological_sort(self, handlers): The output is a sorted sequence of handlers based on their dependencies. """ - # TODO implement cycle detection + # TODO implement cycle detection parent = {} sequence = [] @@ -116,7 +116,7 @@ def eval_resource_attr_diff(self, key): def get_resource_definition(self, key): def retrieve_def(d): - if isinstance(d, str) and d.startswith("res-"): + if isinstance(d, basestring) and d.startswith("res-"): name = d[4:].split(".")[0] res_type = d.split(".")[1] k = d.split(".")[2] if len(d.split(".")) > 2 else key From 2f930cdad009577098fcb56f12ba68d546222e68 Mon Sep 17 00:00:00 2001 From: srghma Date: Sun, 17 Jun 2018 14:57:56 +0300 Subject: [PATCH 108/123] refactor: tests.functional.test_vpc --- tests/functional/test_vpc/__init__.py | 59 ++++++---- .../test_vpc/enable_dns_support.nix | 3 + tests/functional/test_vpc/helpers.py | 19 --- tests/functional/test_vpc/igw_route.nix | 35 ++++++ tests/functional/test_vpc/ipv6.nix | 3 + tests/functional/test_vpc/nat_gtw.nix | 15 +++ tests/functional/test_vpc/network.nix | 24 ++++ tests/functional/test_vpc/resources.py | 109 ------------------ tests/functional/test_vpc/subnet.nix | 14 +++ 9 files changed, 130 insertions(+), 151 deletions(-) create mode 100644 tests/functional/test_vpc/enable_dns_support.nix delete mode 100644 tests/functional/test_vpc/helpers.py create mode 100644 tests/functional/test_vpc/igw_route.nix create mode 100644 tests/functional/test_vpc/ipv6.nix create mode 100644 tests/functional/test_vpc/nat_gtw.nix create mode 100644 tests/functional/test_vpc/network.nix delete mode 100644 tests/functional/test_vpc/resources.py create mode 100644 tests/functional/test_vpc/subnet.nix diff --git a/tests/functional/test_vpc/__init__.py b/tests/functional/test_vpc/__init__.py index 0fc4c4386..d243355fc 100644 --- a/tests/functional/test_vpc/__init__.py +++ b/tests/functional/test_vpc/__init__.py @@ -8,13 +8,8 @@ from tests.functional.shared.using_unique_state_file import using_unique_state_file from tests.functional.shared.create_deployment import create_deployment -from tests.functional.test_vpc.helpers import compose_expressions -from tests.functional.test_vpc.resources import CFG_VPC_MACHINE, CFG_INTERNET_ROUTE, CFG_DNS_SUPPORT, CFG_IPV6, CFG_NAT_GTW, CFG_SUBNET - parent_dir = path.dirname(__file__) -base_spec = "{}/vpc.nix".format(parent_dir) - @parameterized([ 'json', 'nixops' @@ -24,7 +19,9 @@ def test_deploy_vpc(state_extension): [test_deploy_vpc.__name__], state_extension ) as state: - deployment = create_deployment(state, [base_spec]) + deployment = create_deployment(state, [ + "{}/vpc.nix".format(parent_dir) + ]) deployment.deploy() @@ -35,17 +32,22 @@ def test_deploy_vpc(state_extension): @parameterized([ 'json', - # 'nixops' + 'nixops' ]) def test_deploy_vpc_machine(state_extension): with using_unique_state_file( [test_deploy_vpc_machine.__name__], state_extension ) as state: - nix_expressions = compose_expressions([CFG_SUBNET, CFG_INTERNET_ROUTE, CFG_VPC_MACHINE]) - nix_expressions_ = [base_spec] + nix_expressions - deployment = create_deployment(state, nix_expressions_) + nix_expressions = [ + "{}/vpc.nix".format(parent_dir), + "{}/subnet.nix".format(parent_dir), + "{}/igw_route.nix".format(parent_dir), + "{}/network.nix".format(parent_dir), + ] + + deployment = create_deployment(state, nix_expressions) deployment.deploy(plan_only=True) deployment.deploy() @@ -59,10 +61,13 @@ def test_enable_dns_support(state_extension): [test_enable_dns_support.__name__], state_extension ) as state: - nix_expressions = compose_expressions([CFG_DNS_SUPPORT]) - nix_expressions_ = [base_spec] + nix_expressions - deployment = create_deployment(state, nix_expressions_) + nix_expressions = [ + "{}/vpc.nix".format(parent_dir), + "{}/enable_dns_support.nix".format(parent_dir), + ] + + deployment = create_deployment(state, nix_expressions) deployment.deploy(plan_only=True) deployment.deploy() @@ -71,15 +76,17 @@ def test_enable_dns_support(state_extension): 'json', 'nixops' ]) -def test_enable_ipv6(): +def test_enable_ipv6(state_extension): with using_unique_state_file( [test_enable_ipv6.__name__], state_extension ) as state: - nix_expressions = compose_expressions([CFG_IPV6]) - nix_expressions_ = [base_spec] + nix_expressions + nix_expressions = [ + "{}/vpc.nix".format(parent_dir), + "{}/ipv6.nix".format(parent_dir), + ] - deployment = create_deployment(state, nix_expressions_) + deployment = create_deployment(state, nix_expressions) deployment.deploy(plan_only=True) deployment.deploy() @@ -101,10 +108,13 @@ def test_deploy_subnets(state_extension): ) as state: # FIXME might need to factor out resources into separate test # classes depending on the number of tests needed. - nix_expressions = compose_expressions([CFG_SUBNET]) - nix_expressions_ = [base_spec] + nix_expressions - deployment = create_deployment(state, nix_expressions_) + nix_expressions = [ + "{}/vpc.nix".format(parent_dir), + "{}/subnet.nix".format(parent_dir), + ] + + deployment = create_deployment(state, nix_expressions) deployment.deploy(plan_only=True) deployment.deploy() @@ -121,10 +131,13 @@ def test_deploy_nat_gtw(state_extension): [test_deploy_nat_gtw.__name__], state_extension ) as state: - nix_expressions = compose_expressions([CFG_SUBNET, CFG_NAT_GTW]) - nix_expressions_ = [base_spec] + nix_expressions + nix_expressions = [ + "{}/vpc.nix".format(parent_dir), + "{}/subnet.nix".format(parent_dir), + "{}/nat_gtw.nix".format(parent_dir), + ] - deployment = create_deployment(state, nix_expressions_) + deployment = create_deployment(state, nix_expressions) deployment.deploy(plan_only=True) deployment.deploy() diff --git a/tests/functional/test_vpc/enable_dns_support.nix b/tests/functional/test_vpc/enable_dns_support.nix new file mode 100644 index 000000000..0a359f9e6 --- /dev/null +++ b/tests/functional/test_vpc/enable_dns_support.nix @@ -0,0 +1,3 @@ +{ + resources.vpc.vpc-test.enableDnsSupport = true; +} diff --git a/tests/functional/test_vpc/helpers.py b/tests/functional/test_vpc/helpers.py deleted file mode 100644 index 579615712..000000000 --- a/tests/functional/test_vpc/helpers.py +++ /dev/null @@ -1,19 +0,0 @@ -import nixops.util -import tempfile - -def compose_expressions(configurations): - exprs_dir = create_exprs_dir() - - nix_exprs = list(map(lambda x: generate_config(exprs_dir, x), configurations)) - - return nix_exprs - -def create_exprs_dir(): - return tempfile.mkdtemp("nixos-tests") - -def generate_config(exprs_dir, config): - basename, expr = config - expr_path = "{0}/{1}".format(exprs_dir, basename) - with open(expr_path, "w") as cfg: - cfg.write(expr) - return expr_path diff --git a/tests/functional/test_vpc/igw_route.nix b/tests/functional/test_vpc/igw_route.nix new file mode 100644 index 000000000..8749740fa --- /dev/null +++ b/tests/functional/test_vpc/igw_route.nix @@ -0,0 +1,35 @@ +let + region = "us-east-1"; +in +{ + resources = { + + vpcRouteTables.route-table = + { resources, ... }: + { inherit region; vpcId = resources.vpc.vpc-test; }; + + vpcRouteTableAssociations.association-test = + { resources, ... }: + { + inherit region; + subnetId = resources.vpcSubnets.subnet-test; + routeTableId = resources.vpcRouteTables.route-table; + }; + + vpcRoutes.igw-route = + { resources, ... }: + { + inherit region; + routeTableId = resources.vpcRouteTables.route-table; + destinationCidrBlock = "0.0.0.0/0"; + gatewayId = resources.vpcInternetGateways.igw-test; + }; + + vpcInternetGateways.igw-test = + { resources, ... }: + { + inherit region; + vpcId = resources.vpc.vpc-test; + }; + }; +} diff --git a/tests/functional/test_vpc/ipv6.nix b/tests/functional/test_vpc/ipv6.nix new file mode 100644 index 000000000..9a6d451a2 --- /dev/null +++ b/tests/functional/test_vpc/ipv6.nix @@ -0,0 +1,3 @@ +{ + resources.vpc.vpc-test.amazonProvidedIpv6CidrBlock = true; +} diff --git a/tests/functional/test_vpc/nat_gtw.nix b/tests/functional/test_vpc/nat_gtw.nix new file mode 100644 index 000000000..a0c33f9a5 --- /dev/null +++ b/tests/functional/test_vpc/nat_gtw.nix @@ -0,0 +1,15 @@ +{ + resources.elasticIPs.nat-eip = + { + region = "us-east-1"; + vpc = true; + }; + + resources.vpcNatGateways.nat = + { resources, ... }: + { + region = "us-east-1"; + allocationId = resources.elasticIPs.nat-eip; + subnetId = resources.vpcSubnets.subnet-test; + }; + } diff --git a/tests/functional/test_vpc/network.nix b/tests/functional/test_vpc/network.nix new file mode 100644 index 000000000..5e60e05f8 --- /dev/null +++ b/tests/functional/test_vpc/network.nix @@ -0,0 +1,24 @@ +{ + machine = + {config, resources, pkgs, lib, ...}: + { + deployment.targetEnv = "ec2"; + deployment.hasFastConnection = true; + deployment.ec2.associatePublicIpAddress = true; + deployment.ec2.region = "us-east-1"; + deployment.ec2.instanceType = "c3.large"; + deployment.ec2.subnetId = resources.vpcSubnets.subnet-test; + deployment.ec2.keyPair = resources.ec2KeyPairs.keypair.name; + deployment.ec2.securityGroups = []; + deployment.ec2.securityGroupIds = [ resources.ec2SecurityGroups.public-ssh.name ]; + }; + + resources.ec2KeyPairs.keypair = { region = "us-east-1"; }; + resources.ec2SecurityGroups.public-ssh = + { resources, ... }: + { + region = "us-east-1"; + vpcId = resources.vpc.vpc-test; + rules = [{ toPort = 22; fromPort = 22; sourceIp = "0.0.0.0/0"; }]; + }; +} diff --git a/tests/functional/test_vpc/resources.py b/tests/functional/test_vpc/resources.py deleted file mode 100644 index 8e7dd8495..000000000 --- a/tests/functional/test_vpc/resources.py +++ /dev/null @@ -1,109 +0,0 @@ -from nixops.nix_expr import py2nix - -CFG_VPC_MACHINE = ("network.nix", """ - { - machine = - {config, resources, pkgs, lib, ...}: - { - deployment.targetEnv = "ec2"; - deployment.hasFastConnection = true; - deployment.ec2.associatePublicIpAddress = true; - deployment.ec2.region = "us-east-1"; - deployment.ec2.instanceType = "c3.large"; - deployment.ec2.subnetId = resources.vpcSubnets.subnet-test; - deployment.ec2.keyPair = resources.ec2KeyPairs.keypair.name; - deployment.ec2.securityGroups = []; - deployment.ec2.securityGroupIds = [ resources.ec2SecurityGroups.public-ssh.name ]; - }; - - resources.ec2KeyPairs.keypair = { region = "us-east-1"; }; - resources.ec2SecurityGroups.public-ssh = - { resources, ... }: - { - region = "us-east-1"; - vpcId = resources.vpc.vpc-test; - rules = [{ toPort = 22; fromPort = 22; sourceIp = "0.0.0.0/0"; }]; - }; - } - """) - -CFG_INTERNET_ROUTE = ("igw_route.nix", """ - let - region = "us-east-1"; - in - { - resources = { - - vpcRouteTables.route-table = - { resources, ... }: - { inherit region; vpcId = resources.vpc.vpc-test; }; - - vpcRouteTableAssociations.association-test = - { resources, ... }: - { - inherit region; - subnetId = resources.vpcSubnets.subnet-test; - routeTableId = resources.vpcRouteTables.route-table; - }; - - vpcRoutes.igw-route = - { resources, ... }: - { - inherit region; - routeTableId = resources.vpcRouteTables.route-table; - destinationCidrBlock = "0.0.0.0/0"; - gatewayId = resources.vpcInternetGateways.igw-test; - }; - - vpcInternetGateways.igw-test = - { resources, ... }: - { - inherit region; - vpcId = resources.vpc.vpc-test; - }; - }; - } - """) - -CFG_DNS_SUPPORT = ("enable_dns_support.nix", py2nix({ - ('resources', 'vpc', 'vpc-test', 'enableDnsSupport'): True -})) - -CFG_IPV6 = ("ipv6.nix", py2nix({ - ('resources', 'vpc', 'vpc-test', 'amazonProvidedIpv6CidrBlock'): True -})) - -CFG_NAT_GTW = ("nat_gtw.nix", """ - { - resources.elasticIPs.nat-eip = - { - region = "us-east-1"; - vpc = true; - }; - - resources.vpcNatGateways.nat = - { resources, ... }: - { - region = "us-east-1"; - allocationId = resources.elasticIPs.nat-eip; - subnetId = resources.vpcSubnets.subnet-test; - }; - } - """) - -CFG_SUBNET = ("subnet.nix", """ - { - resources.vpcSubnets.subnet-test = - { resources, ... }: - { - region = "us-east-1"; - zone = "us-east-1a"; - vpcId = resources.vpc.vpc-test; - cidrBlock = "10.0.0.0/19"; - mapPublicIpOnLaunch = true; - tags = { - Source = "NixOps Tests"; - }; - }; - } - """) diff --git a/tests/functional/test_vpc/subnet.nix b/tests/functional/test_vpc/subnet.nix new file mode 100644 index 000000000..f7e6243eb --- /dev/null +++ b/tests/functional/test_vpc/subnet.nix @@ -0,0 +1,14 @@ +{ + resources.vpcSubnets.subnet-test = + { resources, ... }: + { + region = "us-east-1"; + zone = "us-east-1a"; + vpcId = resources.vpc.vpc-test; + cidrBlock = "10.0.0.0/19"; + mapPublicIpOnLaunch = true; + tags = { + Source = "NixOps Tests"; + }; + }; +} From 3a778db23be3e7b73cdea2f849846690f67fd50e Mon Sep 17 00:00:00 2001 From: srghma Date: Sun, 17 Jun 2018 18:08:05 +0300 Subject: [PATCH 109/123] refactor: destroy_deployments_and_remove_state_file -> change message to show that its not fatal --- .../shared/destroy_deployments_and_remove_state_file.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/functional/shared/destroy_deployments_and_remove_state_file.py b/tests/functional/shared/destroy_deployments_and_remove_state_file.py index bcc157442..de3134970 100644 --- a/tests/functional/shared/destroy_deployments_and_remove_state_file.py +++ b/tests/functional/shared/destroy_deployments_and_remove_state_file.py @@ -33,12 +33,12 @@ def destroy_deployments(state, uuid): try: deployment.clean_backups() except Exception as e: - deployment.logger.error("on clean backups for deployment ‘{}’: {}".format(uuid, e)) + deployment.logger.warn("non-fatal error on clean backups for deployment ‘{}’: {}".format(uuid, e)) traceback.print_exc(file=sys.__stdout__) try: deployment.destroy_resources() except Exception as e: - deployment.logger.error("on destroy resources for deployment ‘{}’: {}".format(uuid, e)) + deployment.logger.warn("non-fatal error on destroy resources for deployment ‘{}’: {}".format(uuid, e)) traceback.print_exc(file=sys.__stdout__) deployment.delete() From 1411a5ad3d0d8f38197bf36bc550c320260686c8 Mon Sep 17 00:00:00 2001 From: srghma Date: Sun, 17 Jun 2018 18:08:17 +0300 Subject: [PATCH 110/123] fix: typo in message --- nixops/backends/ec2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nixops/backends/ec2.py b/nixops/backends/ec2.py index d8f0096c2..e9e56b6f2 100644 --- a/nixops/backends/ec2.py +++ b/nixops/backends/ec2.py @@ -1308,7 +1308,7 @@ def destroy(self, wipe=False): def stop(self): if not self._booted_from_ebs(): - self.warn("cannot stop non-EBS-backed instance") + self.warn("cannot stop non-EBS-backend instance") return if not self.depl.logger.confirm("are you sure you want to stop machine '{}'".format(self.name)): From c91d5a547ecebf6bdae48cd3fc16fa02e248c14b Mon Sep 17 00:00:00 2001 From: srghma Date: Sun, 17 Jun 2018 20:45:41 +0300 Subject: [PATCH 111/123] fix: tests.functional.test_stopping_stops -> cannot delete this deployment because it still has resources --- nixops/backends/ec2.py | 6 ++++++ nixops/ec2_utils.py | 11 +++++++++++ 2 files changed, 17 insertions(+) diff --git a/nixops/backends/ec2.py b/nixops/backends/ec2.py index e9e56b6f2..41cc0f898 100644 --- a/nixops/backends/ec2.py +++ b/nixops/backends/ec2.py @@ -1275,7 +1275,10 @@ def destroy(self, wipe=False): # in create() due to it being interrupted after the instance # was created but before it registered the ID in the database. self.connect() + self.connect_boto3() + instance = None + if self.vm_id: instance = self._get_instance(allow_missing=True) else: @@ -1293,6 +1296,9 @@ def destroy(self, wipe=False): time.sleep(3) instance = self._get_instance(update=True) + # this will prevent `cannot delete ec2-security-group because it still has resources` error + nixops.ec2_utils.wait_for_network_interfaces_deattached(self._conn_boto3, self.vm_id) + self.log_end("") nixops.known_hosts.update(self.public_ipv4, None, self.public_host_key) diff --git a/nixops/ec2_utils.py b/nixops/ec2_utils.py index acb870089..cf8c09582 100644 --- a/nixops/ec2_utils.py +++ b/nixops/ec2_utils.py @@ -194,3 +194,14 @@ def id_to_security_group_name(conn, sg_id, vpc_id): name = sg.name return name raise Exception("could not resolve security group id '{0}' in VPC '{1}'".format(sg_id, vpc_id)) + + +def wait_for_network_interfaces_deattached(connection_boto3, instance_id, max_wait_time_seconds=3): + filters = [{ 'Name': 'attachment.instance-id', 'Values': [ instance_id ] }] + + def check_any(): + response = connection_boto3.describe_network_interfaces(Filters=filters) + is_any_interfaces_left = len(response['NetworkInterfaces']) != 0 + return is_any_interfaces_left + + return nixops.util.check_wait(check_any, initial=1, max_tries=max_wait_time_seconds, exception=False) From da9b31a6bb3ce7bbc0eb42966ac81e226bc35dfd Mon Sep 17 00:00:00 2001 From: srghma Date: Sun, 17 Jun 2018 21:09:33 +0300 Subject: [PATCH 112/123] refactor: destroy_deployments_and_remove_state_file -> comment because it shows error for vbox and libvirt stores ("LibvirtdState" object has no attribute "get_backups") --- .../shared/destroy_deployments_and_remove_state_file.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/functional/shared/destroy_deployments_and_remove_state_file.py b/tests/functional/shared/destroy_deployments_and_remove_state_file.py index de3134970..ea7cdb096 100644 --- a/tests/functional/shared/destroy_deployments_and_remove_state_file.py +++ b/tests/functional/shared/destroy_deployments_and_remove_state_file.py @@ -34,12 +34,12 @@ def destroy_deployments(state, uuid): deployment.clean_backups() except Exception as e: deployment.logger.warn("non-fatal error on clean backups for deployment ‘{}’: {}".format(uuid, e)) - traceback.print_exc(file=sys.__stdout__) + # traceback.print_exc(file=sys.__stdout__) try: deployment.destroy_resources() except Exception as e: deployment.logger.warn("non-fatal error on destroy resources for deployment ‘{}’: {}".format(uuid, e)) - traceback.print_exc(file=sys.__stdout__) + # traceback.print_exc(file=sys.__stdout__) deployment.delete() deployment.logger.log("deployment ‘{}’ destroyed".format(uuid)) From c21aa050c0b240760656fb77b1c0b6dde3749ce3 Mon Sep 17 00:00:00 2001 From: srghma Date: Sun, 17 Jun 2018 21:56:47 +0300 Subject: [PATCH 113/123] refactor: test_ec2_rds_dbinstance -> add documentation and disable test for legacy rdsDbSecurityGroups --- nixops/resources/ec2_common.py | 1 + .../test_ec2_rds_dbinstance/__init__.py | 21 +++++++++++++------ 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/nixops/resources/ec2_common.py b/nixops/resources/ec2_common.py index e3d1eacd5..f3fd37706 100644 --- a/nixops/resources/ec2_common.py +++ b/nixops/resources/ec2_common.py @@ -44,6 +44,7 @@ def updater(tags): self.update_tags_using(updater, user_tags=user_tags, check=check) + # FIXME: it caches client, but doesnt use service as a key? def get_client(self, service="ec2"): ''' Generic method to get a cached AWS client or create it. diff --git a/tests/functional/test_ec2_rds_dbinstance/__init__.py b/tests/functional/test_ec2_rds_dbinstance/__init__.py index 92af900ca..360329e2d 100644 --- a/tests/functional/test_ec2_rds_dbinstance/__init__.py +++ b/tests/functional/test_ec2_rds_dbinstance/__init__.py @@ -22,12 +22,21 @@ '{}/ec2-rds-dbinstance.nix'.format(parent_dir), ] ), - ( - 'sg', - [ - '{}/ec2-rds-dbinstance-with-sg.nix'.format(parent_dir), - ] - ) + # This test with database security group can only be run on aws account, + # that supports EC2-Classic platform. + # (https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/USER_VPC.FindDefaultVPC.html) + # These account are legacy and are not created after 2013. + + # If your account doesn't support EC2-Classic, you will get an error: + # `VPC DB Security Groups cannot be modified with this API version. + # Please use an API version between 2012-01-15 and 2012-10-31 to modify this group.` + # TODO: remove it? + # ( + # 'sg', + # [ + # '{}/ec2-rds-dbinstance-with-sg.nix'.format(parent_dir), + # ] + # ) ], )) def test_ec2_rds_dbinstance(state_extension, nix_expressions_tuple): From fd8109231d2bf7f7133992c21ffb6bac9c78769a Mon Sep 17 00:00:00 2001 From: srghma Date: Wed, 20 Jun 2018 23:05:43 +0300 Subject: [PATCH 114/123] fix: azure -> AzureState object has no attribute user --- nixops/backends/azure_vm.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/nixops/backends/azure_vm.py b/nixops/backends/azure_vm.py index 349c3a68b..e56fee9f2 100644 --- a/nixops/backends/azure_vm.py +++ b/nixops/backends/azure_vm.py @@ -972,7 +972,7 @@ def start(self): def stop(self): if self.vm_id: - #FIXME: there's also "stopped deallocated" version of this. how to integrate? + # FIXME: there's also "stopped deallocated" version of this. how to integrate? self.log("stopping Azure machine... ") self.state = self.STOPPING self.cmc().virtual_machines.power_off(self.resource_group, self.machine_name) @@ -1169,8 +1169,7 @@ def get_backups(self): def _check(self, res): - if(self.subscription_id is None or self.authority_url is None or - self.user is None or self.password is None): + if(self.subscription_id is None or self.authority_url is None): res.exists = False res.is_up = False self.state = self.MISSING; From 7c152552081a6f0fcdbc8f3faa9e2037b1f85970 Mon Sep 17 00:00:00 2001 From: srghma Date: Wed, 20 Jun 2018 23:09:06 +0300 Subject: [PATCH 115/123] fix: comment --- tests/functional/test_ec2_rds_dbinstance/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/test_ec2_rds_dbinstance/__init__.py b/tests/functional/test_ec2_rds_dbinstance/__init__.py index 360329e2d..392731136 100644 --- a/tests/functional/test_ec2_rds_dbinstance/__init__.py +++ b/tests/functional/test_ec2_rds_dbinstance/__init__.py @@ -25,7 +25,7 @@ # This test with database security group can only be run on aws account, # that supports EC2-Classic platform. # (https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/USER_VPC.FindDefaultVPC.html) - # These account are legacy and are not created after 2013. + # These accounts are legacy and are not created after 2013. # If your account doesn't support EC2-Classic, you will get an error: # `VPC DB Security Groups cannot be modified with this API version. From d912e7d9fbb393dc95633faae2a8bb4fba49bbc7 Mon Sep 17 00:00:00 2001 From: Sergei Khoma Date: Sat, 10 Nov 2018 19:27:38 +0200 Subject: [PATCH 116/123] refactor: coretemp review -> change "echo -n" to "printf" --- nix/hetzner-bootstrap.nix | 2 +- tests/functional/test_ec2_backups/helpers.py | 4 ++-- tests/hetzner-backend/repository.nix | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/nix/hetzner-bootstrap.nix b/nix/hetzner-bootstrap.nix index 035e6f9a7..7c0537687 100644 --- a/nix/hetzner-bootstrap.nix +++ b/nix/hetzner-bootstrap.nix @@ -104,7 +104,7 @@ in stdenv.mkDerivation { echo 'scriptheadsize="$(head -n ''${lnum%%:*} "'"$installer"'" | wc -c)"' echo 'scriptsize="$(${pkgsNative.coreutils}/bin/stat -c %s "'"$installer"'")"' echo 'tarsize="$(($scriptsize - $scriptheadsize))"' - echo 'echo -n "$tarsize:"' + echo 'printf "$tarsize:"' echo 'tail -n +$((''${lnum%%:*} + 1)) "'"$installer"'"' # As before, don't quote here! echo '${pkgsNative.gnutar}/bin/tar c -C /' $stripped_full_storepaths diff --git a/tests/functional/test_ec2_backups/helpers.py b/tests/functional/test_ec2_backups/helpers.py index 9880049dc..30f593673 100644 --- a/tests/functional/test_ec2_backups/helpers.py +++ b/tests/functional/test_ec2_backups/helpers.py @@ -4,7 +4,7 @@ def backup_and_restore_path(deployment, path=""): deployment.deploy() - deployment_run_command(deployment, "echo -n important-data > {}/back-me-up".format(path)) + deployment_run_command(deployment, "printf 'important-data' > {}/back-me-up".format(path)) backup_id = deployment.backup() backups = deployment.get_backups() while backups[backup_id]['status'] == "running": @@ -12,4 +12,4 @@ def backup_and_restore_path(deployment, path=""): backups = deployment.get_backups() deployment_run_command(deployment, "rm {}/back-me-up".format(path)) deployment.restore(backup_id=backup_id) - deployment_run_command(deployment, "echo -n important-data | diff {}/back-me-up -".format(path)) + deployment_run_command(deployment, "printf 'important-data' | diff {}/back-me-up -".format(path)) diff --git a/tests/hetzner-backend/repository.nix b/tests/hetzner-backend/repository.nix index 536f8b804..e3bd784c1 100644 --- a/tests/hetzner-backend/repository.nix +++ b/tests/hetzner-backend/repository.nix @@ -141,7 +141,7 @@ let RELEASE # Create APT repository - echo -n "Creating APT repository..." >&2 + printf "Creating APT repository..." >&2 for debfile in $toInclude ${keyringPackage} ${toString extraPackages}; do REPREPRO_BASE_DIR="$out" ${reprepro}/bin/reprepro includedeb \ "${debianCodename}" "$debfile" > /dev/null From b65546f9eafe3f53f62641a38f2cb82231b5e53a Mon Sep 17 00:00:00 2001 From: Sergei Khoma Date: Sat, 10 Nov 2018 19:31:03 +0200 Subject: [PATCH 117/123] refactor: coretemp review -> fix typo -> revert "backend" to "backed" --- nixops/backends/ec2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nixops/backends/ec2.py b/nixops/backends/ec2.py index 2049639e0..0e49cb84e 100644 --- a/nixops/backends/ec2.py +++ b/nixops/backends/ec2.py @@ -1390,7 +1390,7 @@ def destroy(self, wipe=False): def stop(self): if not self._booted_from_ebs(): - self.warn("cannot stop non-EBS-backend instance") + self.warn("cannot stop non-EBS-backed instance") return if not self.depl.logger.confirm("are you sure you want to stop machine '{}'".format(self.name)): From 2ceecefa7da701e5f8e938b804a73bbd4ea4908c Mon Sep 17 00:00:00 2001 From: Sergei Khoma Date: Sat, 10 Nov 2018 19:31:45 +0200 Subject: [PATCH 118/123] refactor: coretemp review -> fix typo -> "of" to "for" --- nixops/state/json_file.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nixops/state/json_file.py b/nixops/state/json_file.py index 5e091223f..07fcf7e1e 100644 --- a/nixops/state/json_file.py +++ b/nixops/state/json_file.py @@ -15,7 +15,7 @@ class TransactionalJsonFile: """ - Transactional access to a JSON file, with support + Transactional access for a JSON file, with support of nested transactions. This is made possible by keeping track of the transaction nest level. From b63914b3bd0b60dcf69de8b33b885599e464c815 Mon Sep 17 00:00:00 2001 From: Sergei Khoma Date: Sat, 10 Nov 2018 20:02:51 +0200 Subject: [PATCH 119/123] refactor: coretemp review -> json_file, _find_deployment method --- nixops/state/json_file.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/nixops/state/json_file.py b/nixops/state/json_file.py index 07fcf7e1e..baa974769 100644 --- a/nixops/state/json_file.py +++ b/nixops/state/json_file.py @@ -160,14 +160,12 @@ def _find_deployment(self, uuid=None): if not all_deployments: return None - if not uuid: - found = filter(lambda(id): all_deployments[id]["attributes"].get("name"), all_deployments) + if uuid: + found = filter(lambda(id): id == uuid, all_deployments) or \ + filter(lambda(id): all_deployments[id]["attributes"].get("name") == uuid, all_deployments) or \ + filter(lambda(id): id.startswith(uuid), all_deployments) else: - found = filter(lambda(id): id == uuid, all_deployments) - if not found: - found = filter(lambda(id): all_deployments[id]["attributes"].get("name") == uuid, all_deployments) - if not found: - found = filter(lambda(id): id.startswith(uuid), all_deployments) + found = filter(lambda(id): all_deployments[id]["attributes"].get("name"), all_deployments) if not found: return None From b5ee9152401848c7cf050891d6aaa8c88840ccb8 Mon Sep 17 00:00:00 2001 From: Sergei Khoma Date: Sat, 10 Nov 2018 20:35:55 +0200 Subject: [PATCH 120/123] refactor: coretemp review -> fix typo -> "michine" to "machine" --- tests/functional/shared/deployment_run_command.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/functional/shared/deployment_run_command.py b/tests/functional/shared/deployment_run_command.py index 728711a09..e8daa6416 100644 --- a/tests/functional/shared/deployment_run_command.py +++ b/tests/functional/shared/deployment_run_command.py @@ -1,8 +1,8 @@ from nixops.util import ansi_highlight -def deployment_run_command(deployment, command, michine_index=0): +def deployment_run_command(deployment, command, machine_index=0): deployment.evaluate() - machine = deployment.machines.values()[michine_index] + machine = deployment.machines.values()[machine_index] debug_message = ansi_highlight('tests> ') + command import sys; print >> sys.__stdout__, debug_message From c4eecfb38d6a0d1c4889f22bf8edf3c7962e36bf Mon Sep 17 00:00:00 2001 From: Sergei Khoma Date: Sun, 18 Nov 2018 16:44:41 +0200 Subject: [PATCH 121/123] fix: build errors --- nixops/diff.py | 4 ++-- nixops/state/__init__.py | 5 +++-- setup.cfg | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/nixops/diff.py b/nixops/diff.py index a0bcd17b7..be56f8b6e 100644 --- a/nixops/diff.py +++ b/nixops/diff.py @@ -5,7 +5,7 @@ from typing import Any, Callable, Optional, List, Dict, Union, AnyStr import nixops.util from nixops.logger import MachineLogger -from nixops.state import StateDict +from nixops.state.state_helper import StateDict class Diff(object): """ @@ -87,7 +87,7 @@ def topological_sort(self, handlers): dependencies. """ - # TODO implement cycle detection + # TODO implement cycle detection parent = {} # type: Dict[Handler, Optional[Handler]] sequence = [] # type: List[Handler] diff --git a/nixops/state/__init__.py b/nixops/state/__init__.py index 10a52c27d..a56e83fba 100644 --- a/nixops/state/__init__.py +++ b/nixops/state/__init__.py @@ -1,8 +1,9 @@ import os import urlparse import sys -import json_file -import sqlite_connector + +import nixops.state.json_file +import nixops.state.sqlite_connector class WrongStateSchemeException(Exception): pass diff --git a/setup.cfg b/setup.cfg index b8b71154f..ef6fc1fa2 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [mypy] -python_version = 2.7 +python_version = 2.7 no_implicit_optional = True strict_optional = True From e0a34e810985f2ca05a23185c85cf88aade9fe04 Mon Sep 17 00:00:00 2001 From: Sergei Khoma Date: Sun, 18 Nov 2018 16:51:49 +0200 Subject: [PATCH 122/123] feat: mypy -> add mypy-parametrized --- setup.cfg | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setup.cfg b/setup.cfg index ef6fc1fa2..3dc1d9ba1 100644 --- a/setup.cfg +++ b/setup.cfg @@ -35,3 +35,6 @@ ignore_missing_imports = True [mypy-libvirt.*] ignore_missing_imports = True + +[mypy-parametrized.*] +ignore_missing_imports = True From 16e0b6c91bc2c6aa3b67f2e054956f538ea702a6 Mon Sep 17 00:00:00 2001 From: Sergei Khoma Date: Sun, 18 Nov 2018 17:17:30 +0200 Subject: [PATCH 123/123] fix: coretemp review -> fix typo -> change "of -> for" and revert prev wrong change --- for and revert prev wrong change | 1 + nixops/state/json_file.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 for and revert prev wrong change diff --git a/for and revert prev wrong change b/for and revert prev wrong change new file mode 100644 index 000000000..8e5c720d8 --- /dev/null +++ b/for and revert prev wrong change @@ -0,0 +1 @@ +fix: coretemp review -> fix typo -> change of - diff --git a/nixops/state/json_file.py b/nixops/state/json_file.py index baa974769..db9fb8870 100644 --- a/nixops/state/json_file.py +++ b/nixops/state/json_file.py @@ -15,8 +15,8 @@ class TransactionalJsonFile: """ - Transactional access for a JSON file, with support - of nested transactions. + Transactional access to a JSON file, with support + for nested transactions. This is made possible by keeping track of the transaction nest level. If a transaction is started, the current JSON file is flocked() and read into memory.