upgrade usagedb-v1 to v2 (with client-version table)

src/wormhole_mailbox_server/database.py

@@ -1,5 +1,5 @@
 from __future__ import unicode_literals
-import os
+import os, shutil
 import sqlite3
 import tempfile
 from pkg_resources import resource_string
@@ -13,10 +13,11 @@ def get_schema(name, version):
                                    "db-schemas/%s-v%d.sql" % (name, version))
     return schema_bytes.decode("utf-8")
 
-## def get_upgrader(new_version):
-##     schema_bytes = resource_string("wormhole_transit_relay",
-##                                    "db-schemas/upgrade-to-v%d.sql" % new_version)
-##     return schema_bytes.decode("utf-8")
+def get_upgrader(name, new_version):
+    schema_bytes = resource_string("wormhole_mailbox_server",
+                                   "db-schemas/upgrade-%s-to-v%d.sql" %
+                                   (name, new_version))
+    return schema_bytes.decode("utf-8")
 
 CHANNELDB_TARGET_VERSION = 1
 USAGEDB_TARGET_VERSION = 2
@@ -96,18 +97,23 @@ def _get_db(dbfile, name, target_version):
 
     version = db.execute("SELECT version FROM version").fetchone()["version"]
 
-    ## while version < target_version:
-    ##     log.msg(" need to upgrade from %s to %s" % (version, target_version))
-    ##     try:
-    ##         upgrader = get_upgrader(version+1)
-    ##     except ValueError: # ResourceError??
-    ##         log.msg(" unable to upgrade %s to %s" % (version, version+1))
-    ##         raise DBError("Unable to upgrade %s to version %s, left at %s"
-    ##                       % (dbfile, version+1, version))
-    ##     log.msg(" executing upgrader v%s->v%s" % (version, version+1))
-    ##     db.executescript(upgrader)
-    ##     db.commit()
-    ##     version = version+1
+    if version < target_version and dbfile != ":memory:":
+        backup_fn = "%s-backup-v%d" % (dbfile, version)
+        log.msg(" storing backup of v%d db in %s" % (version, backup_fn))
+        shutil.copy(dbfile, backup_fn)
+
+    while version < target_version:
+        log.msg(" need to upgrade from %s to %s" % (version, target_version))
+        try:
+            upgrader = get_upgrader(name, version+1)
+        except ValueError:
+            log.msg(" unable to upgrade %s to %s" % (version, version+1))
+            raise DBError("Unable to upgrade %s to version %s, left at %s"
+                          % (dbfile, version+1, version))
+        log.msg(" executing upgrader v%s->v%s" % (version, version+1))
+        db.executescript(upgrader)
+        db.commit()
+        version = version+1
 
     if version != target_version:
         raise DBError("Unable to handle db version %s" % version)

src/wormhole_mailbox_server/db-schemas/upgrade-usage-to-v2.sql

@@ -0,0 +1,19 @@
+CREATE TABLE `client_versions`
+(
+ `app_id` VARCHAR,
+ `side` VARCHAR, -- for deduplication of reconnects
+ `connect_time` INTEGER, -- seconds since epoch, rounded to "blur time"
+ -- the client sends us a 'client_version' tuple of (implementation, version)
+ -- the Python client sends e.g. ("python", "0.11.0")
+ `implementation` VARCHAR,
+ `version` VARCHAR
+);
+CREATE INDEX `client_versions_time_idx` on `client_versions` (`connect_time`);
+CREATE INDEX `client_versions_appid_time_idx` on `client_versions` (`app_id`, `connect_time`);
+
+DROP TABLE `version`;
+CREATE TABLE `version`
+(
+ `version` INTEGER -- contains one row, set to 2
+);
+INSERT INTO `version` (`version`) VALUES (2);

src/wormhole_mailbox_server/test/test_database.py

@@ -3,7 +3,7 @@
 from twisted.python import filepath
 from twisted.trial import unittest
 from .. import database
-from ..database import (CHANNELDB_TARGET_VERSION, #USAGEDB_TARGET_VERSION,
+from ..database import (CHANNELDB_TARGET_VERSION, USAGEDB_TARGET_VERSION,
                         _get_db, dump_db, DBError)
 
 class Get(unittest.TestCase):
@@ -56,35 +56,35 @@ def test_failed_create_allows_subsequent_create(self):
         patch.restore()
         _get_db(dbfile.path, "channel", CHANNELDB_TARGET_VERSION)
 
-    def OFF_test_upgrade(self): # disabled until we add a v2 schema
+    def test_upgrade(self):
         basedir = self.mktemp()
         os.mkdir(basedir)
         fn = os.path.join(basedir, "upgrade.db")
-        self.assertNotEqual(CHANNELDB_TARGET_VERSION, 1)
+        self.assertNotEqual(USAGEDB_TARGET_VERSION, 1)
 
         # create an old-version DB in a file
-        db = _get_db(fn, "channel", 1)
+        db = _get_db(fn, "usage", 1)
         rows = db.execute("SELECT * FROM version").fetchall()
         self.assertEqual(len(rows), 1)
         self.assertEqual(rows[0]["version"], 1)
         del db
 
         # then upgrade the file to the latest version
-        dbA = _get_db(fn, "channel", CHANNELDB_TARGET_VERSION)
+        dbA = _get_db(fn, "usage", USAGEDB_TARGET_VERSION)
         rows = dbA.execute("SELECT * FROM version").fetchall()
         self.assertEqual(len(rows), 1)
-        self.assertEqual(rows[0]["version"], CHANNELDB_TARGET_VERSION)
+        self.assertEqual(rows[0]["version"], USAGEDB_TARGET_VERSION)
         dbA_text = dump_db(dbA)
         del dbA
 
         # make sure the upgrades got committed to disk
-        dbB = _get_db(fn, "channel", CHANNELDB_TARGET_VERSION)
+        dbB = _get_db(fn, "usage", USAGEDB_TARGET_VERSION)
         dbB_text = dump_db(dbB)
         del dbB
         self.assertEqual(dbA_text, dbB_text)
 
         # The upgraded schema should be equivalent to that of a new DB.
-        latest_db = _get_db(":memory:", "channel", CHANNELDB_TARGET_VERSION)
+        latest_db = _get_db(":memory:", "usage", USAGEDB_TARGET_VERSION)
         latest_text = dump_db(latest_db)
         with open("up.sql","w") as f: f.write(dbA_text)
         with open("new.sql","w") as f: f.write(latest_text)