diff --git a/src/bleanser/core/ext/sqlite_dumben.py b/src/bleanser/core/ext/sqlite_dumben.py index 490e420..1f330f0 100755 --- a/src/bleanser/core/ext/sqlite_dumben.py +++ b/src/bleanser/core/ext/sqlite_dumben.py @@ -57,11 +57,21 @@ def _dumben_db(output_db: Path) -> None: # the only easy win is making it single line # "UPDATE sqlite_master SET sql = replace(sql, char(10), ' ');" + allow_writable_schema = [ + # seems like some versions of sqlite (e.g. on osx don't allow writable schema without this pragma) + # https://github.com/tekartik/sqflite/blob/master/sqflite_common_ffi/doc/custom_pragmas.md?plain=1 + "PRAGMA sqflite -- db_config_defensive_off", + "PRAGMA writable_schema=ON", + ] # first delete virtual tables -- they might render it impossible to do anything with database at all due to USING # e.g. fb messenger android msys database has this CREATE VIRTUAL TABLE msys_experiment_cache USING experiment_cache # either way virtual tables are basically views, no need to keep them - check_call(_sqlite(output_db, 'PRAGMA writable_schema=ON; DELETE FROM sqlite_master WHERE sql LIKE "%CREATE VIRTUAL TABLE%";')) + with sqlite3.connect(output_db) as conn: + for cmd in allow_writable_schema: + conn.execute(cmd) + conn.execute('DELETE FROM sqlite_master WHERE sql LIKE "%CREATE VIRTUAL TABLE%"') + conn.close() tables = _get_tables(output_db) @@ -74,7 +84,7 @@ def _dumben_db(output_db: Path) -> None: updates.append(upd) cmds = [ - "PRAGMA writable_schema=ON;", + *allow_writable_schema, # drop table doesn't work for special sqlite_ tables # sqlite_sequence is something to do with autoincrement, ends up with some indices noise otherwise # sqlite_stat{1,2,3,4} is something to do with ANALYZE query @@ -87,15 +97,11 @@ def _dumben_db(output_db: Path) -> None: 'VACUUM', ] - # using temporary file because the argument list might end up too long - # e.g. was the case with facebook android databases, too many tables - with TemporaryFile() as tf: + # need to set isolation level to None, otherwise VACUUM fails + with sqlite3.connect(output_db, isolation_level=None) as conn: for cmd in cmds: - tf.write(cmd.encode('utf8') + b'\n') - tf.seek(0) - - # TODO perhaps instead use python3 interface? so it escapes properly - subprocess.run(_sqlite(output_db), check=True, input=tf.read()) + conn.execute(cmd) + conn.close() # make sure it's not corrupted # redirect output to DEVNULL, otherwise it's printing "ok" which is a bit annoying