From 77b703d396c0f095b533b92a561228ebe263ca7e Mon Sep 17 00:00:00 2001 From: Jaisen Mathai Date: Sat, 7 Jan 2023 01:22:12 -0800 Subject: [PATCH 1/7] Initial commit of working tests for sqlite plugin --- elodie/plugins/sqlite/schema.sql | 46 +++++++++++ elodie/plugins/sqlite/sqlite.py | 93 +++++++++++++++++++++ elodie/tests/plugins/sqlite/sqlite_test.py | 95 ++++++++++++++++++++++ 3 files changed, 234 insertions(+) create mode 100644 elodie/plugins/sqlite/schema.sql create mode 100644 elodie/plugins/sqlite/sqlite.py create mode 100644 elodie/tests/plugins/sqlite/sqlite_test.py diff --git a/elodie/plugins/sqlite/schema.sql b/elodie/plugins/sqlite/schema.sql new file mode 100644 index 00000000..8c6a7574 --- /dev/null +++ b/elodie/plugins/sqlite/schema.sql @@ -0,0 +1,46 @@ +CREATE TABLE "metadata" ( + "id" INTEGER, + "appId" TEXT, + "url" TEXT, + "host" TEXT, + "title" TEXT, + "description" TEXT, + "key" TEXT, + "hash" TEXT, + "tags" TEXT, + "size" INTEGER, + "width" INTEGER, + "height" INTEGER, + "rotation" INTEGER, + "exifOrientation" INTEGER, + "exifCameraMake" TEXT, + "exifExposureTime" TEXT, + "exifFNumber" TEXT, + "exifMaxApertureValue" TEXT, + "exifMeteringMode" TEXT, + "exifFlash" TEXT, + "exifFocalLength" TEXT, + "altitude" INTEGER, + "latitude" REAL, + "longitude" REAL, + "views" INTEGER, + "status" INTEGER, + "permission" INTEGER, + "groups" TEXT, + "license" TEXT, + "dateTaken" INTEGER, + "dateTakenDay" INTEGER, + "dateTakenMonth" INTEGER, + "dateTakenYear" INTEGER, + "dateUploaded" INTEGER, + "dateUploadedDay" INTEGER, + "dateUploadedMonth" INTEGER, + "dateUploadedYear" INTEGER, + "pathOriginal" TEXT, + "pathBase" TEXT, + PRIMARY KEY("id" AUTOINCREMENT) +); + +CREATE UNIQUE INDEX "pathOriginal" ON "metadata" ( + "pathOriginal" +); diff --git a/elodie/plugins/sqlite/sqlite.py b/elodie/plugins/sqlite/sqlite.py new file mode 100644 index 00000000..5d93f4c0 --- /dev/null +++ b/elodie/plugins/sqlite/sqlite.py @@ -0,0 +1,93 @@ +""" +SQLite plugin object. +This plugin stores metadata about all media in an sqlite database. + +You'll need to create a SQLite database using the schema.sql file and + reference it in your configuration. + +``` +[PluginSQLite] +database_file=/path/to/database.db + +.. moduleauthor:: Jaisen Mathai +""" +from __future__ import print_function + +import os +import sqlite3 +#import json + +#from google_auth_oauthlib.flow import InstalledAppFlow +#from google.auth.transport.requests import AuthorizedSession +#from google.oauth2.credentials import Credentials + +from elodie.media.photo import Photo +from elodie.media.video import Video +from elodie.plugins.plugins import PluginBase + +class SQLite(PluginBase): + """A class to execute plugin actions. + + Requires a config file with the following configurations set. + database_file: + The full path to the SQLite database (.db). + + """ + + __name__ = 'SQLite' + + def __init__(self): + super(SQLite, self).__init__() + + self.database_schema = '{}{}{}'.format(os.path.dirname(os.path.realpath(__file__)), os.sep, 'schema.sql') + self.database_file = None + if('database_file' in self.config_for_plugin): + self.database_file = self.config_for_plugin['database_file'] + + self.con = sqlite3.connect(self.database_file) + self.con.row_factory = sqlite3.Row + self.cursor = self.con.cursor() + + def after(self, file_path, destination_folder, final_file_path, metadata): + + # We check if the source path exists in the database already. + # If it does then we assume that this is an update operation. + full_destination_path = '{}{}'.format(destination_folder, final_file_path) + self.cursor.execute("SELECT `pathOriginal` FROM `metadata` WHERE `pathOriginal`=:pathOriginal", {'pathOriginal': file_path}) + if(self.cursor.fetchone() is None): + self.log(u'SQLite plugin inserting {}'.format(file_path)) + sql_statement, sql_values = self._insert_row_sql(file_path, full_destination_path, metadata) + else: + self.log(u'SQLite plugin updating {}'.format(file_path)) + sql_statement, sql_values = self._update_row_sql(file_path, full_destination_path, metadata) + + self.cursor.execute(sql_statement, sql_values) + + def batch(self): + pass + + def before(self, file_path, destination_folder): + pass + + def create_schema(self): + with open(self.database_schema, 'r') as fp_schema: + sql_statement = fp_schema.read() + + with self.con: + self.cursor.executescript(sql_statement) + + def run_query(self, sql, values): + self.cursor.execute(sql, values) + return self.cursor.fetchall() + + def _insert_row_sql(self, current_path, final_path, metadata): + return ( + "INSERT INTO `metadata` (`pathOriginal`) VALUES(:pathOriginal)", + {'pathOriginal': final_path} + ) + + def _update_row_sql(self, current_path, final_path, metadata): + return ( + "UPDATE `metadata` SET `pathOriginal`=:pathOriginal WHERE `pathOriginal`=:currentPathOriginal", + {'currentPathOriginal': current_path, 'pathOriginal': final_path} + ) diff --git a/elodie/tests/plugins/sqlite/sqlite_test.py b/elodie/tests/plugins/sqlite/sqlite_test.py new file mode 100644 index 00000000..d789ac19 --- /dev/null +++ b/elodie/tests/plugins/sqlite/sqlite_test.py @@ -0,0 +1,95 @@ +from __future__ import absolute_import +# Project imports +import mock +import os +import sys +from tempfile import gettempdir + +sys.path.insert(0, os.path.abspath(os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))))) +sys.path.insert(0, os.path.abspath(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))) + +import helper +from elodie.config import load_config +from elodie.plugins.sqlite.sqlite import SQLite +from elodie.media.audio import Audio +from elodie.media.photo import Photo + +# Globals to simplify mocking configs +db_schema = helper.get_file('plugins/sqlite/schema.sql') +config_string = """ +[Plugins] +plugins=SQLite + +[PluginSQLite] +database_file={} + """ +config_string_fmt = config_string.format( + ':memory:' +) + +setup_module = helper.setup_module +teardown_module = helper.teardown_module + +@mock.patch('elodie.config.config_file', '%s/config.ini-sqlite-insert' % gettempdir()) +def test_sqlite_insert(): + with open('%s/config.ini-sqlite-insert' % gettempdir(), 'w') as f: + f.write(config_string_fmt) + if hasattr(load_config, 'config'): + del load_config.config + + sqlite_plugin = SQLite() + sqlite_plugin.create_schema() + sqlite_plugin.after('/some/source/path', '/folder/path', '/file/path.jpg', {}) + results = sqlite_plugin.run_query( + 'SELECT * FROM `metadata` WHERE `pathOriginal`=:pathOriginal', + {'pathOriginal': '/folder/path/file/path.jpg'} + ); + + if hasattr(load_config, 'config'): + del load_config.config + + assert len(results) == 1, results + +@mock.patch('elodie.config.config_file', '%s/config.ini-sqlite-insert-multiple' % gettempdir()) +def test_sqlite_insert_multiple(): + with open('%s/config.ini-sqlite-insert-multiple' % gettempdir(), 'w') as f: + f.write(config_string_fmt) + if hasattr(load_config, 'config'): + del load_config.config + + sqlite_plugin = SQLite() + sqlite_plugin.create_schema() + sqlite_plugin.after('/some/source/path', '/folder/path', '/file/path.jpg', {}) + sqlite_plugin.after('/some/source/path', '/folder/path', '/file/path2.jpg', {}) + results = sqlite_plugin.run_query( + 'SELECT * FROM `metadata`', + {} + ); + + if hasattr(load_config, 'config'): + del load_config.config + + assert len(results) == 2, results + +@mock.patch('elodie.config.config_file', '%s/config.ini-sqlite-update' % gettempdir()) +def test_sqlite_insert_multiple(): + with open('%s/config.ini-sqlite-update' % gettempdir(), 'w') as f: + f.write(config_string_fmt) + if hasattr(load_config, 'config'): + del load_config.config + + sqlite_plugin = SQLite() + sqlite_plugin.create_schema() + sqlite_plugin.after('/some/source/path', '/folder/path', '/file/path.jpg', {}) + sqlite_plugin.after('/folder/path/file/path.jpg', '/new-folder/path', '/new-file/path.jpg', {}) + results = sqlite_plugin.run_query( + 'SELECT * FROM `metadata`', + {} + ); + print(results) + + if hasattr(load_config, 'config'): + del load_config.config + + assert len(results) == 1, results + assert results[0]['pathOriginal'] == '/new-folder/path/new-file/path.jpg', results From 0ecd73b1c31bb5897357d76205378978ee1d3532 Mon Sep 17 00:00:00 2001 From: Jaisen Mathai Date: Sat, 7 Jan 2023 12:31:42 -0800 Subject: [PATCH 2/7] Update sqlite schema and add checksum --- elodie/filesystem.py | 4 ++ elodie/media/base.py | 15 +++++++ elodie/plugins/sqlite/schema.sql | 49 +++------------------- elodie/plugins/sqlite/sqlite.py | 18 ++++---- elodie/tests/media/base_test.py | 27 ++++++++++++ elodie/tests/plugins/sqlite/__init__.py | 0 elodie/tests/plugins/sqlite/sqlite_test.py | 45 ++++++++++++++++---- 7 files changed, 101 insertions(+), 57 deletions(-) create mode 100644 elodie/tests/plugins/sqlite/__init__.py diff --git a/elodie/filesystem.py b/elodie/filesystem.py index 92d7807e..d4ab0c4d 100644 --- a/elodie/filesystem.py +++ b/elodie/filesystem.py @@ -533,6 +533,10 @@ def process_file(self, _file, destination, media, **kwargs): _file) return + # Set the checksum and refresh metadata. + media.set_checksum(checksum) + metadata = media.get_metadata() + # Run `before()` for every loaded plugin and if any of them raise an exception # then we skip importing the file and log a message. plugins_run_before_status = self.plugins.run_all_before(_file, destination) diff --git a/elodie/media/base.py b/elodie/media/base.py index 83f29c5a..3d1c642a 100644 --- a/elodie/media/base.py +++ b/elodie/media/base.py @@ -74,6 +74,10 @@ def get_camera_make(self): def get_camera_model(self): return None + def get_checksum(self): + self.get_metadata() + return self.metadata['checksum'] + def get_metadata(self, update_cache=False): """Get a dictionary of metadata for any file. @@ -90,6 +94,7 @@ def get_metadata(self, update_cache=False): source = self.source self.metadata = { + 'checksum': None, 'date_taken': self.get_date_taken(), 'camera_make': self.get_camera_make(), 'camera_model': self.get_camera_model(), @@ -178,6 +183,16 @@ def set_album_from_folder(self): self.set_album(folder) return True + def set_checksum(self, new_checksum): + """Add the checksum to the metadata. + + The checksum in `metadata` is passed in and not derived from the file itself. + It was added to pass in to plugins. + See gh-68. + """ + self.get_metadata() + self.metadata['checksum'] = new_checksum + def set_metadata_basename(self, new_basename): """Update the basename attribute in the metadata dict for this instance. diff --git a/elodie/plugins/sqlite/schema.sql b/elodie/plugins/sqlite/schema.sql index 8c6a7574..7e59d023 100644 --- a/elodie/plugins/sqlite/schema.sql +++ b/elodie/plugins/sqlite/schema.sql @@ -1,46 +1,9 @@ CREATE TABLE "metadata" ( - "id" INTEGER, - "appId" TEXT, - "url" TEXT, - "host" TEXT, - "title" TEXT, - "description" TEXT, - "key" TEXT, - "hash" TEXT, - "tags" TEXT, - "size" INTEGER, - "width" INTEGER, - "height" INTEGER, - "rotation" INTEGER, - "exifOrientation" INTEGER, - "exifCameraMake" TEXT, - "exifExposureTime" TEXT, - "exifFNumber" TEXT, - "exifMaxApertureValue" TEXT, - "exifMeteringMode" TEXT, - "exifFlash" TEXT, - "exifFocalLength" TEXT, - "altitude" INTEGER, - "latitude" REAL, - "longitude" REAL, - "views" INTEGER, - "status" INTEGER, - "permission" INTEGER, - "groups" TEXT, - "license" TEXT, - "dateTaken" INTEGER, - "dateTakenDay" INTEGER, - "dateTakenMonth" INTEGER, - "dateTakenYear" INTEGER, - "dateUploaded" INTEGER, - "dateUploadedDay" INTEGER, - "dateUploadedMonth" INTEGER, - "dateUploadedYear" INTEGER, - "pathOriginal" TEXT, - "pathBase" TEXT, + "id" INTEGER UNIQUE, + "path" TEXT UNIQUE, + "hash" TEXT UNIQUE, + "metadata" TEXT, + "created" INTEGER, + "modified" INTEGER, PRIMARY KEY("id" AUTOINCREMENT) ); - -CREATE UNIQUE INDEX "pathOriginal" ON "metadata" ( - "pathOriginal" -); diff --git a/elodie/plugins/sqlite/sqlite.py b/elodie/plugins/sqlite/sqlite.py index 5d93f4c0..b4ea400a 100644 --- a/elodie/plugins/sqlite/sqlite.py +++ b/elodie/plugins/sqlite/sqlite.py @@ -13,8 +13,10 @@ """ from __future__ import print_function +import json import os import sqlite3 +import time #import json #from google_auth_oauthlib.flow import InstalledAppFlow @@ -53,10 +55,10 @@ def after(self, file_path, destination_folder, final_file_path, metadata): # We check if the source path exists in the database already. # If it does then we assume that this is an update operation. full_destination_path = '{}{}'.format(destination_folder, final_file_path) - self.cursor.execute("SELECT `pathOriginal` FROM `metadata` WHERE `pathOriginal`=:pathOriginal", {'pathOriginal': file_path}) + self.cursor.execute("SELECT `path` FROM `metadata` WHERE `path`=:path", {'path': file_path}) if(self.cursor.fetchone() is None): self.log(u'SQLite plugin inserting {}'.format(file_path)) - sql_statement, sql_values = self._insert_row_sql(file_path, full_destination_path, metadata) + sql_statement, sql_values = self._insert_row_sql(full_destination_path, metadata) else: self.log(u'SQLite plugin updating {}'.format(file_path)) sql_statement, sql_values = self._update_row_sql(file_path, full_destination_path, metadata) @@ -80,14 +82,16 @@ def run_query(self, sql, values): self.cursor.execute(sql, values) return self.cursor.fetchall() - def _insert_row_sql(self, current_path, final_path, metadata): + def _insert_row_sql(self, final_path, metadata): + timestamp = int(time.time()) return ( - "INSERT INTO `metadata` (`pathOriginal`) VALUES(:pathOriginal)", - {'pathOriginal': final_path} + "INSERT INTO `metadata` (`path`, `metadata`, `created`, `modified`) VALUES(:path, :metadata, :created, :modified)", + {'path': final_path, 'metadata': json.dumps(metadata), 'created': timestamp, 'modified': timestamp} ) def _update_row_sql(self, current_path, final_path, metadata): + timestamp = int(time.time()) return ( - "UPDATE `metadata` SET `pathOriginal`=:pathOriginal WHERE `pathOriginal`=:currentPathOriginal", - {'currentPathOriginal': current_path, 'pathOriginal': final_path} + "UPDATE `metadata` SET `path`=:path, `metadata`=json(:metadata), `modified`=:modified WHERE `path`=:currentPath", + {'currentPath': current_path, 'path': final_path, 'metadata': json.dumps(metadata), 'modified': timestamp} ) diff --git a/elodie/tests/media/base_test.py b/elodie/tests/media/base_test.py index 9838c808..27671515 100644 --- a/elodie/tests/media/base_test.py +++ b/elodie/tests/media/base_test.py @@ -38,6 +38,19 @@ def test_get_class_by_file_without_extension(): assert cls is None, cls +def test_get_checksum(): + temporary_folder, folder = helper.create_working_folder() + + origin = '%s/%s' % (folder, 'with-original-name.jpg') + file = helper.get_file('with-original-name.jpg') + + shutil.copyfile(file, origin) + + media = Media.get_class_by_file(origin, [Photo]) + checksum = media.get_checksum() + + assert checksum == None, checksum + def test_get_original_name(): temporary_folder, folder = helper.create_working_folder() @@ -101,6 +114,20 @@ def test_set_album_from_folder(): assert metadata_new['album'] == new_album_name, metadata_new['album'] +def test_get_checksum(): + temporary_folder, folder = helper.create_working_folder() + + origin = '%s/%s' % (folder, 'with-original-name.jpg') + file = helper.get_file('with-original-name.jpg') + + shutil.copyfile(file, origin) + + media = Media.get_class_by_file(origin, [Photo]) + media.set_checksum('foo') + checksum = media.get_checksum() + + assert checksum == 'foo', checksum + def test_set_metadata(): temporary_folder, folder = helper.create_working_folder() diff --git a/elodie/tests/plugins/sqlite/__init__.py b/elodie/tests/plugins/sqlite/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/elodie/tests/plugins/sqlite/sqlite_test.py b/elodie/tests/plugins/sqlite/sqlite_test.py index d789ac19..53f569e2 100644 --- a/elodie/tests/plugins/sqlite/sqlite_test.py +++ b/elodie/tests/plugins/sqlite/sqlite_test.py @@ -41,14 +41,15 @@ def test_sqlite_insert(): sqlite_plugin.create_schema() sqlite_plugin.after('/some/source/path', '/folder/path', '/file/path.jpg', {}) results = sqlite_plugin.run_query( - 'SELECT * FROM `metadata` WHERE `pathOriginal`=:pathOriginal', - {'pathOriginal': '/folder/path/file/path.jpg'} + 'SELECT * FROM `metadata` WHERE `path`=:path', + {'path': '/folder/path/file/path.jpg'} ); if hasattr(load_config, 'config'): del load_config.config assert len(results) == 1, results + assert results[0]['path'] == '/folder/path/file/path.jpg', results[0] @mock.patch('elodie.config.config_file', '%s/config.ini-sqlite-insert-multiple' % gettempdir()) def test_sqlite_insert_multiple(): @@ -70,9 +71,11 @@ def test_sqlite_insert_multiple(): del load_config.config assert len(results) == 2, results + assert results[0]['path'] == '/folder/path/file/path.jpg', results[0] + assert results[1]['path'] == '/folder/path/file/path2.jpg', results[1] @mock.patch('elodie.config.config_file', '%s/config.ini-sqlite-update' % gettempdir()) -def test_sqlite_insert_multiple(): +def test_sqlite_update(): with open('%s/config.ini-sqlite-update' % gettempdir(), 'w') as f: f.write(config_string_fmt) if hasattr(load_config, 'config'): @@ -80,16 +83,44 @@ def test_sqlite_insert_multiple(): sqlite_plugin = SQLite() sqlite_plugin.create_schema() - sqlite_plugin.after('/some/source/path', '/folder/path', '/file/path.jpg', {}) - sqlite_plugin.after('/folder/path/file/path.jpg', '/new-folder/path', '/new-file/path.jpg', {}) + # write to /folder/path/file/path.jpg and then update it + sqlite_plugin.after('/some/source/path', '/folder/path', '/file/path.jpg', {'foo':'bar'}) + sqlite_plugin.after('/folder/path/file/path.jpg', '/new-folder/path', '/new-file/path.jpg', {'foo':'updated'}) results = sqlite_plugin.run_query( 'SELECT * FROM `metadata`', {} ); - print(results) if hasattr(load_config, 'config'): del load_config.config assert len(results) == 1, results - assert results[0]['pathOriginal'] == '/new-folder/path/new-file/path.jpg', results + assert results[0]['path'] == '/new-folder/path/new-file/path.jpg', results + assert results[0]['metadata'] == '{"foo":"updated"}', results + +@mock.patch('elodie.config.config_file', '%s/config.ini-sqlite-update-multiple' % gettempdir()) +def test_sqlite_update_multiple(): + with open('%s/config.ini-sqlite-update-multiple' % gettempdir(), 'w') as f: + f.write(config_string_fmt) + if hasattr(load_config, 'config'): + del load_config.config + + sqlite_plugin = SQLite() + sqlite_plugin.create_schema() + sqlite_plugin.after('/some/source/path', '/folder/path', '/file/path.jpg', {'foo':'bar'}) + sqlite_plugin.after('/some/source/path', '/folder/path', '/file/path2.jpg', {'foo':'bar'}) + sqlite_plugin.after('/folder/path/file/path.jpg', '/new-folder/path', '/new-file/path.jpg', {'foo':'updated'}) + sqlite_plugin.after('/folder/path/file/path2.jpg', '/new-folder/path', '/new-file/path2.jpg', {'foo':'updated2'}) + results = sqlite_plugin.run_query( + 'SELECT * FROM `metadata`', + {} + ); + + if hasattr(load_config, 'config'): + del load_config.config + + assert len(results) == 2, results + assert results[0]['path'] == '/new-folder/path/new-file/path.jpg', results[0] + assert results[0]['metadata'] == '{"foo":"updated"}', results[0] + assert results[1]['path'] == '/new-folder/path/new-file/path2.jpg', results[1] + assert results[1]['metadata'] == '{"foo":"updated2"}', results[1] From 83e1723f91fb5a43c2d7fa3cb21e1dbc67abad29 Mon Sep 17 00:00:00 2001 From: Jaisen Mathai Date: Fri, 10 Mar 2023 21:51:52 -0800 Subject: [PATCH 3/7] add plugin functions --- elodie.py | 4 ++++ elodie/plugins/plugins.py | 3 +++ elodie/plugins/sqlite/sqlite.py | 6 +++++- 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/elodie.py b/elodie.py index 521548c2..d80e682a 100755 --- a/elodie.py +++ b/elodie.py @@ -174,6 +174,10 @@ def _generate_db(source, debug): log.progress() db.update_hash_db() + + plugins = Plugins() + plugins.run_generate_db() + log.progress('', True) result.write() diff --git a/elodie/plugins/plugins.py b/elodie/plugins/plugins.py index bbf16f81..7bc70eb8 100644 --- a/elodie/plugins/plugins.py +++ b/elodie/plugins/plugins.py @@ -59,6 +59,9 @@ def display(self, msg): {self.__name__: msg} )) + def generate_db(self, hash_db): + pass + class PluginDb(object): """A database module which provides a simple key/value database. The database is a JSON file located at %application_directory%/plugins/%pluginname.lower()%.json diff --git a/elodie/plugins/sqlite/sqlite.py b/elodie/plugins/sqlite/sqlite.py index b4ea400a..6e4d8217 100644 --- a/elodie/plugins/sqlite/sqlite.py +++ b/elodie/plugins/sqlite/sqlite.py @@ -71,12 +71,16 @@ def batch(self): def before(self, file_path, destination_folder): pass - def create_schema(self): + def generate_db(self, hash_db): + pass + + """def create_schema(self): with open(self.database_schema, 'r') as fp_schema: sql_statement = fp_schema.read() with self.con: self.cursor.executescript(sql_statement) + """ def run_query(self, sql, values): self.cursor.execute(sql, values) From 52da40ba5e39eb9eb399f204e485d86decfccbac Mon Sep 17 00:00:00 2001 From: Jaisen Mathai Date: Sat, 6 May 2023 22:32:22 -0700 Subject: [PATCH 4/7] Working tests --- elodie/plugins/googlephotos/Readme.markdown | 2 - elodie/plugins/plugins.py | 3 + elodie/plugins/sqlite/schema.sql | 19 +++-- elodie/plugins/sqlite/sqlite.py | 85 ++++++++++++++++----- elodie/tests/plugins/sqlite/sqlite_test.py | 59 +++++++++----- 5 files changed, 122 insertions(+), 46 deletions(-) diff --git a/elodie/plugins/googlephotos/Readme.markdown b/elodie/plugins/googlephotos/Readme.markdown index 64e955e9..f6c9af0b 100644 --- a/elodie/plugins/googlephotos/Readme.markdown +++ b/elodie/plugins/googlephotos/Readme.markdown @@ -1,7 +1,5 @@ # Google Photos Plugin for Elodie -[![Build Status](https://travis-ci.org/jmathai/elodie.svg?branch=master)](https://travis-ci.org/jmathai/elodie) [![Coverage Status](https://coveralls.io/repos/github/jmathai/elodie/badge.svg?branch=master)](https://coveralls.io/github/jmathai/elodie?branch=master) [![Scrutinizer Code Quality](https://scrutinizer-ci.com/g/jmathai/elodie/badges/quality-score.png?b=master)](https://scrutinizer-ci.com/g/jmathai/elodie/?branch=master) - This plugin uploads all photos imported using Elodie to Google Photos. It was created after [Google Photos and Google Drive synchronization was deprecated](https://www.blog.google/products/photos/simplifying-google-photos-and-google-drive/). It aims to replicate my [workflow using Google Photos, Google Drive and Elodie](https://artplusmarketing.com/one-year-of-using-an-automated-photo-organization-and-archiving-workflow-89cf9ad7bddf). I didn't intend on it, but it turned out that with this plugin you can use Google Photos with Google Drive, iCloud Drive, Dropbox or no cloud storage service while still using Google Photos for viewing and experiencing your photo library. diff --git a/elodie/plugins/plugins.py b/elodie/plugins/plugins.py index 7bc70eb8..755fbc77 100644 --- a/elodie/plugins/plugins.py +++ b/elodie/plugins/plugins.py @@ -34,8 +34,11 @@ class PluginBase(object): __name__ = 'PluginBase' def __init__(self): + # Initializes variables for the plugin + self.application_directory = application_directory # Loads the config for the plugin from config.ini self.config_for_plugin = load_config_for_plugin(self.__name__) + # Initializes a database for the plugin. self.db = PluginDb(self.__name__) def after(self, file_path, destination_folder, final_file_path, metadata): diff --git a/elodie/plugins/sqlite/schema.sql b/elodie/plugins/sqlite/schema.sql index 7e59d023..6df11924 100644 --- a/elodie/plugins/sqlite/schema.sql +++ b/elodie/plugins/sqlite/schema.sql @@ -1,9 +1,16 @@ CREATE TABLE "metadata" ( - "id" INTEGER UNIQUE, - "path" TEXT UNIQUE, - "hash" TEXT UNIQUE, - "metadata" TEXT, - "created" INTEGER, - "modified" INTEGER, + "id" INTEGER NOT NULL, + "hash" TEXT NOT NULL UNIQUE, + "path" TEXT NOT NULL UNIQUE, + "album" INTEGER, + "camera_make" TEXT, + "camera_model" TEXT, + "date_taken" TEXT, + "latitude" REAL, + "location_name" TEXT, + "longitude" REAL, + "original_name" TEXT, + "title" TEXT, + "_modified" INTEGER, PRIMARY KEY("id" AUTOINCREMENT) ); diff --git a/elodie/plugins/sqlite/sqlite.py b/elodie/plugins/sqlite/sqlite.py index 6e4d8217..4198d21b 100644 --- a/elodie/plugins/sqlite/sqlite.py +++ b/elodie/plugins/sqlite/sqlite.py @@ -2,8 +2,10 @@ SQLite plugin object. This plugin stores metadata about all media in an sqlite database. -You'll need to create a SQLite database using the schema.sql file and - reference it in your configuration. +You'll need to include [PluginSQLite] in your config file. +By default, the sqlite database will be created in your application + directory. If you want to specify a different path then + specify a fully qualified `database_file` path. ``` [PluginSQLite] @@ -41,14 +43,20 @@ class SQLite(PluginBase): def __init__(self): super(SQLite, self).__init__() - self.database_schema = '{}{}{}'.format(os.path.dirname(os.path.realpath(__file__)), os.sep, 'schema.sql') - self.database_file = None + # Default the database file to be in the application plugin directory. + # Override with value from config file. + self.database_file = '{}/plugins/{}/elodie.db'.format( + self.application_directory, + __name__.lower() + ) if('database_file' in self.config_for_plugin): self.database_file = self.config_for_plugin['database_file'] self.con = sqlite3.connect(self.database_file) self.con.row_factory = sqlite3.Row self.cursor = self.con.cursor() + if(not self._validate_schema()): + self._create_schema() def after(self, file_path, destination_folder, final_file_path, metadata): @@ -56,6 +64,7 @@ def after(self, file_path, destination_folder, final_file_path, metadata): # If it does then we assume that this is an update operation. full_destination_path = '{}{}'.format(destination_folder, final_file_path) self.cursor.execute("SELECT `path` FROM `metadata` WHERE `path`=:path", {'path': file_path}) + if(self.cursor.fetchone() is None): self.log(u'SQLite plugin inserting {}'.format(file_path)) sql_statement, sql_values = self._insert_row_sql(full_destination_path, metadata) @@ -71,31 +80,71 @@ def batch(self): def before(self, file_path, destination_folder): pass - def generate_db(self, hash_db): - pass + def _create_schema(self): + self.database_schema = '{}{}{}'.format( + os.path.dirname(os.path.realpath(__file__)), + os.sep, + 'schema.sql' + ) - """def create_schema(self): with open(self.database_schema, 'r') as fp_schema: sql_statement = fp_schema.read() - - with self.con: + print(sql_statement) self.cursor.executescript(sql_statement) - """ - def run_query(self, sql, values): + def _insert_row_sql(self, final_path, metadata): + path = '{}/{}.{}'.format( + metadata['directory_path'], + metadata['base_name'], + metadata['extension'] + ) + return ( + """INSERT INTO `metadata` ( + `hash`, `path`, `album`, `camera_make`, `camera_model`, + `date_taken`, `latitude`, `location_name`, `longitude`, + `original_name`, `title`) + VALUES ( + :hash, :path, :album, :camera_make, :camera_model, + :date_taken, :latitude, :location_name, :longitude, + :original_name, :title)""", + self._sql_values(final_path, metadata) + ) + + def _run_query(self, sql, values): self.cursor.execute(sql, values) return self.cursor.fetchall() - def _insert_row_sql(self, final_path, metadata): + def _sql_values(self, final_path, metadata, current_path=None): timestamp = int(time.time()) - return ( - "INSERT INTO `metadata` (`path`, `metadata`, `created`, `modified`) VALUES(:path, :metadata, :created, :modified)", - {'path': final_path, 'metadata': json.dumps(metadata), 'created': timestamp, 'modified': timestamp} - ) + return { + 'hash': metadata['checksum'], + 'path': final_path, + 'album': metadata['album'], + 'camera_make': metadata['camera_make'], + 'camera_model': metadata['camera_model'], + 'date_taken': metadata['date_taken'], + 'latitude': metadata['latitude'], + 'location_name': None, + 'longitude': metadata['longitude'], + 'original_name': metadata['original_name'], + 'title': metadata['title'], + 'current_path': current_path, + '_modified': timestamp + } def _update_row_sql(self, current_path, final_path, metadata): timestamp = int(time.time()) return ( - "UPDATE `metadata` SET `path`=:path, `metadata`=json(:metadata), `modified`=:modified WHERE `path`=:currentPath", - {'currentPath': current_path, 'path': final_path, 'metadata': json.dumps(metadata), 'modified': timestamp} + """UPDATE `metadata` SET `hash`=:hash, `path`=:path, `album`=:album, `camera_make`=:camera_make, + `camera_model`=:camera_model, `date_taken`=:date_taken, `latitude`=:latitude, + `longitude`=:longitude, `original_name`=:original_name, `title`=:title + WHERE `path`=:current_path""", + self._sql_values(final_path, metadata, current_path) ) + + def _validate_schema(self): + try: + self.cursor.execute('SELECT * FROM `metadata` LIMIT 1'); + return True + except sqlite3.OperationalError: + return False diff --git a/elodie/tests/plugins/sqlite/sqlite_test.py b/elodie/tests/plugins/sqlite/sqlite_test.py index 53f569e2..24fca68e 100644 --- a/elodie/tests/plugins/sqlite/sqlite_test.py +++ b/elodie/tests/plugins/sqlite/sqlite_test.py @@ -27,6 +27,22 @@ ':memory:' ) +mock_metadata = { + 'checksum': 'checksum-val', + 'date_taken': 1234567890, + 'camera_make': 'camera_make-val', + 'camera_model': 'camera_model-val', + 'latitude': 0.1, + 'longitude': 0.2, + 'album': 'album-val', + 'title': 'title-val', + 'mime_type': 'mime_type-val', + 'original_name': 'original_name-val', + 'base_name': 'base_name-val', + 'extension': 'extension-val', + 'directory_path': 'directory_path-val' + } + setup_module = helper.setup_module teardown_module = helper.teardown_module @@ -38,9 +54,8 @@ def test_sqlite_insert(): del load_config.config sqlite_plugin = SQLite() - sqlite_plugin.create_schema() - sqlite_plugin.after('/some/source/path', '/folder/path', '/file/path.jpg', {}) - results = sqlite_plugin.run_query( + sqlite_plugin.after('/some/source/path.jpg', '/folder/path', '/file/path.jpg', mock_metadata) + results = sqlite_plugin._run_query( 'SELECT * FROM `metadata` WHERE `path`=:path', {'path': '/folder/path/file/path.jpg'} ); @@ -59,10 +74,10 @@ def test_sqlite_insert_multiple(): del load_config.config sqlite_plugin = SQLite() - sqlite_plugin.create_schema() - sqlite_plugin.after('/some/source/path', '/folder/path', '/file/path.jpg', {}) - sqlite_plugin.after('/some/source/path', '/folder/path', '/file/path2.jpg', {}) - results = sqlite_plugin.run_query( + sqlite_plugin.after('/some/source/path.jpg', '/folder/path', '/file/path.jpg', mock_metadata) + mock_metadata_2 = {**mock_metadata, **{'checksum': 'new-hash'}} + sqlite_plugin.after('/some/source/path.jpg', '/folder/path', '/file/path2.jpg', mock_metadata_2) + results = sqlite_plugin._run_query( 'SELECT * FROM `metadata`', {} ); @@ -82,11 +97,11 @@ def test_sqlite_update(): del load_config.config sqlite_plugin = SQLite() - sqlite_plugin.create_schema() # write to /folder/path/file/path.jpg and then update it - sqlite_plugin.after('/some/source/path', '/folder/path', '/file/path.jpg', {'foo':'bar'}) - sqlite_plugin.after('/folder/path/file/path.jpg', '/new-folder/path', '/new-file/path.jpg', {'foo':'updated'}) - results = sqlite_plugin.run_query( + sqlite_plugin.after('/some/source/path.jpg', '/folder/path', '/file/path.jpg', mock_metadata) + mock_metadata_2 = {**mock_metadata, **{'title': 'title-val-new'}} + sqlite_plugin.after('/folder/path/file/path.jpg', '/new-folder/path', '/new-file/path.jpg', mock_metadata_2) + results = sqlite_plugin._run_query( 'SELECT * FROM `metadata`', {} ); @@ -96,7 +111,7 @@ def test_sqlite_update(): assert len(results) == 1, results assert results[0]['path'] == '/new-folder/path/new-file/path.jpg', results - assert results[0]['metadata'] == '{"foo":"updated"}', results + assert results[0]['title'] == 'title-val-new', results @mock.patch('elodie.config.config_file', '%s/config.ini-sqlite-update-multiple' % gettempdir()) def test_sqlite_update_multiple(): @@ -106,12 +121,16 @@ def test_sqlite_update_multiple(): del load_config.config sqlite_plugin = SQLite() - sqlite_plugin.create_schema() - sqlite_plugin.after('/some/source/path', '/folder/path', '/file/path.jpg', {'foo':'bar'}) - sqlite_plugin.after('/some/source/path', '/folder/path', '/file/path2.jpg', {'foo':'bar'}) - sqlite_plugin.after('/folder/path/file/path.jpg', '/new-folder/path', '/new-file/path.jpg', {'foo':'updated'}) - sqlite_plugin.after('/folder/path/file/path2.jpg', '/new-folder/path', '/new-file/path2.jpg', {'foo':'updated2'}) - results = sqlite_plugin.run_query( + mock_metadata_1 = mock_metadata + mock_metadata_2 = {**mock_metadata, **{'checksum': 'checksum-val-2', 'title': 'title-val-2'}} + sqlite_plugin.after('/some/source/path.jpg', '/folder/path', '/file/path.jpg', mock_metadata) + sqlite_plugin.after('/some/source/path2.jpg', '/folder/path', '/file/path2.jpg', mock_metadata_2) + + mock_metadata_1_upd = {**mock_metadata_1, **{'title': 'title-val-upd'}} + mock_metadata_2_upd = {**mock_metadata_2, **{'title': 'title-val-2-upd'}} + sqlite_plugin.after('/folder/path/file/path.jpg', '/new-folder/path', '/new-file/path.jpg', mock_metadata_1_upd) + sqlite_plugin.after('/folder/path/file/path2.jpg', '/new-folder/path', '/new-file/path2.jpg', mock_metadata_2_upd) + results = sqlite_plugin._run_query( 'SELECT * FROM `metadata`', {} ); @@ -121,6 +140,6 @@ def test_sqlite_update_multiple(): assert len(results) == 2, results assert results[0]['path'] == '/new-folder/path/new-file/path.jpg', results[0] - assert results[0]['metadata'] == '{"foo":"updated"}', results[0] + assert results[0]['title'] == 'title-val-upd', results[0] assert results[1]['path'] == '/new-folder/path/new-file/path2.jpg', results[1] - assert results[1]['metadata'] == '{"foo":"updated2"}', results[1] + assert results[1]['title'] == 'title-val-2-upd', results[1] From 0e15c34bdb551274573e4f9a4bf7e7e63e48edc3 Mon Sep 17 00:00:00 2001 From: Jaisen Mathai Date: Sat, 6 May 2023 22:40:27 -0700 Subject: [PATCH 5/7] Add location name to db --- elodie/plugins/sqlite/sqlite.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/elodie/plugins/sqlite/sqlite.py b/elodie/plugins/sqlite/sqlite.py index 4198d21b..085fa489 100644 --- a/elodie/plugins/sqlite/sqlite.py +++ b/elodie/plugins/sqlite/sqlite.py @@ -25,6 +25,7 @@ #from google.auth.transport.requests import AuthorizedSession #from google.oauth2.credentials import Credentials +from elodie.geolocation import place_name from elodie.media.photo import Photo from elodie.media.video import Video from elodie.plugins.plugins import PluginBase @@ -124,7 +125,7 @@ def _sql_values(self, final_path, metadata, current_path=None): 'camera_model': metadata['camera_model'], 'date_taken': metadata['date_taken'], 'latitude': metadata['latitude'], - 'location_name': None, + 'location_name': place_name(metadata['latitude'], metadata['longitude'])['default'], 'longitude': metadata['longitude'], 'original_name': metadata['original_name'], 'title': metadata['title'], From b3377b1ab0b597433ebf017f932ea9bd903bc310 Mon Sep 17 00:00:00 2001 From: Jaisen Mathai Date: Sun, 7 May 2023 00:27:10 -0700 Subject: [PATCH 6/7] Regenerate db working --- elodie/plugins/sqlite/schema.sql | 2 +- elodie/plugins/sqlite/sqlite.py | 59 +++++++++++++--------- elodie/tests/plugins/sqlite/sqlite_test.py | 34 +++++++++++-- 3 files changed, 68 insertions(+), 27 deletions(-) diff --git a/elodie/plugins/sqlite/schema.sql b/elodie/plugins/sqlite/schema.sql index 6df11924..682926c7 100644 --- a/elodie/plugins/sqlite/schema.sql +++ b/elodie/plugins/sqlite/schema.sql @@ -11,6 +11,6 @@ CREATE TABLE "metadata" ( "longitude" REAL, "original_name" TEXT, "title" TEXT, - "_modified" INTEGER, + "_modified" TEXT, PRIMARY KEY("id" AUTOINCREMENT) ); diff --git a/elodie/plugins/sqlite/sqlite.py b/elodie/plugins/sqlite/sqlite.py index 085fa489..ba6f87c3 100644 --- a/elodie/plugins/sqlite/sqlite.py +++ b/elodie/plugins/sqlite/sqlite.py @@ -26,8 +26,8 @@ #from google.oauth2.credentials import Credentials from elodie.geolocation import place_name -from elodie.media.photo import Photo -from elodie.media.video import Video +from elodie.localstorage import Db +from elodie.media.base import Base, get_all_subclasses from elodie.plugins.plugins import PluginBase class SQLite(PluginBase): @@ -60,20 +60,7 @@ def __init__(self): self._create_schema() def after(self, file_path, destination_folder, final_file_path, metadata): - - # We check if the source path exists in the database already. - # If it does then we assume that this is an update operation. - full_destination_path = '{}{}'.format(destination_folder, final_file_path) - self.cursor.execute("SELECT `path` FROM `metadata` WHERE `path`=:path", {'path': file_path}) - - if(self.cursor.fetchone() is None): - self.log(u'SQLite plugin inserting {}'.format(file_path)) - sql_statement, sql_values = self._insert_row_sql(full_destination_path, metadata) - else: - self.log(u'SQLite plugin updating {}'.format(file_path)) - sql_statement, sql_values = self._update_row_sql(file_path, full_destination_path, metadata) - - self.cursor.execute(sql_statement, sql_values) + self._upsert(file_path, destination_folder, final_file_path, metadata) def batch(self): pass @@ -81,6 +68,19 @@ def batch(self): def before(self, file_path, destination_folder): pass + def generate_db(self): + db = Db() + for checksum, file_path in db.all(): + subclasses = get_all_subclasses() + media = Base.get_class_by_file(file_path, get_all_subclasses()) + media.set_checksum( + db.checksum(file_path) + ) + metadata = media.get_metadata() + destination_folder = os.path.dirname(file_path) + final_file_path = '{}{}'.format(os.path.sep, os.path.basename(file_path)) + self._upsert(file_path, destination_folder, final_file_path, metadata) + def _create_schema(self): self.database_schema = '{}{}{}'.format( os.path.dirname(os.path.realpath(__file__)), @@ -90,7 +90,6 @@ def _create_schema(self): with open(self.database_schema, 'r') as fp_schema: sql_statement = fp_schema.read() - print(sql_statement) self.cursor.executescript(sql_statement) def _insert_row_sql(self, final_path, metadata): @@ -103,11 +102,11 @@ def _insert_row_sql(self, final_path, metadata): """INSERT INTO `metadata` ( `hash`, `path`, `album`, `camera_make`, `camera_model`, `date_taken`, `latitude`, `location_name`, `longitude`, - `original_name`, `title`) + `original_name`, `title`, `_modified`) VALUES ( :hash, :path, :album, :camera_make, :camera_model, :date_taken, :latitude, :location_name, :longitude, - :original_name, :title)""", + :original_name, :title, datetime('now'))""", self._sql_values(final_path, metadata) ) @@ -123,14 +122,13 @@ def _sql_values(self, final_path, metadata, current_path=None): 'album': metadata['album'], 'camera_make': metadata['camera_make'], 'camera_model': metadata['camera_model'], - 'date_taken': metadata['date_taken'], + 'date_taken': time.strftime('%Y-%m-%d %H:%M:%S', metadata['date_taken']), 'latitude': metadata['latitude'], 'location_name': place_name(metadata['latitude'], metadata['longitude'])['default'], 'longitude': metadata['longitude'], 'original_name': metadata['original_name'], 'title': metadata['title'], - 'current_path': current_path, - '_modified': timestamp + 'current_path': current_path } def _update_row_sql(self, current_path, final_path, metadata): @@ -138,11 +136,26 @@ def _update_row_sql(self, current_path, final_path, metadata): return ( """UPDATE `metadata` SET `hash`=:hash, `path`=:path, `album`=:album, `camera_make`=:camera_make, `camera_model`=:camera_model, `date_taken`=:date_taken, `latitude`=:latitude, - `longitude`=:longitude, `original_name`=:original_name, `title`=:title + `longitude`=:longitude, `original_name`=:original_name, `title`=:title, `_modified`=datetime('now') WHERE `path`=:current_path""", self._sql_values(final_path, metadata, current_path) ) + def _upsert(self, file_path, destination_folder, final_file_path, metadata): + # We check if the source path exists in the database already. + # If it does then we assume that this is an update operation. + full_destination_path = '{}{}'.format(destination_folder, final_file_path) + self.cursor.execute("SELECT `path` FROM `metadata` WHERE `path`=:path", {'path': file_path}) + + if(self.cursor.fetchone() is None): + self.log(u'SQLite plugin inserting {}'.format(file_path)) + sql_statement, sql_values = self._insert_row_sql(full_destination_path, metadata) + else: + self.log(u'SQLite plugin updating {}'.format(file_path)) + sql_statement, sql_values = self._update_row_sql(file_path, full_destination_path, metadata) + + self.cursor.execute(sql_statement, sql_values) + def _validate_schema(self): try: self.cursor.execute('SELECT * FROM `metadata` LIMIT 1'); diff --git a/elodie/tests/plugins/sqlite/sqlite_test.py b/elodie/tests/plugins/sqlite/sqlite_test.py index 24fca68e..2175d27e 100644 --- a/elodie/tests/plugins/sqlite/sqlite_test.py +++ b/elodie/tests/plugins/sqlite/sqlite_test.py @@ -3,6 +3,7 @@ import mock import os import sys +import time from tempfile import gettempdir sys.path.insert(0, os.path.abspath(os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))))) @@ -10,9 +11,9 @@ import helper from elodie.config import load_config +from elodie.localstorage import Db +from elodie.media.text import Text from elodie.plugins.sqlite.sqlite import SQLite -from elodie.media.audio import Audio -from elodie.media.photo import Photo # Globals to simplify mocking configs db_schema = helper.get_file('plugins/sqlite/schema.sql') @@ -29,7 +30,7 @@ mock_metadata = { 'checksum': 'checksum-val', - 'date_taken': 1234567890, + 'date_taken': time.localtime(), 'camera_make': 'camera_make-val', 'camera_model': 'camera_model-val', 'latitude': 0.1, @@ -143,3 +144,30 @@ def test_sqlite_update_multiple(): assert results[0]['title'] == 'title-val-upd', results[0] assert results[1]['path'] == '/new-folder/path/new-file/path2.jpg', results[1] assert results[1]['title'] == 'title-val-2-upd', results[1] + +@mock.patch('elodie.config.config_file', '%s/config.ini-sqlite-regenerate-db' % gettempdir()) +def test_sqlite_regenerate_db(): + with open('%s/config.ini-sqlite-regenerate-db' % gettempdir(), 'w') as f: + f.write(config_string_fmt) + if hasattr(load_config, 'config'): + del load_config.config + + sqlite_plugin = SQLite() + db = Db() + file_path_1 = helper.get_file('with-original-name.txt') + file_path_2 = helper.get_file('valid.txt') + db.add_hash('1', file_path_1, True) + db.add_hash('2', file_path_2, True) + + sqlite_plugin.generate_db() + + results = sqlite_plugin._run_query( + 'SELECT * FROM `metadata`', + {} + ); + + assert len(results) == 2, results + assert results[0]['hash'] == 'e2275f3d95c4b55e35bd279bec3f86fcf76b3f3cc0abbf4183725c89a72f94c4', results[0]['hash'] + assert results[0]['path'] == file_path_1, results[0]['path'] + assert results[1]['hash'] == '3c19a5d751cf19e093b7447297731124d9cc987d3f91a9d1872c3b1c1b15639a', results[1]['hash'] + assert results[1]['path'] == file_path_2, results[1]['path'] From 6745798dbc45845bccf8aecb9549077ab2f1d449 Mon Sep 17 00:00:00 2001 From: Jaisen Mathai Date: Sun, 7 May 2023 00:28:55 -0700 Subject: [PATCH 7/7] Added placeholder Readme --- elodie/plugins/sqlite/Readme.markdown | 1 + 1 file changed, 1 insertion(+) create mode 100644 elodie/plugins/sqlite/Readme.markdown diff --git a/elodie/plugins/sqlite/Readme.markdown b/elodie/plugins/sqlite/Readme.markdown new file mode 100644 index 00000000..49602264 --- /dev/null +++ b/elodie/plugins/sqlite/Readme.markdown @@ -0,0 +1 @@ +# This is a beta plugin