Store metadata in an SQLite database (gh-68)

elodie.py

@@ -174,6 +174,10 @@ def _generate_db(source, debug):
         log.progress()
 
     db.update_hash_db()
+
+    plugins = Plugins()
+    plugins.run_generate_db()
+
     log.progress('', True)
     result.write()
 

elodie/filesystem.py

@@ -533,6 +533,10 @@ def process_file(self, _file, destination, media, **kwargs):
                      _file)
             return
 
+        # Set the checksum and refresh metadata.
+        media.set_checksum(checksum)
+        metadata = media.get_metadata()
+
         # Run `before()` for every loaded plugin and if any of them raise an exception
         #  then we skip importing the file and log a message.
         plugins_run_before_status = self.plugins.run_all_before(_file, destination)

elodie/media/base.py

@@ -74,6 +74,10 @@ def get_camera_make(self):
     def get_camera_model(self):
         return None
 
+    def get_checksum(self):
+        self.get_metadata()
+        return self.metadata['checksum']
+
     def get_metadata(self, update_cache=False):
         """Get a dictionary of metadata for any file.
 
@@ -90,6 +94,7 @@ def get_metadata(self, update_cache=False):
         source = self.source
 
         self.metadata = {
+            'checksum': None,
             'date_taken': self.get_date_taken(),
             'camera_make': self.get_camera_make(),
             'camera_model': self.get_camera_model(),
@@ -178,6 +183,16 @@ def set_album_from_folder(self):
         self.set_album(folder)
         return True
 
+    def set_checksum(self, new_checksum):
+        """Add the checksum to the metadata.
+
+        The checksum in `metadata` is passed in and not derived from the file itself.
+        It was added to pass in to plugins.
+        See gh-68.
+        """
+        self.get_metadata()
+        self.metadata['checksum'] = new_checksum
+
     def set_metadata_basename(self, new_basename):
         """Update the basename attribute in the metadata dict for this instance.
 

elodie/plugins/googlephotos/Readme.markdown

@@ -1,7 +1,5 @@
 # Google Photos Plugin for Elodie
 
-[![Build Status](https://travis-ci.org/jmathai/elodie.svg?branch=master)](https://travis-ci.org/jmathai/elodie) [![Coverage Status](https://coveralls.io/repos/github/jmathai/elodie/badge.svg?branch=master)](https://coveralls.io/github/jmathai/elodie?branch=master) [![Scrutinizer Code Quality](https://scrutinizer-ci.com/g/jmathai/elodie/badges/quality-score.png?b=master)](https://scrutinizer-ci.com/g/jmathai/elodie/?branch=master)
-
 This plugin uploads all photos imported using Elodie to Google Photos. It was created after [Google Photos and Google Drive synchronization was deprecated](https://www.blog.google/products/photos/simplifying-google-photos-and-google-drive/). It aims to replicate my [workflow using Google Photos, Google Drive and Elodie](https://artplusmarketing.com/one-year-of-using-an-automated-photo-organization-and-archiving-workflow-89cf9ad7bddf).
 
 I didn't intend on it, but it turned out that with this plugin you can use Google Photos with Google Drive, iCloud Drive, Dropbox or no cloud storage service while still using Google Photos for viewing and experiencing your photo library.

elodie/plugins/plugins.py

@@ -34,8 +34,11 @@ class PluginBase(object):
     __name__ = 'PluginBase'
 
     def __init__(self):
+        # Initializes variables for the plugin
+        self.application_directory = application_directory
         # Loads the config for the plugin from config.ini
         self.config_for_plugin = load_config_for_plugin(self.__name__)
+        # Initializes a database for the plugin.
         self.db = PluginDb(self.__name__)
 
     def after(self, file_path, destination_folder, final_file_path, metadata):
@@ -59,6 +62,9 @@ def display(self, msg):
             {self.__name__: msg}
         ))
 
+    def generate_db(self, hash_db):
+        pass
+
 class PluginDb(object):
     """A database module which provides a simple key/value database.
        The database is a JSON file located at %application_directory%/plugins/%pluginname.lower()%.json

elodie/plugins/sqlite/Readme.markdown

@@ -0,0 +1 @@
+# This is a beta plugin

elodie/plugins/sqlite/schema.sql

@@ -0,0 +1,16 @@
+CREATE TABLE "metadata" (
+	"id"	INTEGER NOT NULL,
+	"hash"	TEXT NOT NULL UNIQUE,
+	"path"	TEXT NOT NULL UNIQUE,
+	"album"	INTEGER,
+	"camera_make"	TEXT,
+	"camera_model"	TEXT,
+	"date_taken"	TEXT,
+	"latitude"	REAL,
+	"location_name"	TEXT,
+	"longitude"	REAL,
+	"original_name"	TEXT,
+	"title"	TEXT,
+	"_modified"	TEXT,
+	PRIMARY KEY("id" AUTOINCREMENT)
+);

elodie/plugins/sqlite/sqlite.py

@@ -0,0 +1,164 @@
+"""
+SQLite plugin object.
+This plugin stores metadata about all media in an sqlite database.
+
+You'll need to include [PluginSQLite] in your config file.
+By default, the sqlite database will be created in your application 
+    directory. If you want to specify a different path then 
+    specify a fully qualified `database_file` path.
+
+```
+[PluginSQLite]
+database_file=/path/to/database.db
+
+.. moduleauthor:: Jaisen Mathai <[email protected]>
+"""
+from __future__ import print_function
+
+import json
+import os
+import sqlite3
+import time
+#import json
+
+#from google_auth_oauthlib.flow import InstalledAppFlow
+#from google.auth.transport.requests import AuthorizedSession
+#from google.oauth2.credentials import Credentials
+
+from elodie.geolocation import place_name
+from elodie.localstorage import Db
+from elodie.media.base import Base, get_all_subclasses
+from elodie.plugins.plugins import PluginBase
+
+class SQLite(PluginBase):
+    """A class to execute plugin actions.
+
+       Requires a config file with the following configurations set.
+       database_file:
+            The full path to the SQLite database (.db).
+
+    """
+
+    __name__ = 'SQLite'
+
+    def __init__(self):
+        super(SQLite, self).__init__()
+
+        # Default the database file to be in the application plugin directory.
+        # Override with value from config file.
+        self.database_file = '{}/plugins/{}/elodie.db'.format(
+            self.application_directory,
+            __name__.lower()
+        )
+        if('database_file' in self.config_for_plugin):
+            self.database_file = self.config_for_plugin['database_file']
+
+        self.con = sqlite3.connect(self.database_file)
+        self.con.row_factory = sqlite3.Row
+        self.cursor = self.con.cursor()
+        if(not self._validate_schema()):
+            self._create_schema()
+
+    def after(self, file_path, destination_folder, final_file_path, metadata):
+        self._upsert(file_path, destination_folder, final_file_path, metadata)
+
+    def batch(self):
+        pass
+
+    def before(self, file_path, destination_folder):
+        pass
+
+    def generate_db(self):
+        db = Db()
+        for checksum, file_path in db.all():
+            subclasses = get_all_subclasses()
+            media = Base.get_class_by_file(file_path, get_all_subclasses())
+            media.set_checksum(
+                db.checksum(file_path)
+            )
+            metadata = media.get_metadata()
+            destination_folder = os.path.dirname(file_path)
+            final_file_path = '{}{}'.format(os.path.sep, os.path.basename(file_path))
+            self._upsert(file_path, destination_folder, final_file_path, metadata)
+
+    def _create_schema(self):
+        self.database_schema = '{}{}{}'.format(
+            os.path.dirname(os.path.realpath(__file__)),
+            os.sep,
+            'schema.sql'
+        )
+
+        with open(self.database_schema, 'r') as fp_schema:
+            sql_statement = fp_schema.read()
+            self.cursor.executescript(sql_statement)
+
+    def _insert_row_sql(self, final_path, metadata):
+        path = '{}/{}.{}'.format(
+                    metadata['directory_path'],
+                    metadata['base_name'],
+                    metadata['extension']
+                )
+        return (
+                """INSERT INTO `metadata` (
+                    `hash`, `path`, `album`, `camera_make`, `camera_model`,
+                    `date_taken`, `latitude`, `location_name`, `longitude`,
+                    `original_name`, `title`, `_modified`)
+                    VALUES (
+                    :hash, :path, :album, :camera_make, :camera_model,
+                    :date_taken, :latitude, :location_name, :longitude,
+                    :original_name, :title, datetime('now'))""",
+                self._sql_values(final_path, metadata)
+        )
+
+    def _run_query(self, sql, values):
+        self.cursor.execute(sql, values)
+        return self.cursor.fetchall()
+
+    def _sql_values(self, final_path, metadata, current_path=None):
+        timestamp = int(time.time())
+        return {
+                    'hash': metadata['checksum'],
+                    'path': final_path,
+                    'album': metadata['album'],
+                    'camera_make': metadata['camera_make'],
+                    'camera_model': metadata['camera_model'],
+                    'date_taken': time.strftime('%Y-%m-%d %H:%M:%S', metadata['date_taken']),
+                    'latitude': metadata['latitude'],
+                    'location_name': place_name(metadata['latitude'], metadata['longitude'])['default'],
+                    'longitude': metadata['longitude'],
+                    'original_name': metadata['original_name'],
+                    'title': metadata['title'],
+                    'current_path': current_path
+                }
+
+    def _update_row_sql(self, current_path, final_path, metadata):
+        timestamp = int(time.time())
+        return (
+            """UPDATE `metadata` SET `hash`=:hash, `path`=:path, `album`=:album, `camera_make`=:camera_make,
+                `camera_model`=:camera_model, `date_taken`=:date_taken, `latitude`=:latitude,
+                `longitude`=:longitude, `original_name`=:original_name, `title`=:title, `_modified`=datetime('now')
+                WHERE `path`=:current_path""",
+            self._sql_values(final_path, metadata, current_path)
+        )
+
+    def _upsert(self, file_path, destination_folder, final_file_path, metadata):
+        # We check if the source path exists in the database already.
+        # If it does then we assume that this is an update operation.
+        full_destination_path = '{}{}'.format(destination_folder, final_file_path)
+        self.cursor.execute("SELECT `path` FROM `metadata` WHERE `path`=:path", {'path': file_path})
+
+        if(self.cursor.fetchone() is None):
+            self.log(u'SQLite plugin inserting {}'.format(file_path))
+            sql_statement, sql_values = self._insert_row_sql(full_destination_path, metadata)
+        else:
+            self.log(u'SQLite plugin updating {}'.format(file_path))
+            sql_statement, sql_values = self._update_row_sql(file_path, full_destination_path, metadata)
+
+        self.cursor.execute(sql_statement, sql_values)
+
+    def _validate_schema(self):
+        try:
+            self.cursor.execute('SELECT * FROM `metadata` LIMIT 1');
+            return True
+        except sqlite3.OperationalError:
+            return False

elodie/tests/media/base_test.py

@@ -38,6 +38,19 @@ def test_get_class_by_file_without_extension():
 
     assert cls is None, cls
 
+def test_get_checksum():
+    temporary_folder, folder = helper.create_working_folder()
+
+    origin = '%s/%s' % (folder, 'with-original-name.jpg')
+    file = helper.get_file('with-original-name.jpg')
+
+    shutil.copyfile(file, origin)
+
+    media = Media.get_class_by_file(origin, [Photo])
+    checksum = media.get_checksum()
+
+    assert checksum == None, checksum
+
 def test_get_original_name():
     temporary_folder, folder = helper.create_working_folder()
 
@@ -101,6 +114,20 @@ def test_set_album_from_folder():
 
     assert metadata_new['album'] == new_album_name, metadata_new['album']
 
+def test_get_checksum():
+    temporary_folder, folder = helper.create_working_folder()
+
+    origin = '%s/%s' % (folder, 'with-original-name.jpg')
+    file = helper.get_file('with-original-name.jpg')
+
+    shutil.copyfile(file, origin)
+
+    media = Media.get_class_by_file(origin, [Photo])
+    media.set_checksum('foo')
+    checksum = media.get_checksum()
+
+    assert checksum == 'foo', checksum
+
 def test_set_metadata():
     temporary_folder, folder = helper.create_working_folder()