Merge pull request #19 from aaronkollasch/dev

Updates for 0.0.7

.github/workflows/main.yml

@@ -31,6 +31,10 @@ jobs:
           python-version: "${{ matrix.python-version }}"
       - name: Install exiftool
         run: sudo apt-get install libimage-exiftool-perl
+      - name: "Install b3sum"
+        run: |
+          sudo wget https://github.com/BLAKE3-team/BLAKE3/releases/download/1.2.0/b3sum_linux_x64_bin -O /usr/local/bin/b3sum
+          sudo chmod +x /usr/local/bin/b3sum
       - name: "Install dependencies"
         run: |
           set -xe

CHANGES.rst

@@ -1,6 +1,14 @@
 Changelog for PhotoManager
 ==========================
 
+0.0.7 - 2021-12-07
+------------------
+
+Added
+^^^^^
+- Incremental indexing and importing with ``--skip-indexing`` (`c984ee7 <https://github.com/aaronkollasch/photomanager/commit/c984ee786cbe4c27cf6b0b12ed953a78b2bfd8dd>`_)
+- ``save()`` function to Database (`751f94b <https://github.com/aaronkollasch/photomanager/commit/751f94bef448291ada7c9cf2815d9828cf3d53d9>`_)
+
 0.0.6 - 2021-11-24
 ------------------
 

src/photomanager/actions/fileops.py

@@ -33,6 +33,7 @@ def list_files(
     file: Optional[Union[str, PathLike]] = None,
     paths: Iterable[Union[str, PathLike]] = tuple(),
     exclude: Iterable[str] = tuple(),
+    exclude_files: Iterable[Union[str, PathLike]] = tuple(),
 ) -> dict[str, None]:
     """
     List all files in sources, excluding regex patterns.
@@ -41,6 +42,7 @@ def list_files(
     :param file: File to list. If `-`, read files from stdin.
     :param paths: Paths (directories or files) to list.
     :param exclude: Regex patterns to exclude.
+    :param exclude_files: File paths to exclude
     :return: A dictionary with paths as keys.
     """
     logger = logging.getLogger(__name__)
@@ -73,10 +75,13 @@ def list_files(
         for p in path.glob("**/*.*"):
             files[p] = None
 
+    exclude_files = {Path(f).expanduser().resolve() for f in exclude_files}
     filtered_files = {}
     exclude_patterns = [re.compile(pat) for pat in set(exclude)]
     skipped_extensions = set()
     for p in files:
+        if p in exclude_files:
+            continue
         if p.suffix.lower().lstrip(".") not in extensions:
             skipped_extensions.add(p.suffix.lower().lstrip("."))
             continue

src/photomanager/cli.py

@@ -2,9 +2,7 @@
 from __future__ import annotations
 
 import sys
-from os import makedirs, PathLike
-from pathlib import Path
-import shlex
+from os import PathLike
 from typing import Union, Optional, Iterable
 import logging
 import click
@@ -63,10 +61,9 @@ def _create(
         database = Database.from_file(db)
     except FileNotFoundError:
         database = Database()
-        database.hash_algorithm = HashAlgorithm(hash_algorithm)
-        database.db["timezone_default"] = timezone_default
-        database.add_command("photomanager " + shlex.join(sys.argv[1:]))
-    database.to_file(db)
+    database.hash_algorithm = HashAlgorithm(hash_algorithm)
+    database.db["timezone_default"] = timezone_default
+    database.save(path=db, argv=sys.argv, force=True)
 
 
 # fmt: off
@@ -81,6 +78,8 @@ def _create(
               help="File to index")
 @click.option("--exclude", multiple=True,
               help="Name patterns to exclude")
+@click.option("--skip-existing", default=False, is_flag=True,
+              help="Don't index files that are already in the database")
 @click.option("--priority", type=int, default=10,
               help="Priority of indexed photos (lower is preferred, default=10)")
 @click.option("--timezone-default", type=str, default=None,
@@ -100,20 +99,26 @@ def _index(
     file: Optional[Union[str, PathLike]] = None,
     paths: Iterable[Union[str, PathLike]] = tuple(),
     exclude: Iterable[str] = tuple(),
-    debug=False,
-    dry_run=False,
-    priority=10,
+    skip_existing: bool = False,
+    debug: bool = False,
+    dry_run: bool = False,
+    priority: int = 10,
     timezone_default: Optional[str] = None,
-    storage_type="HDD",
+    storage_type: str = "HDD",
 ):
     if not source and not file and not paths:
         print("Nothing to index")
         print(click.get_current_context().get_help())
         click_exit(1)
     config_logging(debug=debug)
     database = Database.from_file(db, create_new=True)
+    skip_existing = set(database.sources) if skip_existing else set()
     filtered_files = fileops.list_files(
-        source=source, file=file, exclude=exclude, paths=paths
+        source=source,
+        file=file,
+        exclude=exclude,
+        exclude_files=skip_existing,
+        paths=paths,
     )
     index_result = actions.index(
         database=database,
@@ -122,9 +127,8 @@ def _index(
         timezone_default=timezone_default,
         storage_type=storage_type,
     )
-    database.add_command("photomanager " + shlex.join(sys.argv[1:]))
     if not dry_run:
-        database.to_file(db)
+        database.save(path=db, argv=sys.argv)
     click_exit(1 if index_result["num_error_photos"] else 0)
 
 
@@ -153,12 +157,10 @@ def _collect(
     collect_result = actions.collect(
         database=database, destination=destination, dry_run=dry_run
     )
-    database.add_command("photomanager " + shlex.join(sys.argv[1:]))
     if not dry_run:
-        database.to_file(db)
-        if collect_db:
-            makedirs(Path(destination) / "database", exist_ok=True)
-            database.to_file(Path(destination) / "database" / Path(db).name)
+        database.save(
+            path=db, argv=sys.argv, collect_db=collect_db, destination=destination
+        )
     click_exit(
         1
         if collect_result["num_missed_photos"] or collect_result["num_error_photos"]
@@ -180,6 +182,8 @@ def _collect(
               help="File to index")
 @click.option("--exclude", multiple=True,
               help="Name patterns to exclude")
+@click.option("--skip-existing", default=False, is_flag=True,
+              help="Don't index files that are already in the database")
 @click.option("--priority", type=int, default=10,
               help="Priority of indexed photos (lower is preferred, default=10)")
 @click.option("--timezone-default", type=str, default=None,
@@ -202,6 +206,7 @@ def _import(
     file: Optional[Union[str, PathLike]] = None,
     paths: Iterable[Union[str, PathLike]] = tuple(),
     exclude: Iterable[str] = tuple(),
+    skip_existing: bool = False,
     debug: bool = False,
     dry_run: bool = False,
     priority: int = 10,
@@ -211,8 +216,13 @@ def _import(
 ):
     config_logging(debug=debug)
     database = Database.from_file(db, create_new=True)
+    skip_existing = set(database.sources) if skip_existing else set()
     filtered_files = fileops.list_files(
-        source=source, file=file, exclude=exclude, paths=paths
+        source=source,
+        file=file,
+        exclude=exclude,
+        exclude_files=skip_existing,
+        paths=paths,
     )
     index_result = actions.index(
         database=database,
@@ -224,12 +234,10 @@ def _import(
     collect_result = actions.collect(
         database=database, destination=destination, dry_run=dry_run
     )
-    database.add_command("photomanager " + shlex.join(sys.argv[1:]))
     if not dry_run:
-        database.to_file(db)
-        if collect_db:
-            makedirs(Path(destination) / "database", exist_ok=True)
-            database.to_file(Path(destination) / "database" / Path(db).name)
+        database.save(
+            path=db, argv=sys.argv, collect_db=collect_db, destination=destination
+        )
     click_exit(
         1
         if index_result["num_error_photos"]
@@ -267,9 +275,8 @@ def _clean(
         subdir=subdir,
         dry_run=dry_run,
     )
-    database.add_command("photomanager " + shlex.join(sys.argv[1:]))
     if not dry_run:
-        database.to_file(db)
+        database.save(path=db, argv=sys.argv)
     click_exit(1 if result["num_missing_photos"] else 0)
 
 

src/photomanager/database.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from os import PathLike, rename, cpu_count
+from os import PathLike, rename, cpu_count, makedirs
 from os.path import exists
 from math import log
 import random
@@ -10,11 +10,13 @@
 import logging
 from typing import Union, Optional, Type, TypeVar
 from collections.abc import Iterable
+import shlex
 
 from tqdm import tqdm
 import orjson
 import zstandard as zstd
 import xxhash
+import blake3
 
 from photomanager import PhotoManagerBaseException
 from photomanager.hasher import (
@@ -99,10 +101,20 @@ def __init__(self):
         }
         self.hash_to_uid: dict[str, str] = {}
         self.timestamp_to_uids: dict[float, dict[str, None]] = {}
+        self._hash: int = hash(self)
 
     def __eq__(self, other: DB) -> bool:
         return self.db == other.db
 
+    def __hash__(self) -> int:
+        return hash(blake3.blake3(self.json, multithreading=True).digest())
+
+    def reset_saved(self):
+        self._hash = hash(self)
+
+    def is_modified(self) -> bool:
+        return self._hash != hash(self)
+
     @property
     def version(self) -> int:
         """Get the Database version number."""
@@ -137,6 +149,12 @@ def command_history(self) -> dict[str, str]:
         """Get the Database command history."""
         return self.db["command_history"]
 
+    @property
+    def sources(self) -> str:
+        for photos in self.photo_db.values():
+            for photo in photos:
+                yield photo.src
+
     @property
     def db(self) -> dict:
         """Get the Database parameters as a dict."""
@@ -176,6 +194,8 @@ def db(self, db: dict):
                 else:
                     self.timestamp_to_uids[photo.ts] = {uid: None}
 
+        self.reset_saved()
+
     @classmethod
     def from_dict(cls: Type[DB], db_dict: dict) -> DB:
         """Load a Database from a dictionary. Warning: can modify the dictionary."""
@@ -246,7 +266,7 @@ def from_file(cls: Type[DB], path: Union[str, PathLike], create_new=False) -> DB
         db = cls.from_dict(db)
         return db
 
-    def to_file(self, path: Union[str, PathLike], overwrite=False) -> None:
+    def to_file(self, path: Union[str, PathLike], overwrite: bool = False) -> None:
         """Save the Database to path.
 
         :param path: the destination path
@@ -318,6 +338,44 @@ def to_file(self, path: Union[str, PathLike], overwrite=False) -> None:
             with open(path, "wb") as f:
                 f.write(save_bytes)
 
+    def save(
+        self,
+        path: Union[str, PathLike],
+        argv: list[str],
+        overwrite: bool = False,
+        force: bool = False,
+        collect_db: bool = False,
+        destination: Optional[Union[str, PathLike]] = None,
+    ) -> bool:
+        """Save the database if it has been modified.
+
+        :param path: the destination path
+        :param overwrite: if false, do not overwrite an existing database at `path`
+            and instead rename the it based on its last modified timestamp.
+        :param argv: Add argv to the databases command_history
+        :param force: save even if not modified
+        :param collect_db: also collect the database to the storage destination
+        :param destination: the base storage directory for collect_db
+        :return True if save was successful
+        """
+        if force or self.is_modified():
+            self.add_command(shlex.join(["photomanager"] + argv[1:]))
+            try:
+                self.to_file(path, overwrite=overwrite)
+            except (OSError, PermissionError):  # pragma: no cover
+                return False
+            if collect_db and destination:
+                try:
+                    makedirs(Path(destination) / "database", exist_ok=True)
+                    self.to_file(Path(destination) / "database" / Path(path).name)
+                except (OSError, PermissionError):  # pragma: no cover
+                    return False
+            self.reset_saved()
+            return True
+        else:
+            logging.info("The database was not modified and will not be saved")
+            return False
+
     def add_command(self, command: str) -> str:
         """Adds a command to the command history.