csv import and export fixes

bookwyrm/importers/__init__.py

@@ -1,7 +1,7 @@
 """ import classes """
 
 from .importer import Importer
-from .bookwyrm_import import BookwyrmImporter
+from .bookwyrm_import import BookwyrmImporter, BookwyrmBooksImporter
 from .calibre_import import CalibreImporter
 from .goodreads_import import GoodreadsImporter
 from .librarything_import import LibrarythingImporter

bookwyrm/importers/bookwyrm_import.py

@@ -3,6 +3,7 @@
 
 from bookwyrm.models import User
 from bookwyrm.models.bookwyrm_import_job import BookwyrmImportJob
+from . import Importer
 
 
 class BookwyrmImporter:
@@ -22,3 +23,17 @@ def process_import(
             user=user, archive_file=archive_file, required=required
         )
         return job
+
+
+class BookwyrmBooksImporter(Importer):
+    """
+    Handle reading a csv from BookWyrm.
+    Goodreads is the default importer, we basically just use the same structure
+    But BookWyrm has additional attributes in the csv
+    """
+
+    service = "BookWyrm"
+    row_mappings_guesses = Importer.row_mappings_guesses + [
+        ("shelf_name", ["shelf_name"]),
+        ("review_published", ["review_published"]),
+    ]

bookwyrm/importers/importer.py

@@ -18,26 +18,41 @@ class Importer:
     row_mappings_guesses = [
         ("id", ["id", "book id"]),
         ("title", ["title"]),
-        ("authors", ["author", "authors", "primary author"]),
-        ("isbn_10", ["isbn10", "isbn", "isbn/uid"]),
-        ("isbn_13", ["isbn13", "isbn", "isbns", "isbn/uid"]),
+        ("authors", ["author_text", "author", "authors", "primary author"]),
+        ("isbn_10", ["isbn_10", "isbn10", "isbn", "isbn/uid"]),
+        ("isbn_13", ["isbn_13", "isbn13", "isbn", "isbns", "isbn/uid"]),
         ("shelf", ["shelf", "exclusive shelf", "read status", "bookshelf"]),
-        ("review_name", ["review name"]),
-        ("review_body", ["my review", "review"]),
+        ("review_name", ["review_name", "review name"]),
+        ("review_body", ["review_content", "my review", "review"]),
         ("rating", ["my rating", "rating", "star rating"]),
-        ("date_added", ["date added", "entry date", "added"]),
-        ("date_started", ["date started", "started"]),
-        ("date_finished", ["date finished", "last date read", "date read", "finished"]),
+        (
+            "date_added",
+            ["shelf_date", "date_added", "date added", "entry date", "added"],
+        ),
+        ("date_started", ["start_date", "date started", "started"]),
+        (
+            "date_finished",
+            ["finish_date", "date finished", "last date read", "date read", "finished"],
+        ),
     ]
+
+    # TODO: stopped
+
     date_fields = ["date_added", "date_started", "date_finished"]
     shelf_mapping_guesses = {
         "to-read": ["to-read", "want to read"],
         "read": ["read", "already read"],
         "reading": ["currently-reading", "reading", "currently reading"],
     }
 
+    # pylint: disable=too-many-arguments
     def create_job(
-        self, user: User, csv_file: Iterable[str], include_reviews: bool, privacy: str
+        self,
+        user: User,
+        csv_file: Iterable[str],
+        include_reviews: bool,
+        privacy: str,
+        create_shelves: bool = True,
     ) -> ImportJob:
         """check over a csv and creates a database entry for the job"""
         csv_reader = csv.DictReader(csv_file, delimiter=self.delimiter)
@@ -54,6 +69,7 @@ def create_job(
         job = ImportJob.objects.create(
             user=user,
             include_reviews=include_reviews,
+            create_shelves=create_shelves,
             privacy=privacy,
             mappings=mappings,
             source=self.service,
@@ -113,7 +129,7 @@ def get_shelf(self, normalized_row: dict[str, Optional[str]]) -> Optional[str]:
         shelf = [
             s for (s, gs) in self.shelf_mapping_guesses.items() if shelf_name in gs
         ]
-        return shelf[0] if shelf else None
+        return shelf[0] if shelf else normalized_row.get("shelf") or None
 
     # pylint: disable=no-self-use
     def normalize_row(
@@ -148,6 +164,7 @@ def create_retry_job(
         job = ImportJob.objects.create(
             user=user,
             include_reviews=original_job.include_reviews,
+            create_shelves=original_job.create_shelves,
             privacy=original_job.privacy,
             source=original_job.source,
             # TODO: allow users to adjust mappings

bookwyrm/migrations/0189_importjob_create_shelves.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.23 on 2023-11-25 05:49
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0188_theme_loads"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="importjob",
+            name="create_shelves",
+            field=models.BooleanField(default=True),
+        ),
+    ]

bookwyrm/migrations/0207_merge_20240629_0626.py

@@ -0,0 +1,13 @@
+# Generated by Django 4.2.11 on 2024-06-29 06:26
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0189_importjob_create_shelves"),
+        ("bookwyrm", "0206_merge_20240415_1537"),
+    ]
+
+    operations = []

bookwyrm/migrations/0208_merge_0207_merge_20240629_0626_0207_sqlparse_update.py

@@ -0,0 +1,13 @@
+# Generated by Django 4.2.11 on 2024-07-28 11:07
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0207_merge_20240629_0626"),
+        ("bookwyrm", "0207_sqlparse_update"),
+    ]
+
+    operations = []

bookwyrm/models/import_job.py

@@ -4,6 +4,7 @@
 import re
 import dateutil.parser
 
+from django.core.exceptions import ObjectDoesNotExist
 from django.db import models
 from django.utils import timezone
 from django.utils.translation import gettext_lazy as _
@@ -59,6 +60,7 @@ class ImportJob(models.Model):
     created_date = models.DateTimeField(default=timezone.now)
     updated_date = models.DateTimeField(default=timezone.now)
     include_reviews: bool = models.BooleanField(default=True)
+    create_shelves: bool = models.BooleanField(default=True)
     mappings = models.JSONField()
     source = models.CharField(max_length=100)
     privacy = models.CharField(max_length=255, default="public", choices=PrivacyLevels)
@@ -245,11 +247,26 @@ def shelf(self):
         """the goodreads shelf field"""
         return self.normalized_data.get("shelf")
 
+    @property
+    def shelf_name(self):
+        """the goodreads shelf field"""
+        return self.normalized_data.get("shelf_name")
+
     @property
     def review(self):
         """a user-written review, to be imported with the book data"""
         return self.normalized_data.get("review_body")
 
+    @property
+    def review_name(self):
+        """a user-written review name, to be imported with the book data"""
+        return self.normalized_data.get("review_name")
+
+    @property
+    def review_published(self):
+        """date the review was published - included in BookWyrm export csv"""
+        return self.normalized_data.get("review_published", None)
+
     @property
     def rating(self):
         """x/5 star rating for a book"""
@@ -368,7 +385,7 @@ def import_item_task(item_id):
     item.update_job()
 
 
-def handle_imported_book(item):
+def handle_imported_book(item):  # pylint: disable=too-many-branches
     """process a csv and then post about it"""
     job = item.job
     if job.complete:
@@ -385,13 +402,31 @@ def handle_imported_book(item):
         item.book = item.book.edition
 
     existing_shelf = ShelfBook.objects.filter(book=item.book, user=user).exists()
+    if job.create_shelves and item.shelf and not existing_shelf:
+        # shelve the book if it hasn't been shelved already
 
-    # shelve the book if it hasn't been shelved already
-    if item.shelf and not existing_shelf:
-        desired_shelf = Shelf.objects.get(identifier=item.shelf, user=user)
         shelved_date = item.date_added or timezone.now()
+        shelfname = getattr(item, "shelf_name", item.shelf)
+
+        try:
+            shelf = Shelf.objects.get(name=shelfname, user=user)
+        except ObjectDoesNotExist:
+            try:
+                shelf = Shelf.objects.get(identifier=item.shelf, user=user)
+            except ObjectDoesNotExist:
+
+                shelf = Shelf.objects.create(
+                    user=user,
+                    identifier=item.shelf,
+                    name=shelfname,
+                    privacy=job.privacy,
+                )
+
         ShelfBook(
-            book=item.book, shelf=desired_shelf, user=user, shelved_date=shelved_date
+            book=item.book,
+            shelf=shelf,
+            user=user,
+            shelved_date=shelved_date,
         ).save(priority=IMPORT_TRIGGERED)
 
     for read in item.reads:
@@ -408,27 +443,33 @@ def handle_imported_book(item):
         read.save()
 
     if job.include_reviews and (item.rating or item.review) and not item.linked_review:
-        # we don't know the publication date of the review,
-        # but "now" is a bad guess
-        published_date_guess = item.date_read or item.date_added
+        # we don't necessarily know the publication date of the review,
+        # but "now" is a bad guess unless we have no choice
+
+        published_date_guess = (
+            item.review_published or item.date_read or item.date_added or timezone.now()
+        )
         if item.review:
+
             # pylint: disable=consider-using-f-string
             review_title = "Review of {!r} on {!r}".format(
                 item.book.title,
                 job.source,
             )
+            review_name = getattr(item, "review_name", review_title)
+
             review = Review.objects.filter(
                 user=user,
                 book=item.book,
-                name=review_title,
+                name=review_name,
                 rating=item.rating,
                 published_date=published_date_guess,
             ).first()
             if not review:
                 review = Review(
                     user=user,
                     book=item.book,
-                    name=review_title,
+                    name=review_name,
                     content=item.review,
                     rating=item.rating,
                     published_date=published_date_guess,

bookwyrm/templates/import/import.html

@@ -69,6 +69,9 @@
                             <option value="Calibre" {% if current == 'Calibre' %}selected{% endif %}>
                                 {% trans "Calibre (CSV)" %}
                             </option>
+                            <option value="BookWyrm" {% if current == 'BookWyrm' %}selected{% endif %}>
+                                {% trans "BookWyrm (CSV)" %}
+                            </option>
                         </select>
                     </div>
 
@@ -93,9 +96,14 @@
                         <input type="checkbox" name="include_reviews" checked> {% trans "Include reviews" %}
                     </label>
                 </div>
+                <div class="field">
+                    <label class="label">
+                        <input type="checkbox" name="create_shelves" checked> {% trans "Create new shelves if they do not exist" %}
+                    </label>
+                </div>
                 <div class="field">
                     <label class="label" for="privacy_import">
-                        {% trans "Privacy setting for imported reviews:" %}
+                        {% trans "Privacy setting for imported reviews and shelves:" %}
                     </label>
                     {% include 'snippets/privacy_select.html' with no_label=True privacy_uuid="import" %}
                 </div>

bookwyrm/tests/data/bookwyrm.csv

@@ -0,0 +1,4 @@
+title,author_text,remote_id,openlibrary_key,inventaire_id,librarything_key,goodreads_key,bnf_id,viaf,wikidata,asin,aasin,isfdb,isbn_10,isbn_13,oclc_number,start_date,finish_date,stopped_date,rating,review_name,review_cw,review_content,review_published,shelf,shelf_name,shelf_date
+我穿我自己,琅俨,https://example.com/book/2010,,,,,,,,,,,,,,,,,,,,,,to-read,To Read,2024-08-10
+Ottolenghi Simple,Yotam Ottolenghi,https://example.com/book/2,OL43065148M,,,,,,,,,,0449017036,9780449017036,,2022-08-10,2022-10-10,,4,Too much tahini,,...in his hummus,2022-11-10,cooking-9,Cooking,2024-08-10
+The Blue Bedspread,Raj Kamal Jha,https://example.com/book/270,OL7425890M,,,,,,,,,,0375503129,9780375503122,41754476,2001-06-01,2001-07-10,,5,,,,,read,Read,2024-08-10