From 1d880efedefceddf8745b10aaf5bd1ea427be1a4 Mon Sep 17 00:00:00 2001 From: WithoutPants <53250216+WithoutPants@users.noreply.github.com> Date: Tue, 12 Sep 2023 13:31:53 +1000 Subject: [PATCH] Multiple image URLs (#4000) * Backend changes - ported from scene impl * Front end changes * Refactor URL mutation code --- graphql/documents/data/image-slim.graphql | 2 +- graphql/documents/data/image.graphql | 2 +- graphql/schema/types/image.graphql | 9 +- graphql/schema/types/scene.graphql | 2 +- internal/api/changeset_translator.go | 40 +++++++++ internal/api/resolver_model_image.go | 29 +++++++ internal/api/resolver_mutation_image.go | 6 +- internal/api/resolver_mutation_scene.go | 22 +---- internal/manager/task_export.go | 5 ++ pkg/image/export.go | 15 +--- pkg/image/export_test.go | 4 +- pkg/image/import.go | 9 +- pkg/models/jsonschema/image.go | 12 ++- pkg/models/jsonschema/scene.go | 4 +- pkg/models/mocks/ImageReaderWriter.go | 23 +++++ pkg/models/model_image.go | 20 +++-- pkg/models/repository_image.go | 1 + pkg/sqlite/anonymise.go | 8 +- pkg/sqlite/database.go | 2 +- pkg/sqlite/image.go | 57 ++++++++++--- pkg/sqlite/image_test.go | 85 +++++++++++++++++-- pkg/sqlite/migrations/50_image_urls.up.sql | 70 +++++++++++++++ pkg/sqlite/setup_test.go | 23 ++++- pkg/sqlite/tables.go | 9 ++ .../Images/ImageDetails/ImageEditPanel.tsx | 45 +++++----- .../ImageDetails/ImageFileInfoPanel.tsx | 19 ++--- .../Scenes/SceneDetails/SceneEditPanel.tsx | 35 +------- ui/v2.5/src/components/Shared/URLField.tsx | 32 ++++--- ui/v2.5/src/utils/yup.ts | 40 +++++++++ 29 files changed, 457 insertions(+), 173 deletions(-) create mode 100644 pkg/sqlite/migrations/50_image_urls.up.sql create mode 100644 ui/v2.5/src/utils/yup.ts diff --git a/graphql/documents/data/image-slim.graphql b/graphql/documents/data/image-slim.graphql index 9f84904dcfe..1c7784c9ede 100644 --- a/graphql/documents/data/image-slim.graphql +++ b/graphql/documents/data/image-slim.graphql @@ -2,7 +2,7 @@ fragment SlimImageData on Image { id title date - url + urls rating100 organized o_counter diff --git a/graphql/documents/data/image.graphql b/graphql/documents/data/image.graphql index d55a8108121..64c801401e7 100644 --- a/graphql/documents/data/image.graphql +++ b/graphql/documents/data/image.graphql @@ -3,7 +3,7 @@ fragment ImageData on Image { title rating100 date - url + urls organized o_counter created_at diff --git a/graphql/schema/types/image.graphql b/graphql/schema/types/image.graphql index 5d13cbdd6e4..f0307b962ae 100644 --- a/graphql/schema/types/image.graphql +++ b/graphql/schema/types/image.graphql @@ -6,7 +6,8 @@ type Image { rating: Int @deprecated(reason: "Use 1-100 range with rating100") # rating expressed as 1-100 rating100: Int - url: String + url: String @deprecated(reason: "Use urls") + urls: [String!]! date: String o_counter: Int organized: Boolean! @@ -48,7 +49,8 @@ input ImageUpdateInput { # rating expressed as 1-100 rating100: Int organized: Boolean - url: String + url: String @deprecated(reason: "Use urls") + urls: [String!] date: String studio_id: ID @@ -68,7 +70,8 @@ input BulkImageUpdateInput { # rating expressed as 1-100 rating100: Int organized: Boolean - url: String + url: String @deprecated(reason: "Use urls") + urls: BulkUpdateStrings date: String studio_id: ID diff --git a/graphql/schema/types/scene.graphql b/graphql/schema/types/scene.graphql index cb0831b0aea..2a8b1ddf5f2 100644 --- a/graphql/schema/types/scene.graphql +++ b/graphql/schema/types/scene.graphql @@ -41,7 +41,7 @@ type Scene { details: String director: String url: String @deprecated(reason: "Use urls") - urls: [String!] + urls: [String!]! date: String # rating expressed as 1-5 rating: Int @deprecated(reason: "Use 1-100 range with rating100") diff --git a/internal/api/changeset_translator.go b/internal/api/changeset_translator.go index 6d2590a3da4..0472edc4c76 100644 --- a/internal/api/changeset_translator.go +++ b/internal/api/changeset_translator.go @@ -307,6 +307,46 @@ func (t changesetTranslator) updateIdsBulk(value *BulkUpdateIds, field string) ( }, nil } +func (t changesetTranslator) optionalURLs(value []string, legacyValue *string) *models.UpdateStrings { + const ( + legacyField = "url" + field = "urls" + ) + + // prefer urls over url + if t.hasField(field) { + return t.updateStrings(value, field) + } else if t.hasField(legacyField) { + var valueSlice []string + if legacyValue != nil { + valueSlice = []string{*legacyValue} + } + return t.updateStrings(valueSlice, legacyField) + } + + return nil +} + +func (t changesetTranslator) optionalURLsBulk(value *BulkUpdateStrings, legacyValue *string) *models.UpdateStrings { + const ( + legacyField = "url" + field = "urls" + ) + + // prefer urls over url + if t.hasField("urls") { + return t.updateStringsBulk(value, field) + } else if t.hasField(legacyField) { + var valueSlice []string + if legacyValue != nil { + valueSlice = []string{*legacyValue} + } + return t.updateStrings(valueSlice, legacyField) + } + + return nil +} + func (t changesetTranslator) updateStrings(value []string, field string) *models.UpdateStrings { if !t.hasField(field) { return nil diff --git a/internal/api/resolver_model_image.go b/internal/api/resolver_model_image.go index 246ff8b4450..f4e699b7ba6 100644 --- a/internal/api/resolver_model_image.go +++ b/internal/api/resolver_model_image.go @@ -198,3 +198,32 @@ func (r *imageResolver) Performers(ctx context.Context, obj *models.Image) (ret ret, errs = loaders.From(ctx).PerformerByID.LoadAll(obj.PerformerIDs.List()) return ret, firstError(errs) } + +func (r *imageResolver) URL(ctx context.Context, obj *models.Image) (*string, error) { + if !obj.URLs.Loaded() { + if err := r.withReadTxn(ctx, func(ctx context.Context) error { + return obj.LoadURLs(ctx, r.repository.Image) + }); err != nil { + return nil, err + } + } + + urls := obj.URLs.List() + if len(urls) == 0 { + return nil, nil + } + + return &urls[0], nil +} + +func (r *imageResolver) Urls(ctx context.Context, obj *models.Image) ([]string, error) { + if !obj.URLs.Loaded() { + if err := r.withReadTxn(ctx, func(ctx context.Context) error { + return obj.LoadURLs(ctx, r.repository.Image) + }); err != nil { + return nil, err + } + } + + return obj.URLs.List(), nil +} diff --git a/internal/api/resolver_mutation_image.go b/internal/api/resolver_mutation_image.go index d6ac1f8cd8f..8b2cf447831 100644 --- a/internal/api/resolver_mutation_image.go +++ b/internal/api/resolver_mutation_image.go @@ -108,7 +108,6 @@ func (r *mutationResolver) imageUpdate(ctx context.Context, input ImageUpdateInp updatedImage.Title = translator.optionalString(input.Title, "title") updatedImage.Rating = translator.optionalRatingConversion(input.Rating, input.Rating100) - updatedImage.URL = translator.optionalString(input.URL, "url") updatedImage.Organized = translator.optionalBool(input.Organized, "organized") updatedImage.Date, err = translator.optionalDate(input.Date, "date") @@ -120,6 +119,8 @@ func (r *mutationResolver) imageUpdate(ctx context.Context, input ImageUpdateInp return nil, fmt.Errorf("converting studio id: %w", err) } + updatedImage.URLs = translator.optionalURLs(input.Urls, input.URL) + updatedImage.PrimaryFileID, err = translator.fileIDPtrFromString(input.PrimaryFileID) if err != nil { return nil, fmt.Errorf("converting primary file id: %w", err) @@ -203,7 +204,6 @@ func (r *mutationResolver) BulkImageUpdate(ctx context.Context, input BulkImageU updatedImage.Title = translator.optionalString(input.Title, "title") updatedImage.Rating = translator.optionalRatingConversion(input.Rating, input.Rating100) - updatedImage.URL = translator.optionalString(input.URL, "url") updatedImage.Organized = translator.optionalBool(input.Organized, "organized") updatedImage.Date, err = translator.optionalDate(input.Date, "date") @@ -215,6 +215,8 @@ func (r *mutationResolver) BulkImageUpdate(ctx context.Context, input BulkImageU return nil, fmt.Errorf("converting studio id: %w", err) } + updatedImage.URLs = translator.optionalURLsBulk(input.Urls, input.URL) + updatedImage.GalleryIDs, err = translator.updateIdsBulk(input.GalleryIds, "gallery_ids") if err != nil { return nil, fmt.Errorf("converting gallery ids: %w", err) diff --git a/internal/api/resolver_mutation_scene.go b/internal/api/resolver_mutation_scene.go index 7652827f33f..782d0aff0af 100644 --- a/internal/api/resolver_mutation_scene.go +++ b/internal/api/resolver_mutation_scene.go @@ -186,16 +186,7 @@ func scenePartialFromInput(input models.SceneUpdateInput, translator changesetTr return nil, fmt.Errorf("converting studio id: %w", err) } - // prefer urls over url - if translator.hasField("urls") { - updatedScene.URLs = translator.updateStrings(input.Urls, "urls") - } else if translator.hasField("url") { - var urls []string - if input.URL != nil { - urls = []string{*input.URL} - } - updatedScene.URLs = translator.updateStrings(urls, "url") - } + updatedScene.URLs = translator.optionalURLs(input.Urls, input.URL) updatedScene.PrimaryFileID, err = translator.fileIDPtrFromString(input.PrimaryFileID) if err != nil { @@ -342,16 +333,7 @@ func (r *mutationResolver) BulkSceneUpdate(ctx context.Context, input BulkSceneU return nil, fmt.Errorf("converting studio id: %w", err) } - // prefer urls over url - if translator.hasField("urls") { - updatedScene.URLs = translator.updateStringsBulk(input.Urls, "urls") - } else if translator.hasField("url") { - var urls []string - if input.URL != nil { - urls = []string{*input.URL} - } - updatedScene.URLs = translator.updateStrings(urls, "url") - } + updatedScene.URLs = translator.optionalURLsBulk(input.Urls, input.URL) updatedScene.PerformerIDs, err = translator.updateIdsBulk(input.PerformerIds, "performer_ids") if err != nil { diff --git a/internal/manager/task_export.go b/internal/manager/task_export.go index 98ae1918f31..a7278253ecc 100644 --- a/internal/manager/task_export.go +++ b/internal/manager/task_export.go @@ -647,6 +647,11 @@ func exportImage(ctx context.Context, wg *sync.WaitGroup, jobChan <-chan *models continue } + if err := s.LoadURLs(ctx, repo.Image); err != nil { + logger.Errorf("[images] <%s> error getting image urls: %s", imageHash, err.Error()) + continue + } + newImageJSON := image.ToBasicJSON(s) // export files diff --git a/pkg/image/export.go b/pkg/image/export.go index a7c4d8575eb..41eac446fe2 100644 --- a/pkg/image/export.go +++ b/pkg/image/export.go @@ -14,7 +14,7 @@ import ( func ToBasicJSON(image *models.Image) *jsonschema.Image { newImageJSON := jsonschema.Image{ Title: image.Title, - URL: image.URL, + URLs: image.URLs.List(), CreatedAt: json.JSONTime{Time: image.CreatedAt}, UpdatedAt: json.JSONTime{Time: image.UpdatedAt}, } @@ -37,19 +37,6 @@ func ToBasicJSON(image *models.Image) *jsonschema.Image { return &newImageJSON } -// func getImageFileJSON(image *models.Image) *jsonschema.ImageFile { -// ret := &jsonschema.ImageFile{} - -// f := image.PrimaryFile() - -// ret.ModTime = json.JSONTime{Time: f.ModTime} -// ret.Size = f.Size -// ret.Width = f.Width -// ret.Height = f.Height - -// return ret -// } - // GetStudioName returns the name of the provided image's studio. It returns an // empty string if there is no studio assigned to the image. func GetStudioName(ctx context.Context, reader models.StudioGetter, image *models.Image) (string, error) { diff --git a/pkg/image/export_test.go b/pkg/image/export_test.go index 3b64f40cbcd..1a5897271ef 100644 --- a/pkg/image/export_test.go +++ b/pkg/image/export_test.go @@ -53,7 +53,7 @@ func createFullImage(id int) models.Image { OCounter: ocounter, Rating: &rating, Date: &dateObj, - URL: url, + URLs: models.NewRelatedStrings([]string{url}), Organized: organized, CreatedAt: createTime, UpdatedAt: updateTime, @@ -66,7 +66,7 @@ func createFullJSONImage() *jsonschema.Image { OCounter: ocounter, Rating: rating, Date: date, - URL: url, + URLs: []string{url}, Organized: organized, Files: []string{path}, CreatedAt: json.JSONTime{ diff --git a/pkg/image/import.go b/pkg/image/import.go index d5ad591a997..8b90fa8a7c6 100644 --- a/pkg/image/import.go +++ b/pkg/image/import.go @@ -62,8 +62,6 @@ func (i *Importer) PreImport(ctx context.Context) error { func (i *Importer) imageJSONToImage(imageJSON jsonschema.Image) models.Image { newImage := models.Image{ - // Checksum: imageJSON.Checksum, - // Path: i.Path, PerformerIDs: models.NewRelatedIDs([]int{}), TagIDs: models.NewRelatedIDs([]int{}), GalleryIDs: models.NewRelatedIDs([]int{}), @@ -81,9 +79,12 @@ func (i *Importer) imageJSONToImage(imageJSON jsonschema.Image) models.Image { if imageJSON.Rating != 0 { newImage.Rating = &imageJSON.Rating } - if imageJSON.URL != "" { - newImage.URL = imageJSON.URL + if len(imageJSON.URLs) > 0 { + newImage.URLs = models.NewRelatedStrings(imageJSON.URLs) + } else if imageJSON.URL != "" { + newImage.URLs = models.NewRelatedStrings([]string{imageJSON.URL}) } + if imageJSON.Date != "" { d, err := models.ParseDate(imageJSON.Date) if err == nil { diff --git a/pkg/models/jsonschema/image.go b/pkg/models/jsonschema/image.go index 1862ffc8290..7ff0b21621f 100644 --- a/pkg/models/jsonschema/image.go +++ b/pkg/models/jsonschema/image.go @@ -10,10 +10,14 @@ import ( ) type Image struct { - Title string `json:"title,omitempty"` - Studio string `json:"studio,omitempty"` - Rating int `json:"rating,omitempty"` - URL string `json:"url,omitempty"` + Title string `json:"title,omitempty"` + Studio string `json:"studio,omitempty"` + Rating int `json:"rating,omitempty"` + + // deprecated - for import only + URL string `json:"url,omitempty"` + + URLs []string `json:"urls,omitempty"` Date string `json:"date,omitempty"` Organized bool `json:"organized,omitempty"` OCounter int `json:"o_counter,omitempty"` diff --git a/pkg/models/jsonschema/scene.go b/pkg/models/jsonschema/scene.go index 7ebae7a1785..8a081f3b610 100644 --- a/pkg/models/jsonschema/scene.go +++ b/pkg/models/jsonschema/scene.go @@ -42,8 +42,10 @@ type Scene struct { Title string `json:"title,omitempty"` Code string `json:"code,omitempty"` Studio string `json:"studio,omitempty"` + // deprecated - for import only - URL string `json:"url,omitempty"` + URL string `json:"url,omitempty"` + URLs []string `json:"urls,omitempty"` Date string `json:"date,omitempty"` Rating int `json:"rating,omitempty"` diff --git a/pkg/models/mocks/ImageReaderWriter.go b/pkg/models/mocks/ImageReaderWriter.go index a3ec6987792..bd651108ab0 100644 --- a/pkg/models/mocks/ImageReaderWriter.go +++ b/pkg/models/mocks/ImageReaderWriter.go @@ -462,6 +462,29 @@ func (_m *ImageReaderWriter) GetTagIDs(ctx context.Context, relatedID int) ([]in return r0, r1 } +// GetURLs provides a mock function with given fields: ctx, relatedID +func (_m *ImageReaderWriter) GetURLs(ctx context.Context, relatedID int) ([]string, error) { + ret := _m.Called(ctx, relatedID) + + var r0 []string + if rf, ok := ret.Get(0).(func(context.Context, int) []string); ok { + r0 = rf(ctx, relatedID) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]string) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, int) error); ok { + r1 = rf(ctx, relatedID) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // IncrementOCounter provides a mock function with given fields: ctx, id func (_m *ImageReaderWriter) IncrementOCounter(ctx context.Context, id int) (int, error) { ret := _m.Called(ctx, id) diff --git a/pkg/models/model_image.go b/pkg/models/model_image.go index 797f638ad35..8f3211dc7ac 100644 --- a/pkg/models/model_image.go +++ b/pkg/models/model_image.go @@ -13,12 +13,12 @@ type Image struct { Title string `json:"title"` // Rating expressed in 1-100 scale - Rating *int `json:"rating"` - Organized bool `json:"organized"` - OCounter int `json:"o_counter"` - StudioID *int `json:"studio_id"` - URL string `json:"url"` - Date *Date `json:"date"` + Rating *int `json:"rating"` + Organized bool `json:"organized"` + OCounter int `json:"o_counter"` + StudioID *int `json:"studio_id"` + URLs RelatedStrings `json:"urls"` + Date *Date `json:"date"` // transient - not persisted Files RelatedFiles @@ -48,7 +48,7 @@ type ImagePartial struct { Title OptionalString // Rating expressed in 1-100 scale Rating OptionalInt - URL OptionalString + URLs *UpdateStrings Date OptionalDate Organized OptionalBool OCounter OptionalInt @@ -69,6 +69,12 @@ func NewImagePartial() ImagePartial { } } +func (i *Image) LoadURLs(ctx context.Context, l URLLoader) error { + return i.URLs.load(func() ([]string, error) { + return l.GetURLs(ctx, i.ID) + }) +} + func (i *Image) LoadFiles(ctx context.Context, l FileLoader) error { return i.Files.load(func() ([]File, error) { return l.GetFiles(ctx, i.ID) diff --git a/pkg/models/repository_image.go b/pkg/models/repository_image.go index b6eb895a06b..1bf8ba440de 100644 --- a/pkg/models/repository_image.go +++ b/pkg/models/repository_image.go @@ -63,6 +63,7 @@ type ImageReader interface { ImageQueryer ImageCounter + URLLoader FileIDLoader GalleryIDLoader PerformerIDLoader diff --git a/pkg/sqlite/anonymise.go b/pkg/sqlite/anonymise.go index d8e6d99d6ad..7e4efd70299 100644 --- a/pkg/sqlite/anonymise.go +++ b/pkg/sqlite/anonymise.go @@ -368,7 +368,6 @@ func (db *Anonymiser) anonymiseImages(ctx context.Context) error { query := dialect.From(table).Select( table.Col(idColumn), table.Col("title"), - table.Col("url"), ).Where(table.Col(idColumn).Gt(lastID)).Limit(1000) gotSome = false @@ -378,20 +377,17 @@ func (db *Anonymiser) anonymiseImages(ctx context.Context) error { var ( id int title sql.NullString - url sql.NullString ) if err := rows.Scan( &id, &title, - &url, ); err != nil { return err } set := goqu.Record{} db.obfuscateNullString(set, "title", title) - db.obfuscateNullString(set, "url", url) if len(set) > 0 { stmt := dialect.Update(table).Set(set).Where(table.Col(idColumn).Eq(id)) @@ -416,6 +412,10 @@ func (db *Anonymiser) anonymiseImages(ctx context.Context) error { } } + if err := db.anonymiseURLs(ctx, goqu.T(imagesURLsTable), "image_id"); err != nil { + return err + } + return nil } diff --git a/pkg/sqlite/database.go b/pkg/sqlite/database.go index cd87a887c25..f5291ca1a72 100644 --- a/pkg/sqlite/database.go +++ b/pkg/sqlite/database.go @@ -33,7 +33,7 @@ const ( dbConnTimeout = 30 ) -var appSchemaVersion uint = 49 +var appSchemaVersion uint = 50 //go:embed migrations/*.sql var migrationsBox embed.FS diff --git a/pkg/sqlite/image.go b/pkg/sqlite/image.go index 0258ce537eb..0ee12f0d947 100644 --- a/pkg/sqlite/image.go +++ b/pkg/sqlite/image.go @@ -24,27 +24,27 @@ const ( performersImagesTable = "performers_images" imagesTagsTable = "images_tags" imagesFilesTable = "images_files" + imagesURLsTable = "image_urls" + imageURLColumn = "url" ) type imageRow struct { ID int `db:"id" goqu:"skipinsert"` Title zero.String `db:"title"` // expressed as 1-100 - Rating null.Int `db:"rating"` - URL zero.String `db:"url"` - Date NullDate `db:"date"` - Organized bool `db:"organized"` - OCounter int `db:"o_counter"` - StudioID null.Int `db:"studio_id,omitempty"` - CreatedAt Timestamp `db:"created_at"` - UpdatedAt Timestamp `db:"updated_at"` + Rating null.Int `db:"rating"` + Date NullDate `db:"date"` + Organized bool `db:"organized"` + OCounter int `db:"o_counter"` + StudioID null.Int `db:"studio_id,omitempty"` + CreatedAt Timestamp `db:"created_at"` + UpdatedAt Timestamp `db:"updated_at"` } func (r *imageRow) fromImage(i models.Image) { r.ID = i.ID r.Title = zero.StringFrom(i.Title) r.Rating = intFromPtr(i.Rating) - r.URL = zero.StringFrom(i.URL) r.Date = NullDateFromDatePtr(i.Date) r.Organized = i.Organized r.OCounter = i.OCounter @@ -66,7 +66,6 @@ func (r *imageQueryRow) resolve() *models.Image { ID: r.ID, Title: r.Title.String, Rating: nullIntPtr(r.Rating), - URL: r.URL.String, Date: r.Date.DatePtr(), Organized: r.Organized, OCounter: r.OCounter, @@ -93,7 +92,6 @@ type imageRowRecord struct { func (r *imageRowRecord) fromPartial(i models.ImagePartial) { r.setNullString("title", i.Title) r.setNullInt("rating", i.Rating) - r.setNullString("url", i.URL) r.setNullDate("date", i.Date) r.setBool("organized", i.Organized) r.setInt("o_counter", i.OCounter) @@ -176,6 +174,13 @@ func (qb *ImageStore) Create(ctx context.Context, newObject *models.Image, fileI } } + if newObject.URLs.Loaded() { + const startPos = 0 + if err := imagesURLsTableMgr.insertJoins(ctx, id, startPos, newObject.URLs.List()); err != nil { + return err + } + } + if newObject.PerformerIDs.Loaded() { if err := imagesPerformersTableMgr.insertJoins(ctx, id, newObject.PerformerIDs.List()); err != nil { return err @@ -223,6 +228,12 @@ func (qb *ImageStore) UpdatePartial(ctx context.Context, id int, partial models. return nil, err } } + + if partial.URLs != nil { + if err := imagesURLsTableMgr.modifyJoins(ctx, id, partial.URLs.Values, partial.URLs.Mode); err != nil { + return nil, err + } + } if partial.PerformerIDs != nil { if err := imagesPerformersTableMgr.modifyJoins(ctx, id, partial.PerformerIDs.IDs, partial.PerformerIDs.Mode); err != nil { return nil, err @@ -251,6 +262,12 @@ func (qb *ImageStore) Update(ctx context.Context, updatedObject *models.Image) e return err } + if updatedObject.URLs.Loaded() { + if err := imagesURLsTableMgr.replaceJoins(ctx, updatedObject.ID, updatedObject.URLs.List()); err != nil { + return err + } + } + if updatedObject.PerformerIDs.Loaded() { if err := imagesPerformersTableMgr.replaceJoins(ctx, updatedObject.ID, updatedObject.PerformerIDs.List()); err != nil { return err @@ -664,7 +681,7 @@ func (qb *ImageStore) makeFilter(ctx context.Context, imageFilter *models.ImageF query.handleCriterion(ctx, intCriterionHandler(imageFilter.OCounter, "images.o_counter", nil)) query.handleCriterion(ctx, boolCriterionHandler(imageFilter.Organized, "images.organized", nil)) query.handleCriterion(ctx, dateCriterionHandler(imageFilter.Date, "images.date")) - query.handleCriterion(ctx, stringCriterionHandler(imageFilter.URL, "images.url")) + query.handleCriterion(ctx, imageURLsCriterionHandler(imageFilter.URL)) query.handleCriterion(ctx, resolutionCriterionHandler(imageFilter.Resolution, "image_files.height", "image_files.width", qb.addImageFilesTable)) query.handleCriterion(ctx, imageIsMissingCriterionHandler(qb, imageFilter.IsMissing)) @@ -855,6 +872,18 @@ func imageIsMissingCriterionHandler(qb *ImageStore, isMissing *string) criterion } } +func imageURLsCriterionHandler(url *models.StringCriterionInput) criterionHandlerFunc { + h := stringListCriterionHandlerBuilder{ + joinTable: imagesURLsTable, + stringColumn: imageURLColumn, + addJoinTable: func(f *filterBuilder) { + imagesURLsTableMgr.join(f, "", "images.id") + }, + } + + return h.handler(url) +} + func (qb *ImageStore) getMultiCriterionHandlerBuilder(foreignTable, joinTable, foreignFK string, addJoinsFunc func(f *filterBuilder)) multiCriterionHandlerBuilder { return multiCriterionHandlerBuilder{ primaryTable: imageTable, @@ -1097,3 +1126,7 @@ func (qb *ImageStore) UpdateTags(ctx context.Context, imageID int, tagIDs []int) // Delete the existing joins and then create new ones return qb.tagsRepository().replace(ctx, imageID, tagIDs) } + +func (qb *ImageStore) GetURLs(ctx context.Context, imageID int) ([]string, error) { + return imagesURLsTableMgr.get(ctx, imageID) +} diff --git a/pkg/sqlite/image_test.go b/pkg/sqlite/image_test.go index 621b72c64cc..7735cb5ec43 100644 --- a/pkg/sqlite/image_test.go +++ b/pkg/sqlite/image_test.go @@ -15,6 +15,11 @@ import ( ) func loadImageRelationships(ctx context.Context, expected models.Image, actual *models.Image) error { + if expected.URLs.Loaded() { + if err := actual.LoadURLs(ctx, db.Image); err != nil { + return err + } + } if expected.GalleryIDs.Loaded() { if err := actual.LoadGalleryIDs(ctx, db.Image); err != nil { return err @@ -74,7 +79,7 @@ func Test_imageQueryBuilder_Create(t *testing.T) { Title: title, Rating: &rating, Date: &date, - URL: url, + URLs: models.NewRelatedStrings([]string{url}), Organized: true, OCounter: ocounter, StudioID: &studioIDs[studioIdxWithImage], @@ -92,7 +97,7 @@ func Test_imageQueryBuilder_Create(t *testing.T) { Title: title, Rating: &rating, Date: &date, - URL: url, + URLs: models.NewRelatedStrings([]string{url}), Organized: true, OCounter: ocounter, StudioID: &studioIDs[studioIdxWithImage], @@ -229,7 +234,7 @@ func Test_imageQueryBuilder_Update(t *testing.T) { ID: imageIDs[imageIdxWithGallery], Title: title, Rating: &rating, - URL: url, + URLs: models.NewRelatedStrings([]string{url}), Date: &date, Organized: true, OCounter: ocounter, @@ -378,7 +383,7 @@ func clearImagePartial() models.ImagePartial { return models.ImagePartial{ Title: models.OptionalString{Set: true, Null: true}, Rating: models.OptionalInt{Set: true, Null: true}, - URL: models.OptionalString{Set: true, Null: true}, + URLs: &models.UpdateStrings{Mode: models.RelationshipUpdateModeSet}, Date: models.OptionalDate{Set: true, Null: true}, StudioID: models.OptionalInt{Set: true, Null: true}, GalleryIDs: &models.UpdateIDs{Mode: models.RelationshipUpdateModeSet}, @@ -409,9 +414,12 @@ func Test_imageQueryBuilder_UpdatePartial(t *testing.T) { "full", imageIDs[imageIdx1WithGallery], models.ImagePartial{ - Title: models.NewOptionalString(title), - Rating: models.NewOptionalInt(rating), - URL: models.NewOptionalString(url), + Title: models.NewOptionalString(title), + Rating: models.NewOptionalInt(rating), + URLs: &models.UpdateStrings{ + Values: []string{url}, + Mode: models.RelationshipUpdateModeSet, + }, Date: models.NewOptionalDate(date), Organized: models.NewOptionalBool(true), OCounter: models.NewOptionalInt(ocounter), @@ -435,7 +443,7 @@ func Test_imageQueryBuilder_UpdatePartial(t *testing.T) { ID: imageIDs[imageIdx1WithGallery], Title: title, Rating: &rating, - URL: url, + URLs: models.NewRelatedStrings([]string{url}), Date: &date, Organized: true, OCounter: ocounter, @@ -1519,6 +1527,67 @@ func imageQueryQ(ctx context.Context, t *testing.T, sqb models.ImageReader, q st assert.Len(t, images, totalImages) } +func verifyImageQuery(t *testing.T, filter models.ImageFilterType, verifyFn func(ctx context.Context, s *models.Image)) { + t.Helper() + withTxn(func(ctx context.Context) error { + t.Helper() + sqb := db.Image + + images := queryImages(ctx, t, sqb, &filter, nil) + + // assume it should find at least one + assert.Greater(t, len(images), 0) + + for _, image := range images { + verifyFn(ctx, image) + } + + return nil + }) +} + +func TestImageQueryURL(t *testing.T) { + const imageIdx = 1 + imageURL := getImageStringValue(imageIdx, urlField) + urlCriterion := models.StringCriterionInput{ + Value: imageURL, + Modifier: models.CriterionModifierEquals, + } + filter := models.ImageFilterType{ + URL: &urlCriterion, + } + + verifyFn := func(ctx context.Context, o *models.Image) { + t.Helper() + + if err := o.LoadURLs(ctx, db.Image); err != nil { + t.Errorf("Error loading scene URLs: %v", err) + } + + urls := o.URLs.List() + var url string + if len(urls) > 0 { + url = urls[0] + } + + verifyString(t, url, urlCriterion) + } + + verifyImageQuery(t, filter, verifyFn) + urlCriterion.Modifier = models.CriterionModifierNotEquals + verifyImageQuery(t, filter, verifyFn) + urlCriterion.Modifier = models.CriterionModifierMatchesRegex + urlCriterion.Value = "image_.*1_URL" + verifyImageQuery(t, filter, verifyFn) + urlCriterion.Modifier = models.CriterionModifierNotMatchesRegex + verifyImageQuery(t, filter, verifyFn) + urlCriterion.Modifier = models.CriterionModifierIsNull + urlCriterion.Value = "" + verifyImageQuery(t, filter, verifyFn) + urlCriterion.Modifier = models.CriterionModifierNotNull + verifyImageQuery(t, filter, verifyFn) +} + func TestImageQueryPath(t *testing.T) { const imageIdx = 1 imagePath := getFilePath(folderIdxWithImageFiles, getImageBasename(imageIdx)) diff --git a/pkg/sqlite/migrations/50_image_urls.up.sql b/pkg/sqlite/migrations/50_image_urls.up.sql new file mode 100644 index 00000000000..47ff373075b --- /dev/null +++ b/pkg/sqlite/migrations/50_image_urls.up.sql @@ -0,0 +1,70 @@ +PRAGMA foreign_keys=OFF; + +CREATE TABLE `image_urls` ( + `image_id` integer NOT NULL, + `position` integer NOT NULL, + `url` varchar(255) NOT NULL, + foreign key(`image_id`) references `images`(`id`) on delete CASCADE, + PRIMARY KEY(`image_id`, `position`, `url`) +); + +CREATE INDEX `image_urls_url` on `image_urls` (`url`); + +-- drop url +CREATE TABLE "images_new" ( + `id` integer not null primary key autoincrement, + `title` varchar(255), + `rating` tinyint, + `studio_id` integer, + `o_counter` tinyint not null default 0, + `organized` boolean not null default '0', + `created_at` datetime not null, + `updated_at` datetime not null, + `date` date, + foreign key(`studio_id`) references `studios`(`id`) on delete SET NULL +); + +INSERT INTO `images_new` + ( + `id`, + `title`, + `rating`, + `studio_id`, + `o_counter`, + `organized`, + `created_at`, + `updated_at`, + `date` + ) + SELECT + `id`, + `title`, + `rating`, + `studio_id`, + `o_counter`, + `organized`, + `created_at`, + `updated_at`, + `date` + FROM `images`; + +INSERT INTO `image_urls` + ( + `image_id`, + `position`, + `url` + ) + SELECT + `id`, + '0', + `url` + FROM `images` + WHERE `images`.`url` IS NOT NULL AND `images`.`url` != ''; + +DROP INDEX `index_images_on_studio_id`; +DROP TABLE `images`; +ALTER TABLE `images_new` rename to `images`; + +CREATE INDEX `index_images_on_studio_id` on `images` (`studio_id`); + +PRAGMA foreign_keys=ON; diff --git a/pkg/sqlite/setup_test.go b/pkg/sqlite/setup_test.go index 04236b3ac1c..e182ef99b5b 100644 --- a/pkg/sqlite/setup_test.go +++ b/pkg/sqlite/setup_test.go @@ -1113,6 +1113,19 @@ func getImageStringValue(index int, field string) string { return fmt.Sprintf("image_%04d_%s", index, field) } +func getImageNullStringPtr(index int, field string) *string { + return getStringPtrFromNullString(getPrefixedNullStringValue("image", index, field)) +} + +func getImageEmptyString(index int, field string) string { + v := getImageNullStringPtr(index, field) + if v == nil { + return "" + } + + return *v +} + func getImageBasename(index int) string { return getImageStringValue(index, pathField) } @@ -1148,10 +1161,12 @@ func makeImage(i int) *models.Image { tids := indexesToIDs(tagIDs, imageTags[i]) return &models.Image{ - Title: title, - Rating: getIntPtr(getRating(i)), - Date: getObjectDate(i), - URL: getImageStringValue(i, urlField), + Title: title, + Rating: getIntPtr(getRating(i)), + Date: getObjectDate(i), + URLs: models.NewRelatedStrings([]string{ + getImageEmptyString(i, urlField), + }), OCounter: getOCounter(i), StudioID: studioID, GalleryIDs: models.NewRelatedIDs(gids), diff --git a/pkg/sqlite/tables.go b/pkg/sqlite/tables.go index 69dc1d6a89f..dc1eb505115 100644 --- a/pkg/sqlite/tables.go +++ b/pkg/sqlite/tables.go @@ -13,6 +13,7 @@ var ( imagesTagsJoinTable = goqu.T(imagesTagsTable) performersImagesJoinTable = goqu.T(performersImagesTable) imagesFilesJoinTable = goqu.T(imagesFilesTable) + imagesURLsJoinTable = goqu.T(imagesURLsTable) galleriesFilesJoinTable = goqu.T(galleriesFilesTable) galleriesTagsJoinTable = goqu.T(galleriesTagsTable) @@ -70,6 +71,14 @@ var ( }, fkColumn: performersImagesJoinTable.Col(performerIDColumn), } + + imagesURLsTableMgr = &orderedValueTable[string]{ + table: table{ + table: imagesURLsJoinTable, + idColumn: imagesURLsJoinTable.Col(imageIDColumn), + }, + valueColumn: imagesURLsJoinTable.Col(imageURLColumn), + } ) var ( diff --git a/ui/v2.5/src/components/Images/ImageDetails/ImageEditPanel.tsx b/ui/v2.5/src/components/Images/ImageDetails/ImageEditPanel.tsx index a684e29dabf..8ff5fb6955e 100644 --- a/ui/v2.5/src/components/Images/ImageDetails/ImageEditPanel.tsx +++ b/ui/v2.5/src/components/Images/ImageDetails/ImageEditPanel.tsx @@ -6,7 +6,7 @@ import * as GQL from "src/core/generated-graphql"; import * as yup from "yup"; import { TagSelect, StudioSelect } from "src/components/Shared/Select"; import { LoadingIndicator } from "src/components/Shared/LoadingIndicator"; -import { URLField } from "src/components/Shared/URLField"; +import { URLListInput } from "src/components/Shared/URLField"; import { useToast } from "src/hooks/Toast"; import FormUtils from "src/utils/form"; import { useFormik } from "formik"; @@ -16,6 +16,7 @@ import { useRatingKeybinds } from "src/hooks/keybinds"; import { ConfigurationContext } from "src/hooks/Config"; import isEqual from "lodash-es/isEqual"; import { DateInput } from "src/components/Shared/DateInput"; +import { yupDateString, yupUniqueStringList } from "src/utils/yup"; import { Performer, PerformerSelect, @@ -46,20 +47,8 @@ export const ImageEditPanel: React.FC = ({ const schema = yup.object({ title: yup.string().ensure(), - url: yup.string().ensure(), - date: yup - .string() - .ensure() - .test({ - name: "date", - test: (value) => { - if (!value) return true; - if (!value.match(/^\d{4}-\d{2}-\d{2}$/)) return false; - if (Number.isNaN(Date.parse(value))) return false; - return true; - }, - message: intl.formatMessage({ id: "validation.date_invalid_form" }), - }), + urls: yupUniqueStringList("urls"), + date: yupDateString(intl), rating100: yup.number().nullable().defined(), studio_id: yup.string().required().nullable(), performer_ids: yup.array(yup.string().required()).defined(), @@ -68,7 +57,7 @@ export const ImageEditPanel: React.FC = ({ const initialValues = { title: image.title ?? "", - url: image?.url ?? "", + urls: image?.urls ?? [], date: image?.date ?? "", rating100: image.rating100 ?? null, studio_id: image.studio?.id ?? null, @@ -162,6 +151,14 @@ export const ImageEditPanel: React.FC = ({ if (isLoading) return ; + const urlsErrors = Array.isArray(formik.errors.urls) + ? formik.errors.urls[0] + : formik.errors.urls; + const urlsErrorMsg = urlsErrors + ? intl.formatMessage({ id: "validation.urls_must_be_unique" }) + : undefined; + const urlsErrorIdx = urlsErrors?.split(" ").map((e) => parseInt(e)); + return (
= ({
{renderTextField("title", intl.formatMessage({ id: "title" }))} - + - + - {}} - urlScrapable={() => { - return false; - }} - isInvalid={!!formik.getFieldMeta("url").error} + formik.setFieldValue("urls", value)} + errors={urlsErrorMsg} + errorIdx={urlsErrorIdx} /> diff --git a/ui/v2.5/src/components/Images/ImageDetails/ImageFileInfoPanel.tsx b/ui/v2.5/src/components/Images/ImageDetails/ImageFileInfoPanel.tsx index 2b906c6d5ef..adf95d2f9f2 100644 --- a/ui/v2.5/src/components/Images/ImageDetails/ImageFileInfoPanel.tsx +++ b/ui/v2.5/src/components/Images/ImageDetails/ImageFileInfoPanel.tsx @@ -7,7 +7,7 @@ import * as GQL from "src/core/generated-graphql"; import { mutateImageSetPrimaryFile } from "src/core/StashService"; import { useToast } from "src/hooks/Toast"; import TextUtils from "src/utils/text"; -import { TextField, URLField } from "src/utils/field"; +import { TextField, URLField, URLsField } from "src/utils/field"; interface IFileInfoPanelProps { file: GQL.ImageFileDataFragment | GQL.VideoFileDataFragment; @@ -120,20 +120,11 @@ export const ImageFileInfoPanel: React.FC = ( if (props.image.visual_files.length === 1) { return ( <> - +
+ +
- {props.image.url ? ( -
- -
- ) : ( - "" - )} + ); } diff --git a/ui/v2.5/src/components/Scenes/SceneDetails/SceneEditPanel.tsx b/ui/v2.5/src/components/Scenes/SceneDetails/SceneEditPanel.tsx index 7842f104dbd..860b26a78cc 100644 --- a/ui/v2.5/src/components/Scenes/SceneDetails/SceneEditPanel.tsx +++ b/ui/v2.5/src/components/Scenes/SceneDetails/SceneEditPanel.tsx @@ -50,6 +50,7 @@ import { useRatingKeybinds } from "src/hooks/keybinds"; import { lazyComponent } from "src/utils/lazyComponent"; import isEqual from "lodash-es/isEqual"; import { DateInput } from "src/components/Shared/DateInput"; +import { yupDateString, yupUniqueStringList } from "src/utils/yup"; import { Performer, PerformerSelect, @@ -114,38 +115,8 @@ export const SceneEditPanel: React.FC = ({ const schema = yup.object({ title: yup.string().ensure(), code: yup.string().ensure(), - urls: yup - .array(yup.string().required()) - .defined() - .test({ - name: "unique", - test: (value) => { - const dupes = value - .map((e, i, a) => { - if (a.indexOf(e) !== i) { - return String(i - 1); - } else { - return null; - } - }) - .filter((e) => e !== null) as string[]; - if (dupes.length === 0) return true; - return new yup.ValidationError(dupes.join(" "), value, "urls"); - }, - }), - date: yup - .string() - .ensure() - .test({ - name: "date", - test: (value) => { - if (!value) return true; - if (!value.match(/^\d{4}-\d{2}-\d{2}$/)) return false; - if (Number.isNaN(Date.parse(value))) return false; - return true; - }, - message: intl.formatMessage({ id: "validation.date_invalid_form" }), - }), + urls: yupUniqueStringList("urls"), + date: yupDateString(intl), director: yup.string().ensure(), rating100: yup.number().nullable().defined(), gallery_ids: yup.array(yup.string().required()).defined(), diff --git a/ui/v2.5/src/components/Shared/URLField.tsx b/ui/v2.5/src/components/Shared/URLField.tsx index 9cea50e7c7a..ca84bd08999 100644 --- a/ui/v2.5/src/components/Shared/URLField.tsx +++ b/ui/v2.5/src/components/Shared/URLField.tsx @@ -50,8 +50,8 @@ export const URLField: React.FC = (props: IProps) => { }; interface IURLListProps extends IStringListInputProps { - onScrapeClick(url: string): void; - urlScrapable(url: string): boolean; + onScrapeClick?: (url: string) => void; + urlScrapable?: (url: string) => boolean; } export const URLListInput: React.FC = ( @@ -64,17 +64,23 @@ export const URLListInput: React.FC = ( {...listProps} placeholder={intl.formatMessage({ id: "url" })} inputComponent={StringInput} - appendComponent={(props) => ( - - )} + appendComponent={(props) => { + if (!onScrapeClick || !urlScrapable) { + return <>; + } + + return ( + + ); + }} /> ); }; diff --git a/ui/v2.5/src/utils/yup.ts b/ui/v2.5/src/utils/yup.ts new file mode 100644 index 00000000000..d93bc5d3e57 --- /dev/null +++ b/ui/v2.5/src/utils/yup.ts @@ -0,0 +1,40 @@ +import { IntlShape } from "react-intl"; +import * as yup from "yup"; + +export function yupUniqueStringList(fieldName: string) { + return yup + .array(yup.string().required()) + .defined() + .test({ + name: "unique", + test: (value) => { + const dupes = value + .map((e, i, a) => { + if (a.indexOf(e) !== i) { + return String(i - 1); + } else { + return null; + } + }) + .filter((e) => e !== null) as string[]; + if (dupes.length === 0) return true; + return new yup.ValidationError(dupes.join(" "), value, fieldName); + }, + }); +} + +export function yupDateString(intl: IntlShape) { + return yup + .string() + .ensure() + .test({ + name: "date", + test: (value) => { + if (!value) return true; + if (!value.match(/^\d{4}-\d{2}-\d{2}$/)) return false; + if (Number.isNaN(Date.parse(value))) return false; + return true; + }, + message: intl.formatMessage({ id: "validation.date_invalid_form" }), + }); +}