From 9577600804517b4ba58e2f7df1afdefb9d649545 Mon Sep 17 00:00:00 2001 From: WithoutPants <53250216+WithoutPants@users.noreply.github.com> Date: Mon, 25 Sep 2023 12:27:20 +1000 Subject: [PATCH] Gallery URLs (#4114) * Initial backend changes * Fix unit tests * UI changes * Fix missing URL filters --- graphql/documents/data/gallery-slim.graphql | 2 +- graphql/documents/data/gallery.graphql | 2 +- graphql/documents/data/scrapers.graphql | 2 +- graphql/schema/types/gallery.graphql | 12 ++- graphql/schema/types/scraper.graphql | 6 +- internal/api/resolver_model_gallery.go | 29 +++++++ internal/api/resolver_mutation_gallery.go | 12 ++- pkg/gallery/export.go | 2 +- pkg/gallery/export_test.go | 4 +- pkg/gallery/import.go | 6 +- pkg/gallery/import_test.go | 2 +- pkg/models/gallery.go | 5 +- pkg/models/jsonschema/gallery.go | 5 +- pkg/models/mocks/GalleryReaderWriter.go | 23 ++++++ pkg/models/model_gallery.go | 16 ++-- pkg/models/repository_gallery.go | 1 + pkg/scraper/gallery.go | 16 ++-- pkg/scraper/query_url.go | 4 +- pkg/scraper/stash.go | 9 ++- pkg/sqlite/database.go | 2 +- pkg/sqlite/gallery.go | 43 +++++++++-- pkg/sqlite/gallery_test.go | 39 +++++++--- pkg/sqlite/migrations/51_gallery_urls.up.sql | 76 +++++++++++++++++++ pkg/sqlite/scene.go | 3 + pkg/sqlite/setup_test.go | 17 ++++- pkg/sqlite/tables.go | 9 +++ .../GalleryDetails/GalleryEditPanel.tsx | 53 +++++++------ .../GalleryDetails/GalleryFileInfoPanel.tsx | 9 +-- .../GalleryDetails/GalleryScrapeDialog.tsx | 63 ++++++++------- 29 files changed, 358 insertions(+), 114 deletions(-) create mode 100644 pkg/sqlite/migrations/51_gallery_urls.up.sql diff --git a/graphql/documents/data/gallery-slim.graphql b/graphql/documents/data/gallery-slim.graphql index ebec042512c..ea44b2997ba 100644 --- a/graphql/documents/data/gallery-slim.graphql +++ b/graphql/documents/data/gallery-slim.graphql @@ -2,7 +2,7 @@ fragment SlimGalleryData on Gallery { id title date - url + urls details rating100 organized diff --git a/graphql/documents/data/gallery.graphql b/graphql/documents/data/gallery.graphql index b4df2589690..89719dfca65 100644 --- a/graphql/documents/data/gallery.graphql +++ b/graphql/documents/data/gallery.graphql @@ -4,7 +4,7 @@ fragment GalleryData on Gallery { updated_at title date - url + urls details rating100 organized diff --git a/graphql/documents/data/scrapers.graphql b/graphql/documents/data/scrapers.graphql index 502a9ac4071..05ed76b5cff 100644 --- a/graphql/documents/data/scrapers.graphql +++ b/graphql/documents/data/scrapers.graphql @@ -185,7 +185,7 @@ fragment ScrapedSceneData on ScrapedScene { fragment ScrapedGalleryData on ScrapedGallery { title details - url + urls date studio { diff --git a/graphql/schema/types/gallery.graphql b/graphql/schema/types/gallery.graphql index c2526fc6298..4f17a673842 100644 --- a/graphql/schema/types/gallery.graphql +++ b/graphql/schema/types/gallery.graphql @@ -4,7 +4,8 @@ type Gallery { checksum: String! @deprecated(reason: "Use files.fingerprints") path: String @deprecated(reason: "Use files.path") title: String - url: String + url: String @deprecated(reason: "Use urls") + urls: [String!]! date: String details: String # rating expressed as 1-5 @@ -33,7 +34,8 @@ type Gallery { input GalleryCreateInput { title: String! - url: String + url: String @deprecated(reason: "Use urls") + urls: [String!] date: String details: String # rating expressed as 1-5 @@ -51,7 +53,8 @@ input GalleryUpdateInput { clientMutationId: String id: ID! title: String - url: String + url: String @deprecated(reason: "Use urls") + urls: [String!] date: String details: String # rating expressed as 1-5 @@ -70,7 +73,8 @@ input GalleryUpdateInput { input BulkGalleryUpdateInput { clientMutationId: String ids: [ID!] - url: String + url: String @deprecated(reason: "Use urls") + urls: BulkUpdateStrings date: String details: String # rating expressed as 1-5 diff --git a/graphql/schema/types/scraper.graphql b/graphql/schema/types/scraper.graphql index 191feca9155..320d6065c00 100644 --- a/graphql/schema/types/scraper.graphql +++ b/graphql/schema/types/scraper.graphql @@ -100,7 +100,8 @@ input ScrapedSceneInput { type ScrapedGallery { title: String details: String - url: String + url: String @deprecated(reason: "use urls") + urls: [String!] date: String studio: ScrapedStudio @@ -111,7 +112,8 @@ type ScrapedGallery { input ScrapedGalleryInput { title: String details: String - url: String + url: String @deprecated(reason: "use urls") + urls: [String!] date: String # no studio, tags or performers diff --git a/internal/api/resolver_model_gallery.go b/internal/api/resolver_model_gallery.go index e7c0cd6a04c..d979d7b5885 100644 --- a/internal/api/resolver_model_gallery.go +++ b/internal/api/resolver_model_gallery.go @@ -226,3 +226,32 @@ func (r *galleryResolver) Chapters(ctx context.Context, obj *models.Gallery) (re return ret, nil } + +func (r *galleryResolver) URL(ctx context.Context, obj *models.Gallery) (*string, error) { + if !obj.URLs.Loaded() { + if err := r.withReadTxn(ctx, func(ctx context.Context) error { + return obj.LoadURLs(ctx, r.repository.Gallery) + }); err != nil { + return nil, err + } + } + + urls := obj.URLs.List() + if len(urls) == 0 { + return nil, nil + } + + return &urls[0], nil +} + +func (r *galleryResolver) Urls(ctx context.Context, obj *models.Gallery) ([]string, error) { + if !obj.URLs.Loaded() { + if err := r.withReadTxn(ctx, func(ctx context.Context) error { + return obj.LoadURLs(ctx, r.repository.Gallery) + }); err != nil { + return nil, err + } + } + + return obj.URLs.List(), nil +} diff --git a/internal/api/resolver_mutation_gallery.go b/internal/api/resolver_mutation_gallery.go index c7dc8d70f64..0f246d857e8 100644 --- a/internal/api/resolver_mutation_gallery.go +++ b/internal/api/resolver_mutation_gallery.go @@ -43,7 +43,6 @@ func (r *mutationResolver) GalleryCreate(ctx context.Context, input GalleryCreat newGallery := models.NewGallery() newGallery.Title = input.Title - newGallery.URL = translator.string(input.URL) newGallery.Details = translator.string(input.Details) newGallery.Rating = translator.ratingConversion(input.Rating, input.Rating100) @@ -71,6 +70,12 @@ func (r *mutationResolver) GalleryCreate(ctx context.Context, input GalleryCreat return nil, fmt.Errorf("converting scene ids: %w", err) } + if input.Urls != nil { + newGallery.URLs = models.NewRelatedStrings(input.Urls) + } else if input.URL != nil { + newGallery.URLs = models.NewRelatedStrings([]string{*input.URL}) + } + // Start the transaction and save the gallery if err := r.withTxn(ctx, func(ctx context.Context) error { qb := r.repository.Gallery @@ -178,7 +183,6 @@ func (r *mutationResolver) galleryUpdate(ctx context.Context, input models.Galle } updatedGallery.Details = translator.optionalString(input.Details, "details") - updatedGallery.URL = translator.optionalString(input.URL, "url") updatedGallery.Rating = translator.optionalRatingConversion(input.Rating, input.Rating100) updatedGallery.Organized = translator.optionalBool(input.Organized, "organized") @@ -191,6 +195,8 @@ func (r *mutationResolver) galleryUpdate(ctx context.Context, input models.Galle return nil, fmt.Errorf("converting studio id: %w", err) } + updatedGallery.URLs = translator.optionalURLs(input.Urls, input.URL) + updatedGallery.PrimaryFileID, err = translator.fileIDPtrFromString(input.PrimaryFileID) if err != nil { return nil, fmt.Errorf("converting primary file id: %w", err) @@ -252,9 +258,9 @@ func (r *mutationResolver) BulkGalleryUpdate(ctx context.Context, input BulkGall updatedGallery := models.NewGalleryPartial() updatedGallery.Details = translator.optionalString(input.Details, "details") - updatedGallery.URL = translator.optionalString(input.URL, "url") updatedGallery.Rating = translator.optionalRatingConversion(input.Rating, input.Rating100) updatedGallery.Organized = translator.optionalBool(input.Organized, "organized") + updatedGallery.URLs = translator.optionalURLsBulk(input.Urls, input.URL) updatedGallery.Date, err = translator.optionalDate(input.Date, "date") if err != nil { diff --git a/pkg/gallery/export.go b/pkg/gallery/export.go index 83f3c31cebc..5412e9a509c 100644 --- a/pkg/gallery/export.go +++ b/pkg/gallery/export.go @@ -14,7 +14,7 @@ import ( func ToBasicJSON(gallery *models.Gallery) (*jsonschema.Gallery, error) { newGalleryJSON := jsonschema.Gallery{ Title: gallery.Title, - URL: gallery.URL, + URLs: gallery.URLs.List(), Details: gallery.Details, CreatedAt: json.JSONTime{Time: gallery.CreatedAt}, UpdatedAt: json.JSONTime{Time: gallery.UpdatedAt}, diff --git a/pkg/gallery/export_test.go b/pkg/gallery/export_test.go index 3a6ffa2ec55..eba08e4e513 100644 --- a/pkg/gallery/export_test.go +++ b/pkg/gallery/export_test.go @@ -59,7 +59,7 @@ func createFullGallery(id int) models.Gallery { Details: details, Rating: &rating, Organized: organized, - URL: url, + URLs: models.NewRelatedStrings([]string{url}), CreatedAt: createTime, UpdatedAt: updateTime, } @@ -85,7 +85,7 @@ func createFullJSONGallery() *jsonschema.Gallery { Details: details, Rating: rating, Organized: organized, - URL: url, + URLs: []string{url}, ZipFiles: []string{path}, CreatedAt: json.JSONTime{ Time: createTime, diff --git a/pkg/gallery/import.go b/pkg/gallery/import.go index 9c892d3b9a9..780b1e63bce 100644 --- a/pkg/gallery/import.go +++ b/pkg/gallery/import.go @@ -65,8 +65,10 @@ func (i *Importer) galleryJSONToGallery(galleryJSON jsonschema.Gallery) models.G if galleryJSON.Details != "" { newGallery.Details = galleryJSON.Details } - if galleryJSON.URL != "" { - newGallery.URL = galleryJSON.URL + if len(galleryJSON.URLs) > 0 { + newGallery.URLs = models.NewRelatedStrings(galleryJSON.URLs) + } else if galleryJSON.URL != "" { + newGallery.URLs = models.NewRelatedStrings([]string{galleryJSON.URL}) } if galleryJSON.Date != "" { d, err := models.ParseDate(galleryJSON.Date) diff --git a/pkg/gallery/import_test.go b/pkg/gallery/import_test.go index 0997b4a57e2..8263f97870c 100644 --- a/pkg/gallery/import_test.go +++ b/pkg/gallery/import_test.go @@ -66,7 +66,7 @@ func TestImporterPreImport(t *testing.T) { Details: details, Rating: &rating, Organized: organized, - URL: url, + URLs: models.NewRelatedStrings([]string{url}), Files: models.NewRelatedFiles([]models.File{}), TagIDs: models.NewRelatedIDs([]int{}), PerformerIDs: models.NewRelatedIDs([]int{}), diff --git a/pkg/models/gallery.go b/pkg/models/gallery.go index d3644d3fd6c..89651dbc41a 100644 --- a/pkg/models/gallery.go +++ b/pkg/models/gallery.go @@ -59,7 +59,7 @@ type GalleryUpdateInput struct { ClientMutationID *string `json:"clientMutationId"` ID string `json:"id"` Title *string `json:"title"` - URL *string `json:"url"` + Urls []string `json:"urls"` Date *string `json:"date"` Details *string `json:"details"` Rating *int `json:"rating"` @@ -70,6 +70,9 @@ type GalleryUpdateInput struct { TagIds []string `json:"tag_ids"` PerformerIds []string `json:"performer_ids"` PrimaryFileID *string `json:"primary_file_id"` + + // deprecated + URL *string `json:"url"` } type GalleryDestroyInput struct { diff --git a/pkg/models/jsonschema/gallery.go b/pkg/models/jsonschema/gallery.go index ca399624ebf..0832cc07a6b 100644 --- a/pkg/models/jsonschema/gallery.go +++ b/pkg/models/jsonschema/gallery.go @@ -21,7 +21,7 @@ type Gallery struct { ZipFiles []string `json:"zip_files,omitempty"` FolderPath string `json:"folder_path,omitempty"` Title string `json:"title,omitempty"` - URL string `json:"url,omitempty"` + URLs []string `json:"urls,omitempty"` Date string `json:"date,omitempty"` Details string `json:"details,omitempty"` Rating int `json:"rating,omitempty"` @@ -32,6 +32,9 @@ type Gallery struct { Tags []string `json:"tags,omitempty"` CreatedAt json.JSONTime `json:"created_at,omitempty"` UpdatedAt json.JSONTime `json:"updated_at,omitempty"` + + // deprecated - for import only + URL string `json:"url,omitempty"` } func (s Gallery) Filename(basename string, hash string) string { diff --git a/pkg/models/mocks/GalleryReaderWriter.go b/pkg/models/mocks/GalleryReaderWriter.go index 2b901466b19..bd1fbf0d2b9 100644 --- a/pkg/models/mocks/GalleryReaderWriter.go +++ b/pkg/models/mocks/GalleryReaderWriter.go @@ -533,6 +533,29 @@ func (_m *GalleryReaderWriter) GetTagIDs(ctx context.Context, relatedID int) ([] return r0, r1 } +// GetURLs provides a mock function with given fields: ctx, relatedID +func (_m *GalleryReaderWriter) GetURLs(ctx context.Context, relatedID int) ([]string, error) { + ret := _m.Called(ctx, relatedID) + + var r0 []string + if rf, ok := ret.Get(0).(func(context.Context, int) []string); ok { + r0 = rf(ctx, relatedID) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]string) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, int) error); ok { + r1 = rf(ctx, relatedID) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // Query provides a mock function with given fields: ctx, galleryFilter, findFilter func (_m *GalleryReaderWriter) Query(ctx context.Context, galleryFilter *models.GalleryFilterType, findFilter *models.FindFilterType) ([]*models.Gallery, int, error) { ret := _m.Called(ctx, galleryFilter, findFilter) diff --git a/pkg/models/model_gallery.go b/pkg/models/model_gallery.go index c7c74a017f3..d35b9a360bb 100644 --- a/pkg/models/model_gallery.go +++ b/pkg/models/model_gallery.go @@ -11,7 +11,6 @@ type Gallery struct { ID int `json:"id"` Title string `json:"title"` - URL string `json:"url"` Date *Date `json:"date"` Details string `json:"details"` // Rating expressed in 1-100 scale @@ -31,9 +30,10 @@ type Gallery struct { CreatedAt time.Time `json:"created_at"` UpdatedAt time.Time `json:"updated_at"` - SceneIDs RelatedIDs `json:"scene_ids"` - TagIDs RelatedIDs `json:"tag_ids"` - PerformerIDs RelatedIDs `json:"performer_ids"` + URLs RelatedStrings `json:"urls"` + SceneIDs RelatedIDs `json:"scene_ids"` + TagIDs RelatedIDs `json:"tag_ids"` + PerformerIDs RelatedIDs `json:"performer_ids"` } func NewGallery() Gallery { @@ -51,7 +51,7 @@ type GalleryPartial struct { // Checksum OptionalString // Zip OptionalBool Title OptionalString - URL OptionalString + URLs *UpdateStrings Date OptionalDate Details OptionalString // Rating expressed in 1-100 scale @@ -81,6 +81,12 @@ func (g *Gallery) IsUserCreated() bool { return g.PrimaryFileID == nil && g.FolderID == nil } +func (g *Gallery) LoadURLs(ctx context.Context, l URLLoader) error { + return g.URLs.load(func() ([]string, error) { + return l.GetURLs(ctx, g.ID) + }) +} + func (g *Gallery) LoadFiles(ctx context.Context, l FileLoader) error { return g.Files.load(func() ([]File, error) { return l.GetFiles(ctx, g.ID) diff --git a/pkg/models/repository_gallery.go b/pkg/models/repository_gallery.go index 64019886cc7..45ad5beb710 100644 --- a/pkg/models/repository_gallery.go +++ b/pkg/models/repository_gallery.go @@ -63,6 +63,7 @@ type GalleryReader interface { GalleryQueryer GalleryCounter + URLLoader FileIDLoader ImageIDLoader SceneIDLoader diff --git a/pkg/scraper/gallery.go b/pkg/scraper/gallery.go index db2c98755dd..db316409bcf 100644 --- a/pkg/scraper/gallery.go +++ b/pkg/scraper/gallery.go @@ -5,18 +5,24 @@ import "github.com/stashapp/stash/pkg/models" type ScrapedGallery struct { Title *string `json:"title"` Details *string `json:"details"` - URL *string `json:"url"` + URLs []string `json:"urls"` Date *string `json:"date"` Studio *models.ScrapedStudio `json:"studio"` Tags []*models.ScrapedTag `json:"tags"` Performers []*models.ScrapedPerformer `json:"performers"` + + // deprecated + URL *string `json:"url"` } func (ScrapedGallery) IsScrapedContent() {} type ScrapedGalleryInput struct { - Title *string `json:"title"` - Details *string `json:"details"` - URL *string `json:"url"` - Date *string `json:"date"` + Title *string `json:"title"` + Details *string `json:"details"` + URLs []string `json:"urls"` + Date *string `json:"date"` + + // deprecated + URL *string `json:"url"` } diff --git a/pkg/scraper/query_url.go b/pkg/scraper/query_url.go index 49cd08cf717..4bae01c06a8 100644 --- a/pkg/scraper/query_url.go +++ b/pkg/scraper/query_url.go @@ -66,8 +66,8 @@ func queryURLParametersFromGallery(gallery *models.Gallery) queryURLParameters { ret["title"] = gallery.Title } - if gallery.URL != "" { - ret["url"] = gallery.URL + if len(gallery.URLs.List()) > 0 { + ret["url"] = gallery.URLs.List()[0] } return ret diff --git a/pkg/scraper/stash.go b/pkg/scraper/stash.go index da204f347e5..b7f483667a3 100644 --- a/pkg/scraper/stash.go +++ b/pkg/scraper/stash.go @@ -354,11 +354,18 @@ func galleryToUpdateInput(gallery *models.Gallery) models.GalleryUpdateInput { // fallback to file basename if title is empty title := gallery.GetTitle() + var url *string + urls := gallery.URLs.List() + if len(urls) > 0 { + url = &urls[0] + } + return models.GalleryUpdateInput{ ID: strconv.Itoa(gallery.ID), Title: &title, Details: &gallery.Details, - URL: &gallery.URL, + URL: url, + Urls: urls, Date: dateToStringPtr(gallery.Date), } } diff --git a/pkg/sqlite/database.go b/pkg/sqlite/database.go index f5291ca1a72..6b3b4171dcf 100644 --- a/pkg/sqlite/database.go +++ b/pkg/sqlite/database.go @@ -33,7 +33,7 @@ const ( dbConnTimeout = 30 ) -var appSchemaVersion uint = 50 +var appSchemaVersion uint = 51 //go:embed migrations/*.sql var migrationsBox embed.FS diff --git a/pkg/sqlite/gallery.go b/pkg/sqlite/gallery.go index 7bdf98bd31a..41729c9ad95 100644 --- a/pkg/sqlite/gallery.go +++ b/pkg/sqlite/gallery.go @@ -26,12 +26,13 @@ const ( galleriesImagesTable = "galleries_images" galleriesScenesTable = "scenes_galleries" galleryIDColumn = "gallery_id" + galleriesURLsTable = "gallery_urls" + galleriesURLColumn = "url" ) type galleryRow struct { ID int `db:"id" goqu:"skipinsert"` Title zero.String `db:"title"` - URL zero.String `db:"url"` Date NullDate `db:"date"` Details zero.String `db:"details"` // expressed as 1-100 @@ -46,7 +47,6 @@ type galleryRow struct { func (r *galleryRow) fromGallery(o models.Gallery) { r.ID = o.ID r.Title = zero.StringFrom(o.Title) - r.URL = zero.StringFrom(o.URL) r.Date = NullDateFromDatePtr(o.Date) r.Details = zero.StringFrom(o.Details) r.Rating = intFromPtr(o.Rating) @@ -70,7 +70,6 @@ func (r *galleryQueryRow) resolve() *models.Gallery { ret := &models.Gallery{ ID: r.ID, Title: r.Title.String, - URL: r.URL.String, Date: r.Date.DatePtr(), Details: r.Details.String, Rating: nullIntPtr(r.Rating), @@ -97,7 +96,6 @@ type galleryRowRecord struct { func (r *galleryRowRecord) fromPartial(o models.GalleryPartial) { r.setNullString("title", o.Title) - r.setNullString("url", o.URL) r.setNullDate("date", o.Date) r.setNullString("details", o.Details) r.setNullInt("rating", o.Rating) @@ -178,6 +176,12 @@ func (qb *GalleryStore) Create(ctx context.Context, newObject *models.Gallery, f } } + if newObject.URLs.Loaded() { + const startPos = 0 + if err := galleriesURLsTableMgr.insertJoins(ctx, id, startPos, newObject.URLs.List()); err != nil { + return err + } + } if newObject.PerformerIDs.Loaded() { if err := galleriesPerformersTableMgr.insertJoins(ctx, id, newObject.PerformerIDs.List()); err != nil { return err @@ -212,6 +216,11 @@ func (qb *GalleryStore) Update(ctx context.Context, updatedObject *models.Galler return err } + if updatedObject.URLs.Loaded() { + if err := galleriesURLsTableMgr.replaceJoins(ctx, updatedObject.ID, updatedObject.URLs.List()); err != nil { + return err + } + } if updatedObject.PerformerIDs.Loaded() { if err := galleriesPerformersTableMgr.replaceJoins(ctx, updatedObject.ID, updatedObject.PerformerIDs.List()); err != nil { return err @@ -257,6 +266,11 @@ func (qb *GalleryStore) UpdatePartial(ctx context.Context, id int, partial model } } + if partial.URLs != nil { + if err := galleriesURLsTableMgr.modifyJoins(ctx, id, partial.URLs.Values, partial.URLs.Mode); err != nil { + return nil, err + } + } if partial.PerformerIDs != nil { if err := galleriesPerformersTableMgr.modifyJoins(ctx, id, partial.PerformerIDs.IDs, partial.PerformerIDs.Mode); err != nil { return nil, err @@ -669,7 +683,7 @@ func (qb *GalleryStore) makeFilter(ctx context.Context, galleryFilter *models.Ga query.handleCriterion(ctx, intCriterionHandler(galleryFilter.Rating100, "galleries.rating", nil)) // legacy rating handler query.handleCriterion(ctx, rating5CriterionHandler(galleryFilter.Rating, "galleries.rating", nil)) - query.handleCriterion(ctx, stringCriterionHandler(galleryFilter.URL, "galleries.url")) + query.handleCriterion(ctx, galleryURLsCriterionHandler(galleryFilter.URL)) query.handleCriterion(ctx, boolCriterionHandler(galleryFilter.Organized, "galleries.organized", nil)) query.handleCriterion(ctx, galleryIsMissingCriterionHandler(qb, galleryFilter.IsMissing)) query.handleCriterion(ctx, galleryTagsCriterionHandler(qb, galleryFilter.Tags)) @@ -793,6 +807,18 @@ func (qb *GalleryStore) QueryCount(ctx context.Context, galleryFilter *models.Ga return query.executeCount(ctx) } +func galleryURLsCriterionHandler(url *models.StringCriterionInput) criterionHandlerFunc { + h := stringListCriterionHandlerBuilder{ + joinTable: galleriesURLsTable, + stringColumn: galleriesURLColumn, + addJoinTable: func(f *filterBuilder) { + galleriesURLsTableMgr.join(f, "", "galleries.id") + }, + } + + return h.handler(url) +} + func (qb *GalleryStore) galleryPathCriterionHandler(c *models.StringCriterionInput) criterionHandlerFunc { return func(ctx context.Context, f *filterBuilder) { if c != nil { @@ -874,6 +900,9 @@ func galleryIsMissingCriterionHandler(qb *GalleryStore, isMissing *string) crite return func(ctx context.Context, f *filterBuilder) { if isMissing != nil && *isMissing != "" { switch *isMissing { + case "url": + galleriesURLsTableMgr.join(f, "", "galleries.id") + f.addWhere("gallery_urls.url IS NULL") case "scenes": f.addLeftJoin("scenes_galleries", "scenes_join", "scenes_join.gallery_id = galleries.id") f.addWhere("scenes_join.gallery_id IS NULL") @@ -1107,6 +1136,10 @@ func (qb *GalleryStore) setGallerySort(query *queryBuilder, findFilter *models.F query.sortAndPagination += ", COALESCE(galleries.title, galleries.id) COLLATE NATURAL_CI ASC" } +func (qb *GalleryStore) GetURLs(ctx context.Context, galleryID int) ([]string, error) { + return galleriesURLsTableMgr.get(ctx, galleryID) +} + func (qb *GalleryStore) filesRepository() *filesRepository { return &filesRepository{ repository: repository{ diff --git a/pkg/sqlite/gallery_test.go b/pkg/sqlite/gallery_test.go index c8dbe02762f..9b1075ecaed 100644 --- a/pkg/sqlite/gallery_test.go +++ b/pkg/sqlite/gallery_test.go @@ -17,6 +17,11 @@ import ( var invalidID = -1 func loadGalleryRelationships(ctx context.Context, expected models.Gallery, actual *models.Gallery) error { + if expected.URLs.Loaded() { + if err := actual.LoadURLs(ctx, db.Gallery); err != nil { + return err + } + } if expected.SceneIDs.Loaded() { if err := actual.LoadSceneIDs(ctx, db.Gallery); err != nil { return err @@ -72,7 +77,7 @@ func Test_galleryQueryBuilder_Create(t *testing.T) { "full", models.Gallery{ Title: title, - URL: url, + URLs: models.NewRelatedStrings([]string{url}), Date: &date, Details: details, Rating: &rating, @@ -90,7 +95,7 @@ func Test_galleryQueryBuilder_Create(t *testing.T) { "with file", models.Gallery{ Title: title, - URL: url, + URLs: models.NewRelatedStrings([]string{url}), Date: &date, Details: details, Rating: &rating, @@ -222,7 +227,7 @@ func Test_galleryQueryBuilder_Update(t *testing.T) { &models.Gallery{ ID: galleryIDs[galleryIdxWithScene], Title: title, - URL: url, + URLs: models.NewRelatedStrings([]string{url}), Date: &date, Details: details, Rating: &rating, @@ -243,6 +248,7 @@ func Test_galleryQueryBuilder_Update(t *testing.T) { "clear nullables", &models.Gallery{ ID: galleryIDs[galleryIdxWithImage], + URLs: models.NewRelatedStrings([]string{}), SceneIDs: models.NewRelatedIDs([]int{}), TagIDs: models.NewRelatedIDs([]int{}), PerformerIDs: models.NewRelatedIDs([]int{}), @@ -384,7 +390,7 @@ func clearGalleryPartial() models.GalleryPartial { return models.GalleryPartial{ Title: models.OptionalString{Set: true, Null: true}, Details: models.OptionalString{Set: true, Null: true}, - URL: models.OptionalString{Set: true, Null: true}, + URLs: &models.UpdateStrings{Mode: models.RelationshipUpdateModeSet}, Date: models.OptionalDate{Set: true, Null: true}, Rating: models.OptionalInt{Set: true, Null: true}, StudioID: models.OptionalInt{Set: true, Null: true}, @@ -416,9 +422,12 @@ func Test_galleryQueryBuilder_UpdatePartial(t *testing.T) { "full", galleryIDs[galleryIdxWithImage], models.GalleryPartial{ - Title: models.NewOptionalString(title), - Details: models.NewOptionalString(details), - URL: models.NewOptionalString(url), + Title: models.NewOptionalString(title), + Details: models.NewOptionalString(details), + URLs: &models.UpdateStrings{ + Values: []string{url}, + Mode: models.RelationshipUpdateModeSet, + }, Date: models.NewOptionalDate(date), Rating: models.NewOptionalInt(rating), Organized: models.NewOptionalBool(true), @@ -443,7 +452,7 @@ func Test_galleryQueryBuilder_UpdatePartial(t *testing.T) { ID: galleryIDs[galleryIdxWithImage], Title: title, Details: details, - URL: url, + URLs: models.NewRelatedStrings([]string{url}), Date: &date, Rating: &rating, Organized: true, @@ -1653,7 +1662,13 @@ func TestGalleryQueryURL(t *testing.T) { verifyFn := func(g *models.Gallery) { t.Helper() - verifyString(t, g.URL, urlCriterion) + urls := g.URLs.List() + var url string + if len(urls) > 0 { + url = urls[0] + } + + verifyString(t, url, urlCriterion) } verifyGalleryQuery(t, filter, verifyFn) @@ -1683,6 +1698,12 @@ func verifyGalleryQuery(t *testing.T, filter models.GalleryFilterType, verifyFn galleries := queryGallery(ctx, t, sqb, &filter, nil) + for _, g := range galleries { + if err := g.LoadURLs(ctx, sqb); err != nil { + t.Errorf("Error loading gallery URLs: %v", err) + } + } + // assume it should find at least one assert.Greater(t, len(galleries), 0) diff --git a/pkg/sqlite/migrations/51_gallery_urls.up.sql b/pkg/sqlite/migrations/51_gallery_urls.up.sql new file mode 100644 index 00000000000..b72ee600c15 --- /dev/null +++ b/pkg/sqlite/migrations/51_gallery_urls.up.sql @@ -0,0 +1,76 @@ +PRAGMA foreign_keys=OFF; + +CREATE TABLE `gallery_urls` ( + `gallery_id` integer NOT NULL, + `position` integer NOT NULL, + `url` varchar(255) NOT NULL, + foreign key(`gallery_id`) references `galleries`(`id`) on delete CASCADE, + PRIMARY KEY(`gallery_id`, `position`, `url`) +); + +CREATE INDEX `gallery_urls_url` on `gallery_urls` (`url`); + +-- drop url +CREATE TABLE `galleries_new` ( + `id` integer not null primary key autoincrement, + `folder_id` integer, + `title` varchar(255), + `date` date, + `details` text, + `studio_id` integer, + `rating` tinyint, + `organized` boolean not null default '0', + `created_at` datetime not null, + `updated_at` datetime not null, + foreign key(`studio_id`) references `studios`(`id`) on delete SET NULL, + foreign key(`folder_id`) references `folders`(`id`) on delete SET NULL +); + +INSERT INTO `galleries_new` + ( + `id`, + `folder_id`, + `title`, + `date`, + `details`, + `studio_id`, + `rating`, + `organized`, + `created_at`, + `updated_at` + ) + SELECT + `id`, + `folder_id`, + `title`, + `date`, + `details`, + `studio_id`, + `rating`, + `organized`, + `created_at`, + `updated_at` + FROM `galleries`; + +INSERT INTO `gallery_urls` + ( + `gallery_id`, + `position`, + `url` + ) + SELECT + `id`, + '0', + `url` + FROM `galleries` + WHERE `galleries`.`url` IS NOT NULL AND `galleries`.`url` != ''; + +DROP INDEX `index_galleries_on_studio_id`; +DROP INDEX `index_galleries_on_folder_id_unique`; +DROP TABLE `galleries`; +ALTER TABLE `galleries_new` rename to `galleries`; + +CREATE INDEX `index_galleries_on_studio_id` on `galleries` (`studio_id`); +CREATE UNIQUE INDEX `index_galleries_on_folder_id_unique` on `galleries` (`folder_id`); + +PRAGMA foreign_keys=ON; diff --git a/pkg/sqlite/scene.go b/pkg/sqlite/scene.go index 215c1740953..e9f1f24dd18 100644 --- a/pkg/sqlite/scene.go +++ b/pkg/sqlite/scene.go @@ -1291,6 +1291,9 @@ func sceneIsMissingCriterionHandler(qb *SceneStore, isMissing *string) criterion return func(ctx context.Context, f *filterBuilder) { if isMissing != nil && *isMissing != "" { switch *isMissing { + case "url": + scenesURLsTableMgr.join(f, "", "scenes.id") + f.addWhere("scene_urls.url IS NULL") case "galleries": qb.galleriesRepository().join(f, "galleries_join", "scenes.id") f.addWhere("galleries_join.scene_id IS NULL") diff --git a/pkg/sqlite/setup_test.go b/pkg/sqlite/setup_test.go index e182ef99b5b..9ca886ac4d2 100644 --- a/pkg/sqlite/setup_test.go +++ b/pkg/sqlite/setup_test.go @@ -1213,7 +1213,16 @@ func getGalleryNullStringValue(index int, field string) sql.NullString { } func getGalleryNullStringPtr(index int, field string) *string { - return getStringPtr(getPrefixedStringValue("gallery", index, field)) + return getStringPtrFromNullString(getPrefixedNullStringValue("gallery", index, field)) +} + +func getGalleryEmptyString(index int, field string) string { + v := getGalleryNullStringPtr(index, field) + if v == nil { + return "" + } + + return *v } func getGalleryBasename(index int) string { @@ -1245,8 +1254,10 @@ func makeGallery(i int, includeScenes bool) *models.Gallery { tids := indexesToIDs(tagIDs, galleryTags[i]) ret := &models.Gallery{ - Title: getGalleryStringValue(i, titleField), - URL: getGalleryNullStringValue(i, urlField).String, + Title: getGalleryStringValue(i, titleField), + URLs: models.NewRelatedStrings([]string{ + getGalleryEmptyString(i, urlField), + }), Rating: getIntPtr(getRating(i)), Date: getObjectDate(i), StudioID: studioID, diff --git a/pkg/sqlite/tables.go b/pkg/sqlite/tables.go index dc1eb505115..3575030337c 100644 --- a/pkg/sqlite/tables.go +++ b/pkg/sqlite/tables.go @@ -19,6 +19,7 @@ var ( galleriesTagsJoinTable = goqu.T(galleriesTagsTable) performersGalleriesJoinTable = goqu.T(performersGalleriesTable) galleriesScenesJoinTable = goqu.T(galleriesScenesTable) + galleriesURLsJoinTable = goqu.T(galleriesURLsTable) scenesFilesJoinTable = goqu.T(scenesFilesTable) scenesTagsJoinTable = goqu.T(scenesTagsTable) @@ -122,6 +123,14 @@ var ( table: goqu.T(galleriesChaptersTable), idColumn: goqu.T(galleriesChaptersTable).Col(idColumn), } + + galleriesURLsTableMgr = &orderedValueTable[string]{ + table: table{ + table: galleriesURLsJoinTable, + idColumn: galleriesURLsJoinTable.Col(galleryIDColumn), + }, + valueColumn: galleriesURLsJoinTable.Col(galleriesURLColumn), + } ) var ( diff --git a/ui/v2.5/src/components/Galleries/GalleryDetails/GalleryEditPanel.tsx b/ui/v2.5/src/components/Galleries/GalleryDetails/GalleryEditPanel.tsx index 1701b5bc7e1..3d40c3c6a78 100644 --- a/ui/v2.5/src/components/Galleries/GalleryDetails/GalleryEditPanel.tsx +++ b/ui/v2.5/src/components/Galleries/GalleryDetails/GalleryEditPanel.tsx @@ -25,7 +25,7 @@ import { } from "src/components/Shared/Select"; import { Icon } from "src/components/Shared/Icon"; import { LoadingIndicator } from "src/components/Shared/LoadingIndicator"; -import { URLField } from "src/components/Shared/URLField"; +import { URLListInput } from "src/components/Shared/URLField"; import { useToast } from "src/hooks/Toast"; import { useFormik } from "formik"; import FormUtils from "src/utils/form"; @@ -42,6 +42,7 @@ import { Performer, PerformerSelect, } from "src/components/Performers/PerformerSelect"; +import { yupDateString, yupUniqueStringList } from "src/utils/yup"; interface IProps { gallery: Partial; @@ -84,20 +85,8 @@ export const GalleryEditPanel: React.FC = ({ const schema = yup.object({ title: titleRequired ? yup.string().required() : yup.string().ensure(), - url: yup.string().ensure(), - date: yup - .string() - .ensure() - .test({ - name: "date", - test: (value) => { - if (!value) return true; - if (!value.match(/^\d{4}-\d{2}-\d{2}$/)) return false; - if (Number.isNaN(Date.parse(value))) return false; - return true; - }, - message: intl.formatMessage({ id: "validation.date_invalid_form" }), - }), + urls: yupUniqueStringList("urls"), + date: yupDateString(intl), rating100: yup.number().nullable().defined(), studio_id: yup.string().required().nullable(), performer_ids: yup.array(yup.string().required()).defined(), @@ -108,7 +97,7 @@ export const GalleryEditPanel: React.FC = ({ const initialValues = { title: gallery?.title ?? "", - url: gallery?.url ?? "", + urls: gallery?.urls ?? [], date: gallery?.date ?? "", rating100: gallery?.rating100 ?? null, studio_id: gallery?.studio?.id ?? null, @@ -313,8 +302,8 @@ export const GalleryEditPanel: React.FC = ({ formik.setFieldValue("date", galleryData.date); } - if (galleryData.url) { - formik.setFieldValue("url", galleryData.url); + if (galleryData.urls) { + formik.setFieldValue("url", galleryData.urls); } if (galleryData.studio?.stored_id) { @@ -351,13 +340,13 @@ export const GalleryEditPanel: React.FC = ({ } } - async function onScrapeGalleryURL() { - if (!formik.values.url) { + async function onScrapeGalleryURL(url: string) { + if (!url) { return; } setIsLoading(true); try { - const result = await queryScrapeGalleryURL(formik.values.url); + const result = await queryScrapeGalleryURL(url); if (!result || !result.data || !result.data.scrapeGalleryURL) { return; } @@ -392,6 +381,14 @@ export const GalleryEditPanel: React.FC = ({ if (isLoading) return ; + const urlsErrors = Array.isArray(formik.errors.urls) + ? formik.errors.urls[0] + : formik.errors.urls; + const urlsErrorMsg = urlsErrors + ? intl.formatMessage({ id: "validation.urls_must_be_unique" }) + : undefined; + const urlsErrorIdx = urlsErrors?.split(" ").map((e) => parseInt(e)); + return (