From 04435b6f4655f53bf6fb7b3c65f7ce76047722b8 Mon Sep 17 00:00:00 2001 From: WithoutPants <53250216+WithoutPants@users.noreply.github.com> Date: Fri, 4 Aug 2023 17:36:24 +1000 Subject: [PATCH 1/3] Backend changes - ported from scene impl --- graphql/documents/data/image-slim.graphql | 2 +- graphql/documents/data/image.graphql | 2 +- graphql/schema/types/image.graphql | 9 ++- graphql/schema/types/scene.graphql | 2 +- internal/api/resolver_model_image.go | 29 ++++++++ internal/api/resolver_mutation_image.go | 26 ++++++- pkg/image/export.go | 15 +--- pkg/image/export_test.go | 4 +- pkg/image/import.go | 9 ++- pkg/models/image.go | 1 + pkg/models/jsonschema/image.go | 12 ++- pkg/models/jsonschema/scene.go | 4 +- pkg/models/mocks/ImageReaderWriter.go | 23 ++++++ pkg/models/mocks/StudioReaderWriter.go | 8 +- pkg/models/model_image.go | 20 +++-- pkg/sqlite/anonymise.go | 8 +- pkg/sqlite/database.go | 2 +- pkg/sqlite/image.go | 57 ++++++++++++--- pkg/sqlite/image_test.go | 85 ++++++++++++++++++++-- pkg/sqlite/migrations/49_image_urls.up.sql | 70 ++++++++++++++++++ pkg/sqlite/setup_test.go | 23 +++++- pkg/sqlite/tables.go | 9 +++ 22 files changed, 347 insertions(+), 73 deletions(-) create mode 100644 pkg/sqlite/migrations/49_image_urls.up.sql diff --git a/graphql/documents/data/image-slim.graphql b/graphql/documents/data/image-slim.graphql index 9f84904dcfe..1c7784c9ede 100644 --- a/graphql/documents/data/image-slim.graphql +++ b/graphql/documents/data/image-slim.graphql @@ -2,7 +2,7 @@ fragment SlimImageData on Image { id title date - url + urls rating100 organized o_counter diff --git a/graphql/documents/data/image.graphql b/graphql/documents/data/image.graphql index d55a8108121..64c801401e7 100644 --- a/graphql/documents/data/image.graphql +++ b/graphql/documents/data/image.graphql @@ -3,7 +3,7 @@ fragment ImageData on Image { title rating100 date - url + urls organized o_counter created_at diff --git a/graphql/schema/types/image.graphql b/graphql/schema/types/image.graphql index 5d13cbdd6e4..f0307b962ae 100644 --- a/graphql/schema/types/image.graphql +++ b/graphql/schema/types/image.graphql @@ -6,7 +6,8 @@ type Image { rating: Int @deprecated(reason: "Use 1-100 range with rating100") # rating expressed as 1-100 rating100: Int - url: String + url: String @deprecated(reason: "Use urls") + urls: [String!]! date: String o_counter: Int organized: Boolean! @@ -48,7 +49,8 @@ input ImageUpdateInput { # rating expressed as 1-100 rating100: Int organized: Boolean - url: String + url: String @deprecated(reason: "Use urls") + urls: [String!] date: String studio_id: ID @@ -68,7 +70,8 @@ input BulkImageUpdateInput { # rating expressed as 1-100 rating100: Int organized: Boolean - url: String + url: String @deprecated(reason: "Use urls") + urls: BulkUpdateStrings date: String studio_id: ID diff --git a/graphql/schema/types/scene.graphql b/graphql/schema/types/scene.graphql index cb0831b0aea..2a8b1ddf5f2 100644 --- a/graphql/schema/types/scene.graphql +++ b/graphql/schema/types/scene.graphql @@ -41,7 +41,7 @@ type Scene { details: String director: String url: String @deprecated(reason: "Use urls") - urls: [String!] + urls: [String!]! date: String # rating expressed as 1-5 rating: Int @deprecated(reason: "Use 1-100 range with rating100") diff --git a/internal/api/resolver_model_image.go b/internal/api/resolver_model_image.go index 9bfadafc7a4..60c0803cedd 100644 --- a/internal/api/resolver_model_image.go +++ b/internal/api/resolver_model_image.go @@ -231,3 +231,32 @@ func (r *imageResolver) Performers(ctx context.Context, obj *models.Image) (ret ret, errs = loaders.From(ctx).PerformerByID.LoadAll(obj.PerformerIDs.List()) return ret, firstError(errs) } + +func (r *imageResolver) URL(ctx context.Context, obj *models.Image) (*string, error) { + if !obj.URLs.Loaded() { + if err := r.withReadTxn(ctx, func(ctx context.Context) error { + return obj.LoadURLs(ctx, r.repository.Image) + }); err != nil { + return nil, err + } + } + + urls := obj.URLs.List() + if len(urls) == 0 { + return nil, nil + } + + return &urls[0], nil +} + +func (r *imageResolver) Urls(ctx context.Context, obj *models.Image) ([]string, error) { + if !obj.URLs.Loaded() { + if err := r.withReadTxn(ctx, func(ctx context.Context) error { + return obj.LoadURLs(ctx, r.repository.Image) + }); err != nil { + return nil, err + } + } + + return obj.URLs.List(), nil +} diff --git a/internal/api/resolver_mutation_image.go b/internal/api/resolver_mutation_image.go index 6d5c3a88ab5..0bc8db77121 100644 --- a/internal/api/resolver_mutation_image.go +++ b/internal/api/resolver_mutation_image.go @@ -106,7 +106,6 @@ func (r *mutationResolver) imageUpdate(ctx context.Context, input ImageUpdateInp updatedImage.Title = translator.optionalString(input.Title, "title") updatedImage.Rating = translator.ratingConversionOptional(input.Rating, input.Rating100) - updatedImage.URL = translator.optionalString(input.URL, "url") updatedImage.Date, err = translator.optionalDate(input.Date, "date") if err != nil { return nil, fmt.Errorf("converting date: %w", err) @@ -117,6 +116,18 @@ func (r *mutationResolver) imageUpdate(ctx context.Context, input ImageUpdateInp } updatedImage.Organized = translator.optionalBool(input.Organized, "organized") + if translator.hasField("urls") { + updatedImage.URLs = &models.UpdateStrings{ + Values: input.Urls, + Mode: models.RelationshipUpdateModeSet, + } + } else if translator.hasField("url") { + updatedImage.URLs = &models.UpdateStrings{ + Values: []string{*input.URL}, + Mode: models.RelationshipUpdateModeSet, + } + } + if input.PrimaryFileID != nil { primaryFileID, err := strconv.Atoi(*input.PrimaryFileID) if err != nil { @@ -208,7 +219,6 @@ func (r *mutationResolver) BulkImageUpdate(ctx context.Context, input BulkImageU updatedImage.Title = translator.optionalString(input.Title, "title") updatedImage.Rating = translator.ratingConversionOptional(input.Rating, input.Rating100) - updatedImage.URL = translator.optionalString(input.URL, "url") updatedImage.Date, err = translator.optionalDate(input.Date, "date") if err != nil { return nil, fmt.Errorf("converting date: %w", err) @@ -219,6 +229,18 @@ func (r *mutationResolver) BulkImageUpdate(ctx context.Context, input BulkImageU } updatedImage.Organized = translator.optionalBool(input.Organized, "organized") + if translator.hasField("urls") { + updatedImage.URLs = &models.UpdateStrings{ + Values: input.Urls.Values, + Mode: input.Urls.Mode, + } + } else if translator.hasField("url") { + updatedImage.URLs = &models.UpdateStrings{ + Values: []string{*input.URL}, + Mode: models.RelationshipUpdateModeSet, + } + } + if translator.hasField("gallery_ids") { updatedImage.GalleryIDs, err = translateUpdateIDs(input.GalleryIds.Ids, input.GalleryIds.Mode) if err != nil { diff --git a/pkg/image/export.go b/pkg/image/export.go index d67351e8dfb..90b785e7804 100644 --- a/pkg/image/export.go +++ b/pkg/image/export.go @@ -15,7 +15,7 @@ import ( func ToBasicJSON(image *models.Image) *jsonschema.Image { newImageJSON := jsonschema.Image{ Title: image.Title, - URL: image.URL, + URLs: image.URLs.List(), CreatedAt: json.JSONTime{Time: image.CreatedAt}, UpdatedAt: json.JSONTime{Time: image.UpdatedAt}, } @@ -38,19 +38,6 @@ func ToBasicJSON(image *models.Image) *jsonschema.Image { return &newImageJSON } -// func getImageFileJSON(image *models.Image) *jsonschema.ImageFile { -// ret := &jsonschema.ImageFile{} - -// f := image.PrimaryFile() - -// ret.ModTime = json.JSONTime{Time: f.ModTime} -// ret.Size = f.Size -// ret.Width = f.Width -// ret.Height = f.Height - -// return ret -// } - // GetStudioName returns the name of the provided image's studio. It returns an // empty string if there is no studio assigned to the image. func GetStudioName(ctx context.Context, reader studio.Finder, image *models.Image) (string, error) { diff --git a/pkg/image/export_test.go b/pkg/image/export_test.go index 4c46aae9578..cc088c587e2 100644 --- a/pkg/image/export_test.go +++ b/pkg/image/export_test.go @@ -54,7 +54,7 @@ func createFullImage(id int) models.Image { OCounter: ocounter, Rating: &rating, Date: &dateObj, - URL: url, + URLs: models.NewRelatedStrings([]string{url}), Organized: organized, CreatedAt: createTime, UpdatedAt: updateTime, @@ -67,7 +67,7 @@ func createFullJSONImage() *jsonschema.Image { OCounter: ocounter, Rating: rating, Date: date, - URL: url, + URLs: []string{url}, Organized: organized, Files: []string{path}, CreatedAt: json.JSONTime{ diff --git a/pkg/image/import.go b/pkg/image/import.go index 3c1e7ac8b53..4db61df5a9d 100644 --- a/pkg/image/import.go +++ b/pkg/image/import.go @@ -66,8 +66,6 @@ func (i *Importer) PreImport(ctx context.Context) error { func (i *Importer) imageJSONToImage(imageJSON jsonschema.Image) models.Image { newImage := models.Image{ - // Checksum: imageJSON.Checksum, - // Path: i.Path, PerformerIDs: models.NewRelatedIDs([]int{}), TagIDs: models.NewRelatedIDs([]int{}), GalleryIDs: models.NewRelatedIDs([]int{}), @@ -85,9 +83,12 @@ func (i *Importer) imageJSONToImage(imageJSON jsonschema.Image) models.Image { if imageJSON.Rating != 0 { newImage.Rating = &imageJSON.Rating } - if imageJSON.URL != "" { - newImage.URL = imageJSON.URL + if len(imageJSON.URLs) > 0 { + newImage.URLs = models.NewRelatedStrings(imageJSON.URLs) + } else if imageJSON.URL != "" { + newImage.URLs = models.NewRelatedStrings([]string{imageJSON.URL}) } + if imageJSON.Date != "" { d, err := models.ParseDate(imageJSON.Date) if err == nil { diff --git a/pkg/models/image.go b/pkg/models/image.go index 288f6997621..3737cf45611 100644 --- a/pkg/models/image.go +++ b/pkg/models/image.go @@ -115,6 +115,7 @@ type ImageReader interface { Query(ctx context.Context, options ImageQueryOptions) (*ImageQueryResult, error) QueryCount(ctx context.Context, imageFilter *ImageFilterType, findFilter *FindFilterType) (int, error) + URLLoader GalleryIDLoader PerformerIDLoader TagIDLoader diff --git a/pkg/models/jsonschema/image.go b/pkg/models/jsonschema/image.go index 1862ffc8290..7ff0b21621f 100644 --- a/pkg/models/jsonschema/image.go +++ b/pkg/models/jsonschema/image.go @@ -10,10 +10,14 @@ import ( ) type Image struct { - Title string `json:"title,omitempty"` - Studio string `json:"studio,omitempty"` - Rating int `json:"rating,omitempty"` - URL string `json:"url,omitempty"` + Title string `json:"title,omitempty"` + Studio string `json:"studio,omitempty"` + Rating int `json:"rating,omitempty"` + + // deprecated - for import only + URL string `json:"url,omitempty"` + + URLs []string `json:"urls,omitempty"` Date string `json:"date,omitempty"` Organized bool `json:"organized,omitempty"` OCounter int `json:"o_counter,omitempty"` diff --git a/pkg/models/jsonschema/scene.go b/pkg/models/jsonschema/scene.go index 7ebae7a1785..8a081f3b610 100644 --- a/pkg/models/jsonschema/scene.go +++ b/pkg/models/jsonschema/scene.go @@ -42,8 +42,10 @@ type Scene struct { Title string `json:"title,omitempty"` Code string `json:"code,omitempty"` Studio string `json:"studio,omitempty"` + // deprecated - for import only - URL string `json:"url,omitempty"` + URL string `json:"url,omitempty"` + URLs []string `json:"urls,omitempty"` Date string `json:"date,omitempty"` Rating int `json:"rating,omitempty"` diff --git a/pkg/models/mocks/ImageReaderWriter.go b/pkg/models/mocks/ImageReaderWriter.go index f745f8afe27..67cf8ac2abc 100644 --- a/pkg/models/mocks/ImageReaderWriter.go +++ b/pkg/models/mocks/ImageReaderWriter.go @@ -289,6 +289,29 @@ func (_m *ImageReaderWriter) GetTagIDs(ctx context.Context, relatedID int) ([]in return r0, r1 } +// GetURLs provides a mock function with given fields: ctx, relatedID +func (_m *ImageReaderWriter) GetURLs(ctx context.Context, relatedID int) ([]string, error) { + ret := _m.Called(ctx, relatedID) + + var r0 []string + if rf, ok := ret.Get(0).(func(context.Context, int) []string); ok { + r0 = rf(ctx, relatedID) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]string) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, int) error); ok { + r1 = rf(ctx, relatedID) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // IncrementOCounter provides a mock function with given fields: ctx, id func (_m *ImageReaderWriter) IncrementOCounter(ctx context.Context, id int) (int, error) { ret := _m.Called(ctx, id) diff --git a/pkg/models/mocks/StudioReaderWriter.go b/pkg/models/mocks/StudioReaderWriter.go index 56fd6200db7..4fcaf5148bb 100644 --- a/pkg/models/mocks/StudioReaderWriter.go +++ b/pkg/models/mocks/StudioReaderWriter.go @@ -58,13 +58,13 @@ func (_m *StudioReaderWriter) Count(ctx context.Context) (int, error) { return r0, r1 } -// Create provides a mock function with given fields: ctx, input -func (_m *StudioReaderWriter) Create(ctx context.Context, input *models.Studio) error { - ret := _m.Called(ctx, input) +// Create provides a mock function with given fields: ctx, newStudio +func (_m *StudioReaderWriter) Create(ctx context.Context, newStudio *models.Studio) error { + ret := _m.Called(ctx, newStudio) var r0 error if rf, ok := ret.Get(0).(func(context.Context, *models.Studio) error); ok { - r0 = rf(ctx, input) + r0 = rf(ctx, newStudio) } else { r0 = ret.Error(0) } diff --git a/pkg/models/model_image.go b/pkg/models/model_image.go index e025ba0b174..67159f2c699 100644 --- a/pkg/models/model_image.go +++ b/pkg/models/model_image.go @@ -15,12 +15,12 @@ type Image struct { Title string `json:"title"` // Rating expressed in 1-100 scale - Rating *int `json:"rating"` - Organized bool `json:"organized"` - OCounter int `json:"o_counter"` - StudioID *int `json:"studio_id"` - URL string `json:"url"` - Date *Date `json:"date"` + Rating *int `json:"rating"` + Organized bool `json:"organized"` + OCounter int `json:"o_counter"` + StudioID *int `json:"studio_id"` + URLs RelatedStrings `json:"urls"` + Date *Date `json:"date"` // transient - not persisted Files RelatedFiles @@ -38,6 +38,12 @@ type Image struct { PerformerIDs RelatedIDs `json:"performer_ids"` } +func (i *Image) LoadURLs(ctx context.Context, l URLLoader) error { + return i.URLs.load(func() ([]string, error) { + return l.GetURLs(ctx, i.ID) + }) +} + func (i *Image) LoadFiles(ctx context.Context, l FileLoader) error { return i.Files.load(func() ([]file.File, error) { return l.GetFiles(ctx, i.ID) @@ -114,7 +120,7 @@ type ImagePartial struct { Title OptionalString // Rating expressed in 1-100 scale Rating OptionalInt - URL OptionalString + URLs *UpdateStrings Date OptionalDate Organized OptionalBool OCounter OptionalInt diff --git a/pkg/sqlite/anonymise.go b/pkg/sqlite/anonymise.go index d8e6d99d6ad..7e4efd70299 100644 --- a/pkg/sqlite/anonymise.go +++ b/pkg/sqlite/anonymise.go @@ -368,7 +368,6 @@ func (db *Anonymiser) anonymiseImages(ctx context.Context) error { query := dialect.From(table).Select( table.Col(idColumn), table.Col("title"), - table.Col("url"), ).Where(table.Col(idColumn).Gt(lastID)).Limit(1000) gotSome = false @@ -378,20 +377,17 @@ func (db *Anonymiser) anonymiseImages(ctx context.Context) error { var ( id int title sql.NullString - url sql.NullString ) if err := rows.Scan( &id, &title, - &url, ); err != nil { return err } set := goqu.Record{} db.obfuscateNullString(set, "title", title) - db.obfuscateNullString(set, "url", url) if len(set) > 0 { stmt := dialect.Update(table).Set(set).Where(table.Col(idColumn).Eq(id)) @@ -416,6 +412,10 @@ func (db *Anonymiser) anonymiseImages(ctx context.Context) error { } } + if err := db.anonymiseURLs(ctx, goqu.T(imagesURLsTable), "image_id"); err != nil { + return err + } + return nil } diff --git a/pkg/sqlite/database.go b/pkg/sqlite/database.go index 40a2555fd68..4e992dfa1e0 100644 --- a/pkg/sqlite/database.go +++ b/pkg/sqlite/database.go @@ -33,7 +33,7 @@ const ( dbConnTimeout = 30 ) -var appSchemaVersion uint = 48 +var appSchemaVersion uint = 49 //go:embed migrations/*.sql var migrationsBox embed.FS diff --git a/pkg/sqlite/image.go b/pkg/sqlite/image.go index 20e7801d8bc..5e66abe71ce 100644 --- a/pkg/sqlite/image.go +++ b/pkg/sqlite/image.go @@ -25,27 +25,27 @@ const ( performersImagesTable = "performers_images" imagesTagsTable = "images_tags" imagesFilesTable = "images_files" + imagesURLsTable = "image_urls" + imageURLColumn = "url" ) type imageRow struct { ID int `db:"id" goqu:"skipinsert"` Title zero.String `db:"title"` // expressed as 1-100 - Rating null.Int `db:"rating"` - URL zero.String `db:"url"` - Date NullDate `db:"date"` - Organized bool `db:"organized"` - OCounter int `db:"o_counter"` - StudioID null.Int `db:"studio_id,omitempty"` - CreatedAt Timestamp `db:"created_at"` - UpdatedAt Timestamp `db:"updated_at"` + Rating null.Int `db:"rating"` + Date NullDate `db:"date"` + Organized bool `db:"organized"` + OCounter int `db:"o_counter"` + StudioID null.Int `db:"studio_id,omitempty"` + CreatedAt Timestamp `db:"created_at"` + UpdatedAt Timestamp `db:"updated_at"` } func (r *imageRow) fromImage(i models.Image) { r.ID = i.ID r.Title = zero.StringFrom(i.Title) r.Rating = intFromPtr(i.Rating) - r.URL = zero.StringFrom(i.URL) r.Date = NullDateFromDatePtr(i.Date) r.Organized = i.Organized r.OCounter = i.OCounter @@ -67,7 +67,6 @@ func (r *imageQueryRow) resolve() *models.Image { ID: r.ID, Title: r.Title.String, Rating: nullIntPtr(r.Rating), - URL: r.URL.String, Date: r.Date.DatePtr(), Organized: r.Organized, OCounter: r.OCounter, @@ -94,7 +93,6 @@ type imageRowRecord struct { func (r *imageRowRecord) fromPartial(i models.ImagePartial) { r.setNullString("title", i.Title) r.setNullInt("rating", i.Rating) - r.setNullString("url", i.URL) r.setNullDate("date", i.Date) r.setBool("organized", i.Organized) r.setInt("o_counter", i.OCounter) @@ -177,6 +175,13 @@ func (qb *ImageStore) Create(ctx context.Context, newObject *models.ImageCreateI } } + if newObject.URLs.Loaded() { + const startPos = 0 + if err := imagesURLsTableMgr.insertJoins(ctx, id, startPos, newObject.URLs.List()); err != nil { + return err + } + } + if newObject.PerformerIDs.Loaded() { if err := imagesPerformersTableMgr.insertJoins(ctx, id, newObject.PerformerIDs.List()); err != nil { return err @@ -224,6 +229,12 @@ func (qb *ImageStore) UpdatePartial(ctx context.Context, id int, partial models. return nil, err } } + + if partial.URLs != nil { + if err := imagesURLsTableMgr.modifyJoins(ctx, id, partial.URLs.Values, partial.URLs.Mode); err != nil { + return nil, err + } + } if partial.PerformerIDs != nil { if err := imagesPerformersTableMgr.modifyJoins(ctx, id, partial.PerformerIDs.IDs, partial.PerformerIDs.Mode); err != nil { return nil, err @@ -252,6 +263,12 @@ func (qb *ImageStore) Update(ctx context.Context, updatedObject *models.Image) e return err } + if updatedObject.URLs.Loaded() { + if err := imagesURLsTableMgr.replaceJoins(ctx, updatedObject.ID, updatedObject.URLs.List()); err != nil { + return err + } + } + if updatedObject.PerformerIDs.Loaded() { if err := imagesPerformersTableMgr.replaceJoins(ctx, updatedObject.ID, updatedObject.PerformerIDs.List()); err != nil { return err @@ -665,7 +682,7 @@ func (qb *ImageStore) makeFilter(ctx context.Context, imageFilter *models.ImageF query.handleCriterion(ctx, intCriterionHandler(imageFilter.OCounter, "images.o_counter", nil)) query.handleCriterion(ctx, boolCriterionHandler(imageFilter.Organized, "images.organized", nil)) query.handleCriterion(ctx, dateCriterionHandler(imageFilter.Date, "images.date")) - query.handleCriterion(ctx, stringCriterionHandler(imageFilter.URL, "images.url")) + query.handleCriterion(ctx, imageURLsCriterionHandler(imageFilter.URL)) query.handleCriterion(ctx, resolutionCriterionHandler(imageFilter.Resolution, "image_files.height", "image_files.width", qb.addImageFilesTable)) query.handleCriterion(ctx, imageIsMissingCriterionHandler(qb, imageFilter.IsMissing)) @@ -856,6 +873,18 @@ func imageIsMissingCriterionHandler(qb *ImageStore, isMissing *string) criterion } } +func imageURLsCriterionHandler(url *models.StringCriterionInput) criterionHandlerFunc { + h := stringListCriterionHandlerBuilder{ + joinTable: imagesURLsTable, + stringColumn: imageURLColumn, + addJoinTable: func(f *filterBuilder) { + imagesURLsTableMgr.join(f, "", "images.id") + }, + } + + return h.handler(url) +} + func (qb *ImageStore) getMultiCriterionHandlerBuilder(foreignTable, joinTable, foreignFK string, addJoinsFunc func(f *filterBuilder)) multiCriterionHandlerBuilder { return multiCriterionHandlerBuilder{ primaryTable: imageTable, @@ -1098,3 +1127,7 @@ func (qb *ImageStore) UpdateTags(ctx context.Context, imageID int, tagIDs []int) // Delete the existing joins and then create new ones return qb.tagsRepository().replace(ctx, imageID, tagIDs) } + +func (qb *ImageStore) GetURLs(ctx context.Context, imageID int) ([]string, error) { + return imagesURLsTableMgr.get(ctx, imageID) +} diff --git a/pkg/sqlite/image_test.go b/pkg/sqlite/image_test.go index 4f3ebcc22ce..8f784324013 100644 --- a/pkg/sqlite/image_test.go +++ b/pkg/sqlite/image_test.go @@ -16,6 +16,11 @@ import ( ) func loadImageRelationships(ctx context.Context, expected models.Image, actual *models.Image) error { + if expected.URLs.Loaded() { + if err := actual.LoadURLs(ctx, db.Image); err != nil { + return err + } + } if expected.GalleryIDs.Loaded() { if err := actual.LoadGalleryIDs(ctx, db.Image); err != nil { return err @@ -75,7 +80,7 @@ func Test_imageQueryBuilder_Create(t *testing.T) { Title: title, Rating: &rating, Date: &date, - URL: url, + URLs: models.NewRelatedStrings([]string{url}), Organized: true, OCounter: ocounter, StudioID: &studioIDs[studioIdxWithImage], @@ -93,7 +98,7 @@ func Test_imageQueryBuilder_Create(t *testing.T) { Title: title, Rating: &rating, Date: &date, - URL: url, + URLs: models.NewRelatedStrings([]string{url}), Organized: true, OCounter: ocounter, StudioID: &studioIDs[studioIdxWithImage], @@ -233,7 +238,7 @@ func Test_imageQueryBuilder_Update(t *testing.T) { ID: imageIDs[imageIdxWithGallery], Title: title, Rating: &rating, - URL: url, + URLs: models.NewRelatedStrings([]string{url}), Date: &date, Organized: true, OCounter: ocounter, @@ -382,7 +387,7 @@ func clearImagePartial() models.ImagePartial { return models.ImagePartial{ Title: models.OptionalString{Set: true, Null: true}, Rating: models.OptionalInt{Set: true, Null: true}, - URL: models.OptionalString{Set: true, Null: true}, + URLs: &models.UpdateStrings{Mode: models.RelationshipUpdateModeSet}, Date: models.OptionalDate{Set: true, Null: true}, StudioID: models.OptionalInt{Set: true, Null: true}, GalleryIDs: &models.UpdateIDs{Mode: models.RelationshipUpdateModeSet}, @@ -413,9 +418,12 @@ func Test_imageQueryBuilder_UpdatePartial(t *testing.T) { "full", imageIDs[imageIdx1WithGallery], models.ImagePartial{ - Title: models.NewOptionalString(title), - Rating: models.NewOptionalInt(rating), - URL: models.NewOptionalString(url), + Title: models.NewOptionalString(title), + Rating: models.NewOptionalInt(rating), + URLs: &models.UpdateStrings{ + Values: []string{url}, + Mode: models.RelationshipUpdateModeSet, + }, Date: models.NewOptionalDate(date), Organized: models.NewOptionalBool(true), OCounter: models.NewOptionalInt(ocounter), @@ -439,7 +447,7 @@ func Test_imageQueryBuilder_UpdatePartial(t *testing.T) { ID: imageIDs[imageIdx1WithGallery], Title: title, Rating: &rating, - URL: url, + URLs: models.NewRelatedStrings([]string{url}), Date: &date, Organized: true, OCounter: ocounter, @@ -1523,6 +1531,67 @@ func imageQueryQ(ctx context.Context, t *testing.T, sqb models.ImageReader, q st assert.Len(t, images, totalImages) } +func verifyImageQuery(t *testing.T, filter models.ImageFilterType, verifyFn func(ctx context.Context, s *models.Image)) { + t.Helper() + withTxn(func(ctx context.Context) error { + t.Helper() + sqb := db.Image + + images := queryImages(ctx, t, sqb, &filter, nil) + + // assume it should find at least one + assert.Greater(t, len(images), 0) + + for _, image := range images { + verifyFn(ctx, image) + } + + return nil + }) +} + +func TestImageQueryURL(t *testing.T) { + const imageIdx = 1 + imageURL := getImageStringValue(imageIdx, urlField) + urlCriterion := models.StringCriterionInput{ + Value: imageURL, + Modifier: models.CriterionModifierEquals, + } + filter := models.ImageFilterType{ + URL: &urlCriterion, + } + + verifyFn := func(ctx context.Context, o *models.Image) { + t.Helper() + + if err := o.LoadURLs(ctx, db.Image); err != nil { + t.Errorf("Error loading scene URLs: %v", err) + } + + urls := o.URLs.List() + var url string + if len(urls) > 0 { + url = urls[0] + } + + verifyString(t, url, urlCriterion) + } + + verifyImageQuery(t, filter, verifyFn) + urlCriterion.Modifier = models.CriterionModifierNotEquals + verifyImageQuery(t, filter, verifyFn) + urlCriterion.Modifier = models.CriterionModifierMatchesRegex + urlCriterion.Value = "image_.*1_URL" + verifyImageQuery(t, filter, verifyFn) + urlCriterion.Modifier = models.CriterionModifierNotMatchesRegex + verifyImageQuery(t, filter, verifyFn) + urlCriterion.Modifier = models.CriterionModifierIsNull + urlCriterion.Value = "" + verifyImageQuery(t, filter, verifyFn) + urlCriterion.Modifier = models.CriterionModifierNotNull + verifyImageQuery(t, filter, verifyFn) +} + func TestImageQueryPath(t *testing.T) { const imageIdx = 1 imagePath := getFilePath(folderIdxWithImageFiles, getImageBasename(imageIdx)) diff --git a/pkg/sqlite/migrations/49_image_urls.up.sql b/pkg/sqlite/migrations/49_image_urls.up.sql new file mode 100644 index 00000000000..47ff373075b --- /dev/null +++ b/pkg/sqlite/migrations/49_image_urls.up.sql @@ -0,0 +1,70 @@ +PRAGMA foreign_keys=OFF; + +CREATE TABLE `image_urls` ( + `image_id` integer NOT NULL, + `position` integer NOT NULL, + `url` varchar(255) NOT NULL, + foreign key(`image_id`) references `images`(`id`) on delete CASCADE, + PRIMARY KEY(`image_id`, `position`, `url`) +); + +CREATE INDEX `image_urls_url` on `image_urls` (`url`); + +-- drop url +CREATE TABLE "images_new" ( + `id` integer not null primary key autoincrement, + `title` varchar(255), + `rating` tinyint, + `studio_id` integer, + `o_counter` tinyint not null default 0, + `organized` boolean not null default '0', + `created_at` datetime not null, + `updated_at` datetime not null, + `date` date, + foreign key(`studio_id`) references `studios`(`id`) on delete SET NULL +); + +INSERT INTO `images_new` + ( + `id`, + `title`, + `rating`, + `studio_id`, + `o_counter`, + `organized`, + `created_at`, + `updated_at`, + `date` + ) + SELECT + `id`, + `title`, + `rating`, + `studio_id`, + `o_counter`, + `organized`, + `created_at`, + `updated_at`, + `date` + FROM `images`; + +INSERT INTO `image_urls` + ( + `image_id`, + `position`, + `url` + ) + SELECT + `id`, + '0', + `url` + FROM `images` + WHERE `images`.`url` IS NOT NULL AND `images`.`url` != ''; + +DROP INDEX `index_images_on_studio_id`; +DROP TABLE `images`; +ALTER TABLE `images_new` rename to `images`; + +CREATE INDEX `index_images_on_studio_id` on `images` (`studio_id`); + +PRAGMA foreign_keys=ON; diff --git a/pkg/sqlite/setup_test.go b/pkg/sqlite/setup_test.go index c57f272c7d4..e1583c43d6c 100644 --- a/pkg/sqlite/setup_test.go +++ b/pkg/sqlite/setup_test.go @@ -1114,6 +1114,19 @@ func getImageStringValue(index int, field string) string { return fmt.Sprintf("image_%04d_%s", index, field) } +func getImageNullStringPtr(index int, field string) *string { + return getStringPtrFromNullString(getPrefixedNullStringValue("image", index, field)) +} + +func getImageEmptyString(index int, field string) string { + v := getImageNullStringPtr(index, field) + if v == nil { + return "" + } + + return *v +} + func getImageBasename(index int) string { return getImageStringValue(index, pathField) } @@ -1149,10 +1162,12 @@ func makeImage(i int) *models.Image { tids := indexesToIDs(tagIDs, imageTags[i]) return &models.Image{ - Title: title, - Rating: getIntPtr(getRating(i)), - Date: getObjectDate(i), - URL: getImageStringValue(i, urlField), + Title: title, + Rating: getIntPtr(getRating(i)), + Date: getObjectDate(i), + URLs: models.NewRelatedStrings([]string{ + getImageEmptyString(i, urlField), + }), OCounter: getOCounter(i), StudioID: studioID, GalleryIDs: models.NewRelatedIDs(gids), diff --git a/pkg/sqlite/tables.go b/pkg/sqlite/tables.go index 69dc1d6a89f..dc1eb505115 100644 --- a/pkg/sqlite/tables.go +++ b/pkg/sqlite/tables.go @@ -13,6 +13,7 @@ var ( imagesTagsJoinTable = goqu.T(imagesTagsTable) performersImagesJoinTable = goqu.T(performersImagesTable) imagesFilesJoinTable = goqu.T(imagesFilesTable) + imagesURLsJoinTable = goqu.T(imagesURLsTable) galleriesFilesJoinTable = goqu.T(galleriesFilesTable) galleriesTagsJoinTable = goqu.T(galleriesTagsTable) @@ -70,6 +71,14 @@ var ( }, fkColumn: performersImagesJoinTable.Col(performerIDColumn), } + + imagesURLsTableMgr = &orderedValueTable[string]{ + table: table{ + table: imagesURLsJoinTable, + idColumn: imagesURLsJoinTable.Col(imageIDColumn), + }, + valueColumn: imagesURLsJoinTable.Col(imageURLColumn), + } ) var ( From cb5ac1397a92929e8284ee01ca2e3a820c2fcce0 Mon Sep 17 00:00:00 2001 From: WithoutPants <53250216+WithoutPants@users.noreply.github.com> Date: Fri, 4 Aug 2023 19:27:07 +1000 Subject: [PATCH 2/3] Front end changes --- internal/manager/task_export.go | 5 +++ .../Images/ImageDetails/ImageEditPanel.tsx | 45 +++++++++---------- .../ImageDetails/ImageFileInfoPanel.tsx | 19 +++----- .../Scenes/SceneDetails/SceneEditPanel.tsx | 35 ++------------- ui/v2.5/src/components/Shared/URLField.tsx | 32 +++++++------ ui/v2.5/src/utils/yup.ts | 40 +++++++++++++++++ 6 files changed, 92 insertions(+), 84 deletions(-) create mode 100644 ui/v2.5/src/utils/yup.ts diff --git a/internal/manager/task_export.go b/internal/manager/task_export.go index f186d3eb48d..222848d3421 100644 --- a/internal/manager/task_export.go +++ b/internal/manager/task_export.go @@ -648,6 +648,11 @@ func exportImage(ctx context.Context, wg *sync.WaitGroup, jobChan <-chan *models continue } + if err := s.LoadURLs(ctx, repo.Image); err != nil { + logger.Errorf("[images] <%s> error getting image urls: %s", imageHash, err.Error()) + continue + } + newImageJSON := image.ToBasicJSON(s) // export files diff --git a/ui/v2.5/src/components/Images/ImageDetails/ImageEditPanel.tsx b/ui/v2.5/src/components/Images/ImageDetails/ImageEditPanel.tsx index 23e1a899679..966221387e7 100644 --- a/ui/v2.5/src/components/Images/ImageDetails/ImageEditPanel.tsx +++ b/ui/v2.5/src/components/Images/ImageDetails/ImageEditPanel.tsx @@ -10,7 +10,7 @@ import { StudioSelect, } from "src/components/Shared/Select"; import { LoadingIndicator } from "src/components/Shared/LoadingIndicator"; -import { URLField } from "src/components/Shared/URLField"; +import { URLListInput } from "src/components/Shared/URLField"; import { useToast } from "src/hooks/Toast"; import FormUtils from "src/utils/form"; import { useFormik } from "formik"; @@ -20,6 +20,7 @@ import { useRatingKeybinds } from "src/hooks/keybinds"; import { ConfigurationContext } from "src/hooks/Config"; import isEqual from "lodash-es/isEqual"; import { DateInput } from "src/components/Shared/DateInput"; +import { yupDateString, yupUniqueStringList } from "src/utils/yup"; interface IProps { image: GQL.ImageDataFragment; @@ -44,20 +45,8 @@ export const ImageEditPanel: React.FC = ({ const schema = yup.object({ title: yup.string().ensure(), - url: yup.string().ensure(), - date: yup - .string() - .ensure() - .test({ - name: "date", - test: (value) => { - if (!value) return true; - if (!value.match(/^\d{4}-\d{2}-\d{2}$/)) return false; - if (Number.isNaN(Date.parse(value))) return false; - return true; - }, - message: intl.formatMessage({ id: "validation.date_invalid_form" }), - }), + urls: yupUniqueStringList("urls"), + date: yupDateString(intl), rating100: yup.number().nullable().defined(), studio_id: yup.string().required().nullable(), performer_ids: yup.array(yup.string().required()).defined(), @@ -66,7 +55,7 @@ export const ImageEditPanel: React.FC = ({ const initialValues = { title: image.title ?? "", - url: image?.url ?? "", + urls: image?.urls ?? [], date: image?.date ?? "", rating100: image.rating100 ?? null, studio_id: image.studio?.id ?? null, @@ -148,6 +137,14 @@ export const ImageEditPanel: React.FC = ({ if (isLoading) return ; + const urlsErrors = Array.isArray(formik.errors.urls) + ? formik.errors.urls[0] + : formik.errors.urls; + const urlsErrorMsg = urlsErrors + ? intl.formatMessage({ id: "validation.urls_must_be_unique" }) + : undefined; + const urlsErrorIdx = urlsErrors?.split(" ").map((e) => parseInt(e)); + return (
= ({
{renderTextField("title", intl.formatMessage({ id: "title" }))} - + - + - {}} - urlScrapable={() => { - return false; - }} - isInvalid={!!formik.getFieldMeta("url").error} + formik.setFieldValue("urls", value)} + errors={urlsErrorMsg} + errorIdx={urlsErrorIdx} /> diff --git a/ui/v2.5/src/components/Images/ImageDetails/ImageFileInfoPanel.tsx b/ui/v2.5/src/components/Images/ImageDetails/ImageFileInfoPanel.tsx index 2b906c6d5ef..adf95d2f9f2 100644 --- a/ui/v2.5/src/components/Images/ImageDetails/ImageFileInfoPanel.tsx +++ b/ui/v2.5/src/components/Images/ImageDetails/ImageFileInfoPanel.tsx @@ -7,7 +7,7 @@ import * as GQL from "src/core/generated-graphql"; import { mutateImageSetPrimaryFile } from "src/core/StashService"; import { useToast } from "src/hooks/Toast"; import TextUtils from "src/utils/text"; -import { TextField, URLField } from "src/utils/field"; +import { TextField, URLField, URLsField } from "src/utils/field"; interface IFileInfoPanelProps { file: GQL.ImageFileDataFragment | GQL.VideoFileDataFragment; @@ -120,20 +120,11 @@ export const ImageFileInfoPanel: React.FC = ( if (props.image.visual_files.length === 1) { return ( <> - +
+ +
- {props.image.url ? ( -
- -
- ) : ( - "" - )} + ); } diff --git a/ui/v2.5/src/components/Scenes/SceneDetails/SceneEditPanel.tsx b/ui/v2.5/src/components/Scenes/SceneDetails/SceneEditPanel.tsx index 15989fa3cb6..379d8e41871 100644 --- a/ui/v2.5/src/components/Scenes/SceneDetails/SceneEditPanel.tsx +++ b/ui/v2.5/src/components/Scenes/SceneDetails/SceneEditPanel.tsx @@ -51,6 +51,7 @@ import { useRatingKeybinds } from "src/hooks/keybinds"; import { lazyComponent } from "src/utils/lazyComponent"; import isEqual from "lodash-es/isEqual"; import { DateInput } from "src/components/Shared/DateInput"; +import { yupDateString, yupUniqueStringList } from "src/utils/yup"; const SceneScrapeDialog = lazyComponent(() => import("./SceneScrapeDialog")); const SceneQueryModal = lazyComponent(() => import("./SceneQueryModal")); @@ -106,38 +107,8 @@ export const SceneEditPanel: React.FC = ({ const schema = yup.object({ title: yup.string().ensure(), code: yup.string().ensure(), - urls: yup - .array(yup.string().required()) - .defined() - .test({ - name: "unique", - test: (value) => { - const dupes = value - .map((e, i, a) => { - if (a.indexOf(e) !== i) { - return String(i - 1); - } else { - return null; - } - }) - .filter((e) => e !== null) as string[]; - if (dupes.length === 0) return true; - return new yup.ValidationError(dupes.join(" "), value, "urls"); - }, - }), - date: yup - .string() - .ensure() - .test({ - name: "date", - test: (value) => { - if (!value) return true; - if (!value.match(/^\d{4}-\d{2}-\d{2}$/)) return false; - if (Number.isNaN(Date.parse(value))) return false; - return true; - }, - message: intl.formatMessage({ id: "validation.date_invalid_form" }), - }), + urls: yupUniqueStringList("urls"), + date: yupDateString(intl), director: yup.string().ensure(), rating100: yup.number().nullable().defined(), gallery_ids: yup.array(yup.string().required()).defined(), diff --git a/ui/v2.5/src/components/Shared/URLField.tsx b/ui/v2.5/src/components/Shared/URLField.tsx index 9cea50e7c7a..ca84bd08999 100644 --- a/ui/v2.5/src/components/Shared/URLField.tsx +++ b/ui/v2.5/src/components/Shared/URLField.tsx @@ -50,8 +50,8 @@ export const URLField: React.FC = (props: IProps) => { }; interface IURLListProps extends IStringListInputProps { - onScrapeClick(url: string): void; - urlScrapable(url: string): boolean; + onScrapeClick?: (url: string) => void; + urlScrapable?: (url: string) => boolean; } export const URLListInput: React.FC = ( @@ -64,17 +64,23 @@ export const URLListInput: React.FC = ( {...listProps} placeholder={intl.formatMessage({ id: "url" })} inputComponent={StringInput} - appendComponent={(props) => ( - - )} + appendComponent={(props) => { + if (!onScrapeClick || !urlScrapable) { + return <>; + } + + return ( + + ); + }} /> ); }; diff --git a/ui/v2.5/src/utils/yup.ts b/ui/v2.5/src/utils/yup.ts new file mode 100644 index 00000000000..d93bc5d3e57 --- /dev/null +++ b/ui/v2.5/src/utils/yup.ts @@ -0,0 +1,40 @@ +import { IntlShape } from "react-intl"; +import * as yup from "yup"; + +export function yupUniqueStringList(fieldName: string) { + return yup + .array(yup.string().required()) + .defined() + .test({ + name: "unique", + test: (value) => { + const dupes = value + .map((e, i, a) => { + if (a.indexOf(e) !== i) { + return String(i - 1); + } else { + return null; + } + }) + .filter((e) => e !== null) as string[]; + if (dupes.length === 0) return true; + return new yup.ValidationError(dupes.join(" "), value, fieldName); + }, + }); +} + +export function yupDateString(intl: IntlShape) { + return yup + .string() + .ensure() + .test({ + name: "date", + test: (value) => { + if (!value) return true; + if (!value.match(/^\d{4}-\d{2}-\d{2}$/)) return false; + if (Number.isNaN(Date.parse(value))) return false; + return true; + }, + message: intl.formatMessage({ id: "validation.date_invalid_form" }), + }); +} From ae5f5e8b41fe5288feac67909ca21c469f7e4a46 Mon Sep 17 00:00:00 2001 From: WithoutPants <53250216+WithoutPants@users.noreply.github.com> Date: Tue, 12 Sep 2023 13:01:03 +1000 Subject: [PATCH 3/3] Refactor URL mutation code --- internal/api/changeset_translator.go | 40 +++++++++++++++++++++++++ internal/api/resolver_mutation_image.go | 26 ++-------------- internal/api/resolver_mutation_scene.go | 22 ++------------ 3 files changed, 44 insertions(+), 44 deletions(-) diff --git a/internal/api/changeset_translator.go b/internal/api/changeset_translator.go index 6d2590a3da4..0472edc4c76 100644 --- a/internal/api/changeset_translator.go +++ b/internal/api/changeset_translator.go @@ -307,6 +307,46 @@ func (t changesetTranslator) updateIdsBulk(value *BulkUpdateIds, field string) ( }, nil } +func (t changesetTranslator) optionalURLs(value []string, legacyValue *string) *models.UpdateStrings { + const ( + legacyField = "url" + field = "urls" + ) + + // prefer urls over url + if t.hasField(field) { + return t.updateStrings(value, field) + } else if t.hasField(legacyField) { + var valueSlice []string + if legacyValue != nil { + valueSlice = []string{*legacyValue} + } + return t.updateStrings(valueSlice, legacyField) + } + + return nil +} + +func (t changesetTranslator) optionalURLsBulk(value *BulkUpdateStrings, legacyValue *string) *models.UpdateStrings { + const ( + legacyField = "url" + field = "urls" + ) + + // prefer urls over url + if t.hasField("urls") { + return t.updateStringsBulk(value, field) + } else if t.hasField(legacyField) { + var valueSlice []string + if legacyValue != nil { + valueSlice = []string{*legacyValue} + } + return t.updateStrings(valueSlice, legacyField) + } + + return nil +} + func (t changesetTranslator) updateStrings(value []string, field string) *models.UpdateStrings { if !t.hasField(field) { return nil diff --git a/internal/api/resolver_mutation_image.go b/internal/api/resolver_mutation_image.go index f770b8ba44b..8b2cf447831 100644 --- a/internal/api/resolver_mutation_image.go +++ b/internal/api/resolver_mutation_image.go @@ -119,18 +119,7 @@ func (r *mutationResolver) imageUpdate(ctx context.Context, input ImageUpdateInp return nil, fmt.Errorf("converting studio id: %w", err) } - // prefer urls over url - if translator.hasField("urls") { - updatedImage.URLs = &models.UpdateStrings{ - Values: input.Urls, - Mode: models.RelationshipUpdateModeSet, - } - } else if translator.hasField("url") { - updatedImage.URLs = &models.UpdateStrings{ - Values: []string{*input.URL}, - Mode: models.RelationshipUpdateModeSet, - } - } + updatedImage.URLs = translator.optionalURLs(input.Urls, input.URL) updatedImage.PrimaryFileID, err = translator.fileIDPtrFromString(input.PrimaryFileID) if err != nil { @@ -226,18 +215,7 @@ func (r *mutationResolver) BulkImageUpdate(ctx context.Context, input BulkImageU return nil, fmt.Errorf("converting studio id: %w", err) } - // prefer urls over url - if translator.hasField("urls") { - updatedImage.URLs = &models.UpdateStrings{ - Values: input.Urls.Values, - Mode: input.Urls.Mode, - } - } else if translator.hasField("url") { - updatedImage.URLs = &models.UpdateStrings{ - Values: []string{*input.URL}, - Mode: models.RelationshipUpdateModeSet, - } - } + updatedImage.URLs = translator.optionalURLsBulk(input.Urls, input.URL) updatedImage.GalleryIDs, err = translator.updateIdsBulk(input.GalleryIds, "gallery_ids") if err != nil { diff --git a/internal/api/resolver_mutation_scene.go b/internal/api/resolver_mutation_scene.go index 7652827f33f..782d0aff0af 100644 --- a/internal/api/resolver_mutation_scene.go +++ b/internal/api/resolver_mutation_scene.go @@ -186,16 +186,7 @@ func scenePartialFromInput(input models.SceneUpdateInput, translator changesetTr return nil, fmt.Errorf("converting studio id: %w", err) } - // prefer urls over url - if translator.hasField("urls") { - updatedScene.URLs = translator.updateStrings(input.Urls, "urls") - } else if translator.hasField("url") { - var urls []string - if input.URL != nil { - urls = []string{*input.URL} - } - updatedScene.URLs = translator.updateStrings(urls, "url") - } + updatedScene.URLs = translator.optionalURLs(input.Urls, input.URL) updatedScene.PrimaryFileID, err = translator.fileIDPtrFromString(input.PrimaryFileID) if err != nil { @@ -342,16 +333,7 @@ func (r *mutationResolver) BulkSceneUpdate(ctx context.Context, input BulkSceneU return nil, fmt.Errorf("converting studio id: %w", err) } - // prefer urls over url - if translator.hasField("urls") { - updatedScene.URLs = translator.updateStringsBulk(input.Urls, "urls") - } else if translator.hasField("url") { - var urls []string - if input.URL != nil { - urls = []string{*input.URL} - } - updatedScene.URLs = translator.updateStrings(urls, "url") - } + updatedScene.URLs = translator.optionalURLsBulk(input.Urls, input.URL) updatedScene.PerformerIDs, err = translator.updateIdsBulk(input.PerformerIds, "performer_ids") if err != nil {