Merge remote-tracking branch 'upstream/develop' into stash-scraper-fixes

.github/workflows/golangci-lint.yml

@@ -38,7 +38,7 @@ jobs:
         run: docker exec -t build /bin/bash -c "make generate-backend"
 
       - name: Run golangci-lint
-        uses: golangci/golangci-lint-action@v3
+        uses: golangci/golangci-lint-action@v6
         with:
           # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version
           version: latest

internal/manager/json_utils.go

@@ -42,3 +42,7 @@ func (jp *jsonUtils) saveGallery(fn string, gallery *jsonschema.Gallery) error {
 func (jp *jsonUtils) saveFile(fn string, file jsonschema.DirEntry) error {
 	return jsonschema.SaveFileFile(filepath.Join(jp.json.Files, fn), file)
 }
+
+func (jp *jsonUtils) saveSavedFilter(fn string, savedFilter *jsonschema.SavedFilter) error {
+	return jsonschema.SaveSavedFilterFile(filepath.Join(jp.json.SavedFilters, fn), savedFilter)
+}

internal/manager/task_export.go

@@ -23,6 +23,7 @@ import (
 	"github.com/stashapp/stash/pkg/models/jsonschema"
 	"github.com/stashapp/stash/pkg/models/paths"
 	"github.com/stashapp/stash/pkg/performer"
+	"github.com/stashapp/stash/pkg/savedfilter"
 	"github.com/stashapp/stash/pkg/scene"
 	"github.com/stashapp/stash/pkg/sliceutil"
 	"github.com/stashapp/stash/pkg/sliceutil/stringslice"
@@ -176,6 +177,7 @@ func (t *ExportTask) Start(ctx context.Context, wg *sync.WaitGroup) {
 		t.ExportPerformers(ctx, workerCount)
 		t.ExportStudios(ctx, workerCount)
 		t.ExportTags(ctx, workerCount)
+		t.ExportSavedFilters(ctx, workerCount)
 
 		return nil
 	})
@@ -1186,3 +1188,62 @@ func (t *ExportTask) exportGroup(ctx context.Context, wg *sync.WaitGroup, jobCha
 		}
 	}
 }
+
+func (t *ExportTask) ExportSavedFilters(ctx context.Context, workers int) {
+	// don't export saved filters unless we're doing a full export
+	if !t.full {
+		return
+	}
+
+	var wg sync.WaitGroup
+
+	reader := t.repository.SavedFilter
+	var filters []*models.SavedFilter
+	var err error
+	filters, err = reader.All(ctx)
+
+	if err != nil {
+		logger.Errorf("[saved filters] failed to fetch saved filters: %v", err)
+	}
+
+	logger.Info("[saved filters] exporting")
+	startTime := time.Now()
+
+	jobCh := make(chan *models.SavedFilter, workers*2) // make a buffered channel to feed workers
+
+	for w := 0; w < workers; w++ { // create export Saved Filter workers
+		wg.Add(1)
+		go t.exportSavedFilter(ctx, &wg, jobCh)
+	}
+
+	for i, savedFilter := range filters {
+		index := i + 1
+		logger.Progressf("[saved filters] %d of %d", index, len(filters))
+
+		jobCh <- savedFilter // feed workers
+	}
+
+	close(jobCh)
+	wg.Wait()
+
+	logger.Infof("[saved filters] export complete in %s. %d workers used.", time.Since(startTime), workers)
+}
+
+func (t *ExportTask) exportSavedFilter(ctx context.Context, wg *sync.WaitGroup, jobChan <-chan *models.SavedFilter) {
+	defer wg.Done()
+
+	for thisFilter := range jobChan {
+		newJSON, err := savedfilter.ToJSON(ctx, thisFilter)
+
+		if err != nil {
+			logger.Errorf("[saved filter] <%s> error getting saved filter JSON: %v", thisFilter.Name, err)
+			continue
+		}
+
+		fn := newJSON.Filename()
+
+		if err := t.json.saveSavedFilter(fn, newJSON); err != nil {
+			logger.Errorf("[saved filter] <%s> failed to save json: %v", fn, err)
+		}
+	}
+}

internal/manager/task_import.go

@@ -20,6 +20,7 @@ import (
 	"github.com/stashapp/stash/pkg/models/jsonschema"
 	"github.com/stashapp/stash/pkg/models/paths"
 	"github.com/stashapp/stash/pkg/performer"
+	"github.com/stashapp/stash/pkg/savedfilter"
 	"github.com/stashapp/stash/pkg/scene"
 	"github.com/stashapp/stash/pkg/studio"
 	"github.com/stashapp/stash/pkg/tag"
@@ -124,6 +125,7 @@ func (t *ImportTask) Start(ctx context.Context) {
 		}
 	}
 
+	t.ImportSavedFilters(ctx)
 	t.ImportTags(ctx)
 	t.ImportPerformers(ctx)
 	t.ImportStudios(ctx)
@@ -779,3 +781,53 @@ func (t *ImportTask) ImportImages(ctx context.Context) {
 
 	logger.Info("[images] import complete")
 }
+
+func (t *ImportTask) ImportSavedFilters(ctx context.Context) {
+	logger.Info("[saved filters] importing")
+
+	path := t.json.json.SavedFilters
+	files, err := os.ReadDir(path)
+	if err != nil {
+		if !errors.Is(err, os.ErrNotExist) {
+			logger.Errorf("[saved filters] failed to read saved filters directory: %v", err)
+		}
+
+		return
+	}
+
+	r := t.repository
+
+	for i, fi := range files {
+		index := i + 1
+		savedFilterJSON, err := jsonschema.LoadSavedFilterFile(filepath.Join(path, fi.Name()))
+		if err != nil {
+			logger.Errorf("[saved filters] failed to read json: %v", err)
+			continue
+		}
+
+		logger.Progressf("[saved filters] %d of %d", index, len(files))
+
+		if err := r.WithTxn(ctx, func(ctx context.Context) error {
+			return t.importSavedFilter(ctx, savedFilterJSON)
+		}); err != nil {
+			logger.Errorf("[saved filters] <%s> failed to import: %v", fi.Name(), err)
+			continue
+		}
+	}
+
+	logger.Info("[saved filters] import complete")
+}
+
+func (t *ImportTask) importSavedFilter(ctx context.Context, savedFilterJSON *jsonschema.SavedFilter) error {
+	importer := &savedfilter.Importer{
+		ReaderWriter:        t.repository.SavedFilter,
+		Input:               *savedFilterJSON,
+		MissingRefBehaviour: t.MissingRefBehaviour,
+	}
+
+	if err := performImport(ctx, importer, t.DuplicateBehaviour); err != nil {
+		return err
+	}
+
+	return nil
+}

pkg/models/jsonschema/load.go

@@ -0,0 +1,31 @@
+package jsonschema
+
+import (
+	"fmt"
+	"os"
+
+	jsoniter "github.com/json-iterator/go"
+)
+
+func loadFile[T any](filePath string) (*T, error) {
+	var ret T
+	file, err := os.Open(filePath)
+	if err != nil {
+		return nil, err
+	}
+	defer file.Close()
+	var json = jsoniter.ConfigCompatibleWithStandardLibrary
+	jsonParser := json.NewDecoder(file)
+	err = jsonParser.Decode(&ret)
+	if err != nil {
+		return nil, err
+	}
+	return &ret, nil
+}
+
+func saveFile[T any](filePath string, obj *T) error {
+	if obj == nil {
+		return fmt.Errorf("object must not be nil")
+	}
+	return marshalToFile(filePath, obj)
+}

pkg/models/jsonschema/saved_filter.go

@@ -0,0 +1,27 @@
+package jsonschema
+
+import (
+	"github.com/stashapp/stash/pkg/fsutil"
+	"github.com/stashapp/stash/pkg/models"
+)
+
+type SavedFilter struct {
+	Mode         models.FilterMode      `db:"mode" json:"mode"`
+	Name         string                 `db:"name" json:"name"`
+	FindFilter   *models.FindFilterType `json:"find_filter"`
+	ObjectFilter map[string]interface{} `json:"object_filter"`
+	UIOptions    map[string]interface{} `json:"ui_options"`
+}
+
+func (s SavedFilter) Filename() string {
+	ret := fsutil.SanitiseBasename(s.Name + "_" + s.Mode.String())
+	return ret + ".json"
+}
+
+func LoadSavedFilterFile(filePath string) (*SavedFilter, error) {
+	return loadFile[SavedFilter](filePath)
+}
+
+func SaveSavedFilterFile(filePath string, image *SavedFilter) error {
+	return saveFile[SavedFilter](filePath, image)
+}

pkg/models/paths/paths_json.go

@@ -12,14 +12,15 @@ type JSONPaths struct {
 
 	ScrapedFile string
 
-	Performers string
-	Scenes     string
-	Images     string
-	Galleries  string
-	Studios    string
-	Tags       string
-	Groups     string
-	Files      string
+	Performers   string
+	Scenes       string
+	Images       string
+	Galleries    string
+	Studios      string
+	Tags         string
+	Groups       string
+	Files        string
+	SavedFilters string
 }
 
 func newJSONPaths(baseDir string) *JSONPaths {
@@ -34,6 +35,7 @@ func newJSONPaths(baseDir string) *JSONPaths {
 	jp.Groups = filepath.Join(baseDir, "movies")
 	jp.Tags = filepath.Join(baseDir, "tags")
 	jp.Files = filepath.Join(baseDir, "files")
+	jp.SavedFilters = filepath.Join(baseDir, "saved_filters")
 	return &jp
 }
 
@@ -52,6 +54,7 @@ func EmptyJSONDirs(baseDir string) {
 	_ = fsutil.EmptyDir(jsonPaths.Groups)
 	_ = fsutil.EmptyDir(jsonPaths.Tags)
 	_ = fsutil.EmptyDir(jsonPaths.Files)
+	_ = fsutil.EmptyDir(jsonPaths.SavedFilters)
 }
 
 func EnsureJSONDirs(baseDir string) {
@@ -83,4 +86,7 @@ func EnsureJSONDirs(baseDir string) {
 	if err := fsutil.EnsureDir(jsonPaths.Files); err != nil {
 		logger.Warnf("couldn't create directories for Files: %v", err)
 	}
+	if err := fsutil.EnsureDir(jsonPaths.SavedFilters); err != nil {
+		logger.Warnf("couldn't create directories for Saved Filters: %v", err)
+	}
 }

pkg/savedfilter/export.go

@@ -0,0 +1,19 @@
+package savedfilter
+
+import (
+	"context"
+
+	"github.com/stashapp/stash/pkg/models"
+	"github.com/stashapp/stash/pkg/models/jsonschema"
+)
+
+// ToJSON converts a SavedFilter object into its JSON equivalent.
+func ToJSON(ctx context.Context, filter *models.SavedFilter) (*jsonschema.SavedFilter, error) {
+	return &jsonschema.SavedFilter{
+		Name:         filter.Name,
+		Mode:         filter.Mode,
+		FindFilter:   filter.FindFilter,
+		ObjectFilter: filter.ObjectFilter,
+		UIOptions:    filter.UIOptions,
+	}, nil
+}

pkg/savedfilter/export_test.go

@@ -0,0 +1,91 @@
+package savedfilter
+
+import (
+	"github.com/stashapp/stash/pkg/models"
+	"github.com/stashapp/stash/pkg/models/jsonschema"
+	"github.com/stashapp/stash/pkg/models/mocks"
+	"github.com/stretchr/testify/assert"
+
+	"testing"
+)
+
+const (
+	savedFilterID = 1
+	noImageID     = 2
+	errImageID    = 3
+	errAliasID    = 4
+	withParentsID = 5
+	errParentsID  = 6
+)
+
+const (
+	filterName = "testFilter"
+	mode       = models.FilterModeGalleries
+)
+
+var (
+	findFilter   = models.FindFilterType{}
+	objectFilter = make(map[string]interface{})
+	uiOptions    = make(map[string]interface{})
+)
+
+func createSavedFilter(id int) models.SavedFilter {
+	return models.SavedFilter{
+		ID:           id,
+		Name:         filterName,
+		Mode:         mode,
+		FindFilter:   &findFilter,
+		ObjectFilter: objectFilter,
+		UIOptions:    uiOptions,
+	}
+}
+
+func createJSONSavedFilter() *jsonschema.SavedFilter {
+	return &jsonschema.SavedFilter{
+		Name:         filterName,
+		Mode:         mode,
+		FindFilter:   &findFilter,
+		ObjectFilter: objectFilter,
+		UIOptions:    uiOptions,
+	}
+}
+
+type testScenario struct {
+	savedFilter models.SavedFilter
+	expected    *jsonschema.SavedFilter
+	err         bool
+}
+
+var scenarios []testScenario
+
+func initTestTable() {
+	scenarios = []testScenario{
+		{
+			createSavedFilter(savedFilterID),
+			createJSONSavedFilter(),
+			false,
+		},
+	}
+}
+
+func TestToJSON(t *testing.T) {
+	initTestTable()
+
+	db := mocks.NewDatabase()
+
+	for i, s := range scenarios {
+		savedFilter := s.savedFilter
+		json, err := ToJSON(testCtx, &savedFilter)
+
+		switch {
+		case !s.err && err != nil:
+			t.Errorf("[%d] unexpected error: %s", i, err.Error())
+		case s.err && err == nil:
+			t.Errorf("[%d] expected error not returned", i)
+		default:
+			assert.Equal(t, s.expected, json, "[%d]", i)
+		}
+	}
+
+	db.AssertExpectations(t)
+}