Scraper and plugin manager (#4242)

* Add package manager
* Add SettingModal validate
* Reverse modal button order
* Add plugin package management
* Refactor ClearableInput

.gitignore

@@ -21,11 +21,6 @@ vendor
 # GraphQL generated output
 internal/api/generated_*.go
 
-####
-# Jetbrains
-####
-
-
 ####
 # Visual Studio
 ####
@@ -52,9 +47,6 @@ internal/api/generated_*.go
 .idea/**/uiDesigner.xml
 .idea/**/dbnavigator.xml
 
-# Goland Junk
-pkg/pkg
-
 ####
 # Random
 ####

graphql/documents/data/config.graphql

@@ -53,6 +53,17 @@ fragment ConfigGeneralData on ConfigGeneralResult {
   liveTranscodeInputArgs
   liveTranscodeOutputArgs
   drawFunscriptHeatmapRange
+
+  scraperPackageSources {
+    name
+    url
+    local_path
+  }
+  pluginPackageSources {
+    name
+    url
+    local_path
+  }
 }
 
 fragment ConfigInterfaceData on ConfigInterfaceResult {

graphql/documents/data/package.graphql

@@ -0,0 +1,8 @@
+fragment PackageData on Package {
+  package_id
+  name
+  version
+  date
+  metadata
+  sourceURL
+}

graphql/documents/mutations/plugins.graphql

@@ -17,3 +17,15 @@ mutation ConfigurePlugin($plugin_id: ID!, $input: Map!) {
 mutation SetPluginsEnabled($enabledMap: BoolMap!) {
   setPluginsEnabled(enabledMap: $enabledMap)
 }
+
+mutation InstallPluginPackages($packages: [PackageSpecInput!]!) {
+  installPackages(type: Plugin, packages: $packages)
+}
+
+mutation UpdatePluginPackages($packages: [PackageSpecInput!]!) {
+  updatePackages(type: Plugin, packages: $packages)
+}
+
+mutation UninstallPluginPackages($packages: [PackageSpecInput!]!) {
+  uninstallPackages(type: Plugin, packages: $packages)
+}

graphql/documents/mutations/scrapers.graphql

@@ -1,3 +1,15 @@
 mutation ReloadScrapers {
   reloadScrapers
 }
+
+mutation InstallScraperPackages($packages: [PackageSpecInput!]!) {
+  installPackages(type: Scraper, packages: $packages)
+}
+
+mutation UpdateScraperPackages($packages: [PackageSpecInput!]!) {
+  updatePackages(type: Scraper, packages: $packages)
+}
+
+mutation UninstallScraperPackages($packages: [PackageSpecInput!]!) {
+  uninstallPackages(type: Scraper, packages: $packages)
+}

graphql/documents/queries/plugins.graphql

@@ -43,3 +43,27 @@ query PluginTasks {
     }
   }
 }
+
+query InstalledPluginPackages {
+  installedPackages(type: Plugin) {
+    ...PackageData
+  }
+}
+
+query InstalledPluginPackagesStatus {
+  installedPackages(type: Plugin) {
+    ...PackageData
+    upgrade {
+      ...PackageData
+    }
+  }
+}
+
+query AvailablePluginPackages($source: String!) {
+  availablePackages(source: $source, type: Plugin) {
+    ...PackageData
+    requires {
+      package_id
+    }
+  }
+}

graphql/documents/queries/scrapers/scrapers.graphql

@@ -119,3 +119,27 @@ query ScrapeMovieURL($url: String!) {
     ...ScrapedMovieData
   }
 }
+
+query InstalledScraperPackages {
+  installedPackages(type: Scraper) {
+    ...PackageData
+  }
+}
+
+query InstalledScraperPackagesStatus {
+  installedPackages(type: Scraper) {
+    ...PackageData
+    upgrade {
+      ...PackageData
+    }
+  }
+}
+
+query AvailableScraperPackages($source: String!) {
+  availablePackages(source: $source, type: Scraper) {
+    ...PackageData
+    requires {
+      package_id
+    }
+  }
+}

graphql/schema/schema.graphql

@@ -168,6 +168,12 @@ type Query {
   "List available plugin operations"
   pluginTasks: [PluginTask!]
 
+  # Packages
+  "List installed packages"
+  installedPackages(type: PackageType!): [Package!]!
+  "List available packages"
+  availablePackages(type: PackageType!, source: String!): [Package!]!
+
   # Config
   "Returns the current, complete configuration"
   configuration: ConfigResult!
@@ -381,6 +387,29 @@ type Mutation {
   ): ID!
   reloadPlugins: Boolean!
 
+  """
+  Installs the given packages.
+  If a package is already installed, it will be updated if needed..
+  If an error occurs when installing a package, the job will continue to install the remaining packages.
+  Returns the job ID
+  """
+  installPackages(type: PackageType!, packages: [PackageSpecInput!]!): ID!
+  """
+  Updates the given packages.
+  If a package is not installed, it will not be installed.
+  If a package does not need to be updated, it will not be updated.
+  If no packages are provided, all packages of the given type will be updated.
+  If an error occurs when updating a package, the job will continue to update the remaining packages.
+  Returns the job ID.
+  """
+  updatePackages(type: PackageType!, packages: [PackageSpecInput!]): ID!
+  """
+  Uninstalls the given packages.
+  If an error occurs when uninstalling a package, the job will continue to uninstall the remaining packages.
+  Returns the job ID
+  """
+  uninstallPackages(type: PackageType!, packages: [PackageSpecInput!]!): ID!
+
   stopJob(job_id: ID!): Boolean!
   stopAllJobs: Boolean!
 

graphql/schema/types/config.graphql

@@ -167,6 +167,11 @@ input ConfigGeneralInput {
   stashBoxes: [StashBoxInput!]
   "Python path - resolved using path if unset"
   pythonPath: String
+
+  "Source of scraper packages"
+  scraperPackageSources: [PackageSourceInput!]
+  "Source of plugin packages"
+  pluginPackageSources: [PackageSourceInput!]
 }
 
 type ConfigGeneralResult {
@@ -280,6 +285,11 @@ type ConfigGeneralResult {
   stashBoxes: [StashBox!]!
   "Python path - resolved using path if unset"
   pythonPath: String!
+
+  "Source of scraper packages"
+  scraperPackageSources: [PackageSource!]!
+  "Source of plugin packages"
+  pluginPackageSources: [PackageSource!]!
 }
 
 input ConfigDisableDropdownCreateInput {

graphql/schema/types/package.graphql

@@ -0,0 +1,36 @@
+enum PackageType {
+  Scraper
+  Plugin
+}
+
+type Package {
+  package_id: String!
+  name: String!
+  version: String
+  date: Timestamp
+  requires: [Package!]!
+
+  sourceURL: String!
+
+  "The available upgraded version of this package"
+  upgrade: Package
+
+  metadata: Map!
+}
+
+input PackageSpecInput {
+  id: String!
+  sourceURL: String!
+}
+
+type PackageSource {
+  name: String
+  url: String!
+  local_path: String
+}
+
+input PackageSourceInput {
+  name: String
+  url: String!
+  local_path: String
+}

internal/api/resolver_mutation_configure.go

@@ -347,6 +347,18 @@ func (r *mutationResolver) ConfigureGeneral(ctx context.Context, input ConfigGen
 		c.Set(config.DrawFunscriptHeatmapRange, input.DrawFunscriptHeatmapRange)
 	}
 
+	refreshScraperSource := false
+	if input.ScraperPackageSources != nil {
+		c.Set(config.ScraperPackageSources, input.ScraperPackageSources)
+		refreshScraperSource = true
+	}
+
+	refreshPluginSource := false
+	if input.PluginPackageSources != nil {
+		c.Set(config.PluginPackageSources, input.PluginPackageSources)
+		refreshPluginSource = true
+	}
+
 	if err := c.Write(); err != nil {
 		return makeConfigGeneralResult(), err
 	}
@@ -361,6 +373,12 @@ func (r *mutationResolver) ConfigureGeneral(ctx context.Context, input ConfigGen
 	if refreshBlobStorage {
 		manager.GetInstance().SetBlobStoreOptions()
 	}
+	if refreshScraperSource {
+		manager.GetInstance().RefreshScraperSourceManager()
+	}
+	if refreshPluginSource {
+		manager.GetInstance().RefreshPluginSourceManager()
+	}
 
 	return makeConfigGeneralResult(), nil
 }

internal/api/resolver_mutation_package.go

@@ -0,0 +1,82 @@
+package api
+
+import (
+	"context"
+	"strconv"
+
+	"github.com/stashapp/stash/internal/manager"
+	"github.com/stashapp/stash/internal/manager/task"
+	"github.com/stashapp/stash/pkg/logger"
+	"github.com/stashapp/stash/pkg/models"
+)
+
+func refreshPackageType(typeArg PackageType) {
+	mgr := manager.GetInstance()
+
+	if typeArg == PackageTypePlugin {
+		if err := mgr.PluginCache.LoadPlugins(); err != nil {
+			logger.Errorf("Error reading plugin configs: %v", err)
+		}
+	} else if typeArg == PackageTypeScraper {
+		if err := mgr.ScraperCache.ReloadScrapers(); err != nil {
+			logger.Errorf("Error reading scraper configs: %v", err)
+		}
+	}
+}
+
+func (r *mutationResolver) InstallPackages(ctx context.Context, typeArg PackageType, packages []*models.PackageSpecInput) (string, error) {
+	pm, err := getPackageManager(typeArg)
+	if err != nil {
+		return "", err
+	}
+
+	mgr := manager.GetInstance()
+	t := &task.InstallPackagesJob{
+		PackagesJob: task.PackagesJob{
+			PackageManager: pm,
+			OnComplete:     func() { refreshPackageType(typeArg) },
+		},
+		Packages: packages,
+	}
+	jobID := mgr.JobManager.Add(ctx, "Installing packages...", t)
+
+	return strconv.Itoa(jobID), nil
+}
+
+func (r *mutationResolver) UpdatePackages(ctx context.Context, typeArg PackageType, packages []*models.PackageSpecInput) (string, error) {
+	pm, err := getPackageManager(typeArg)
+	if err != nil {
+		return "", err
+	}
+
+	mgr := manager.GetInstance()
+	t := &task.UpdatePackagesJob{
+		PackagesJob: task.PackagesJob{
+			PackageManager: pm,
+			OnComplete:     func() { refreshPackageType(typeArg) },
+		},
+		Packages: packages,
+	}
+	jobID := mgr.JobManager.Add(ctx, "Updating packages...", t)
+
+	return strconv.Itoa(jobID), nil
+}
+
+func (r *mutationResolver) UninstallPackages(ctx context.Context, typeArg PackageType, packages []*models.PackageSpecInput) (string, error) {
+	pm, err := getPackageManager(typeArg)
+	if err != nil {
+		return "", err
+	}
+
+	mgr := manager.GetInstance()
+	t := &task.UninstallPackagesJob{
+		PackagesJob: task.PackagesJob{
+			PackageManager: pm,
+			OnComplete:     func() { refreshPackageType(typeArg) },
+		},
+		Packages: packages,
+	}
+	jobID := mgr.JobManager.Add(ctx, "Updating packages...", t)
+
+	return strconv.Itoa(jobID), nil
+}

internal/api/resolver_query_configuration.go

@@ -127,6 +127,8 @@ func makeConfigGeneralResult() *ConfigGeneralResult {
 		LiveTranscodeInputArgs:        config.GetLiveTranscodeInputArgs(),
 		LiveTranscodeOutputArgs:       config.GetLiveTranscodeOutputArgs(),
 		DrawFunscriptHeatmapRange:     config.GetDrawFunscriptHeatmapRange(),
+		ScraperPackageSources:         config.GetScraperPackageSources(),
+		PluginPackageSources:          config.GetPluginPackageSources(),
 	}
 }