From 2fd7141f0f7a07710fbe7d4357e8ee3c3fdddeca Mon Sep 17 00:00:00 2001 From: WithoutPants <53250216+WithoutPants@users.noreply.github.com> Date: Mon, 16 Oct 2023 17:17:36 +1100 Subject: [PATCH] Javascript scraper postprocess (#4200) * Add javascript post-process action * Add documentation --- pkg/scraper/mapped.go | 72 ++++++++++++++----- .../src/docs/en/Manual/ScraperDevelopment.md | 14 ++++ 2 files changed, 68 insertions(+), 18 deletions(-) diff --git a/pkg/scraper/mapped.go b/pkg/scraper/mapped.go index ffdae91c52b..b812f05b087 100644 --- a/pkg/scraper/mapped.go +++ b/pkg/scraper/mapped.go @@ -11,6 +11,7 @@ import ( "strings" "time" + "github.com/robertkrimen/otto" "github.com/stashapp/stash/pkg/logger" "github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/sliceutil/stringslice" @@ -523,6 +524,31 @@ func (p *postProcessLbToKg) Apply(ctx context.Context, value string, q mappedQue return value } +type postProcessJavascript string + +func (p *postProcessJavascript) Apply(ctx context.Context, value string, q mappedQuery) string { + vm := otto.New() + if err := vm.Set("value", value); err != nil { + logger.Warnf("javascript failed to set value: %v", err) + return value + } + + script, err := vm.Compile("", "(function() { "+string(*p)+"})()") + if err != nil { + logger.Warnf("javascript failed to compile: %v", err) + return value + } + + output, err := vm.Run(script) + if err != nil { + logger.Warnf("javascript failed to run: %v", err) + return value + } + + // assume output is string + return output.String() +} + type mappedPostProcessAction struct { ParseDate string `yaml:"parseDate"` SubtractDays bool `yaml:"subtractDays"` @@ -531,65 +557,75 @@ type mappedPostProcessAction struct { Map map[string]string `yaml:"map"` FeetToCm bool `yaml:"feetToCm"` LbToKg bool `yaml:"lbToKg"` + Javascript string `yaml:"javascript"` } func (a mappedPostProcessAction) ToPostProcessAction() (postProcessAction, error) { var found string var ret postProcessAction + ensureOnly := func(field string) error { + if found != "" { + return fmt.Errorf("post-process actions must have a single field, found %s and %s", found, field) + } + found = field + return nil + } + if a.ParseDate != "" { found = "parseDate" action := postProcessParseDate(a.ParseDate) ret = &action } if len(a.Replace) > 0 { - if found != "" { - return nil, fmt.Errorf("post-process actions must have a single field, found %s and %s", found, "replace") + if err := ensureOnly("replace"); err != nil { + return nil, err } - found = "replace" action := postProcessReplace(a.Replace) ret = &action } if a.SubScraper != nil { - if found != "" { - return nil, fmt.Errorf("post-process actions must have a single field, found %s and %s", found, "subScraper") + if err := ensureOnly("subScraper"); err != nil { + return nil, err } - found = "subScraper" action := postProcessSubScraper(*a.SubScraper) ret = &action } if a.Map != nil { - if found != "" { - return nil, fmt.Errorf("post-process actions must have a single field, found %s and %s", found, "map") + if err := ensureOnly("map"); err != nil { + return nil, err } - found = "map" action := postProcessMap(a.Map) ret = &action } if a.FeetToCm { - if found != "" { - return nil, fmt.Errorf("post-process actions must have a single field, found %s and %s", found, "feetToCm") + if err := ensureOnly("feetToCm"); err != nil { + return nil, err } - found = "feetToCm" action := postProcessFeetToCm(a.FeetToCm) ret = &action } if a.LbToKg { - if found != "" { - return nil, fmt.Errorf("post-process actions must have a single field, found %s and %s", found, "lbToKg") + if err := ensureOnly("lbToKg"); err != nil { + return nil, err } - found = "lbToKg" action := postProcessLbToKg(a.LbToKg) ret = &action } if a.SubtractDays { - if found != "" { - return nil, fmt.Errorf("post-process actions must have a single field, found %s and %s", found, "subtractDays") + if err := ensureOnly("subtractDays"); err != nil { + return nil, err } - // found = "subtractDays" action := postProcessSubtractDays(a.SubtractDays) ret = &action } + if a.Javascript != "" { + if err := ensureOnly("javascript"); err != nil { + return nil, err + } + action := postProcessJavascript(a.Javascript) + ret = &action + } if ret == nil { return nil, errors.New("invalid post-process action") diff --git a/ui/v2.5/src/docs/en/Manual/ScraperDevelopment.md b/ui/v2.5/src/docs/en/Manual/ScraperDevelopment.md index b0ff5cdfeaf..d40d12ac27e 100644 --- a/ui/v2.5/src/docs/en/Manual/ScraperDevelopment.md +++ b/ui/v2.5/src/docs/en/Manual/ScraperDevelopment.md @@ -341,6 +341,20 @@ scene: ### Post-processing options Post-processing operations are contained in the `postProcess` key. Post-processing operations are performed in the order they are specified. The following post-processing operations are available: +* `javascript`: accepts a javascript code block, that must return a string value. The input string is declared in the `value` variable. If an error occurs while compiling or running the script, then the original value is returned. +Example: +```yaml +performer: + Name: + selector: //div[@class="example element"] + postProcess: + - javascript: | + // capitalise the first letter + if (value && value.length) { + return value[0].toUpperCase() + value.substring(1) + } +``` +Note that the `otto` javascript engine is missing a few built-in methods and may not be consistent with other modern javascript implementations. * `feetToCm`: converts a string containing feet and inches numbers into centimeters. Looks for up to two separate integers and interprets the first as the number of feet, and the second as the number of inches. The numbers can be separated by any non-numeric character including the `.` character. It does not handle decimal numbers. For example `6.3` and `6ft3.3` would both be interpreted as 6 feet, 3 inches before converting into centimeters. * `lbToKg`: converts a string containing lbs to kg. * `map`: contains a map of input values to output values. Where a value matches one of the input values, it is replaced with the matching output value. If no value is matched, then value is unmodified.