Skip to content

Commit

Permalink
Fix stash scraper errors and add apikey field (#5474)
Browse files Browse the repository at this point in the history
* Use hasura/go-graphql-client instead of shurcooL version
* Fix graphql query errors
* Support setting api key for stash server
  • Loading branch information
WithoutPants authored Nov 12, 2024
1 parent 64fed35 commit 5f690d9
Show file tree
Hide file tree
Showing 9 changed files with 195 additions and 34 deletions.
3 changes: 2 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ require (
github.com/gorilla/sessions v1.2.1
github.com/gorilla/websocket v1.5.0
github.com/hashicorp/golang-lru/v2 v2.0.7
github.com/hasura/go-graphql-client v0.13.1
github.com/jinzhu/copier v0.4.0
github.com/jmoiron/sqlx v1.4.0
github.com/json-iterator/go v1.1.12
Expand All @@ -39,7 +40,6 @@ require (
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8
github.com/remeh/sizedwaitgroup v1.0.0
github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd
github.com/shurcooL/graphql v0.0.0-20181231061246-d48a9a75455f
github.com/sirupsen/logrus v1.9.3
github.com/spf13/cast v1.6.0
github.com/spf13/pflag v1.0.5
Expand Down Expand Up @@ -67,6 +67,7 @@ require (
github.com/asticode/go-astikit v0.20.0 // indirect
github.com/asticode/go-astits v1.8.0 // indirect
github.com/chromedp/sysutil v1.0.0 // indirect
github.com/coder/websocket v1.8.12 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.5 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dlclark/regexp2 v1.7.0 // indirect
Expand Down
6 changes: 4 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -153,6 +153,8 @@ github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWH
github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/cncf/xds/go v0.0.0-20211130200136-a8f946100490/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/coder/websocket v1.8.12 h1:5bUXkEPPIbewrnkU8LTCLVaxi4N4J8ahufH2vlo4NAo=
github.com/coder/websocket v1.8.12/go.mod h1:LNVeNrXQZfe5qhS9ALED3uA+l5pPqvwXg3CKoDBB2gs=
github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
Expand Down Expand Up @@ -394,6 +396,8 @@ github.com/hashicorp/vault/api v1.0.4/go.mod h1:gDcqh3WGcR1cpF5AJz/B1UFheUEneMoI
github.com/hashicorp/vault/sdk v0.1.13/go.mod h1:B+hVj7TpuQY1Y/GPbCpffmgd+tSEwvhkWnjtSYCaS2M=
github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM=
github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM=
github.com/hasura/go-graphql-client v0.13.1 h1:kKbjhxhpwz58usVl+Xvgah/TDha5K2akNTRQdsEHN6U=
github.com/hasura/go-graphql-client v0.13.1/go.mod h1:k7FF7h53C+hSNFRG3++DdVZWIuHdCaTbI7siTJ//zGQ=
github.com/hjson/hjson-go/v4 v4.0.0 h1:wlm6IYYqHjOdXH1gHev4VoXCaW20HdQAGCxdOEEg2cs=
github.com/hjson/hjson-go/v4 v4.0.0/go.mod h1:KaYt3bTw3zhBjYqnXkYywcYctk0A2nxeEFTse3rH13E=
github.com/huandu/xstrings v1.0.0/go.mod h1:4qWG/gcEcfX4z/mBDHJ++3ReCw9ibxbsNJbcucJdbSo=
Expand Down Expand Up @@ -591,8 +595,6 @@ github.com/sagikazarmark/crypt v0.4.0/go.mod h1:ALv2SRj7GxYV4HO9elxH9nS6M9gW+xDN
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8=
github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I=
github.com/shurcooL/graphql v0.0.0-20181231061246-d48a9a75455f h1:tygelZueB1EtXkPI6mQ4o9DQ0+FKW41hTbunoXZCTqk=
github.com/shurcooL/graphql v0.0.0-20181231061246-d48a9a75455f/go.mod h1:AuYgA5Kyo4c7HfUmvRGs/6rGlMMV/6B1bVnB9JxJEEg=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
Expand Down
2 changes: 1 addition & 1 deletion pkg/plugin/examples/common/graphql.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import (
"errors"
"fmt"

"github.com/shurcooL/graphql"
graphql "github.com/hasura/go-graphql-client"
"github.com/stashapp/stash/pkg/plugin/common/log"
)

Expand Down
2 changes: 1 addition & 1 deletion pkg/plugin/util/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import (
"net/url"
"strconv"

"github.com/shurcooL/graphql"
graphql "github.com/hasura/go-graphql-client"

"github.com/stashapp/stash/pkg/plugin/common"
)
Expand Down
3 changes: 2 additions & 1 deletion pkg/scraper/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,8 @@ func (c config) validate() error {
}

type stashServer struct {
URL string `yaml:"url"`
URL string `yaml:"url"`
ApiKey string `yaml:"apiKey"`
}

type scraperTypeConfig struct {
Expand Down
55 changes: 55 additions & 0 deletions pkg/scraper/graphql.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
package scraper

import (
"errors"
"strings"

"github.com/hasura/go-graphql-client"
)

type graphqlErrors []error

func (e graphqlErrors) Error() string {
b := strings.Builder{}
for _, err := range e {
_, _ = b.WriteString(err.Error())
}
return b.String()
}

type graphqlError struct {
err graphql.Error
}

func (e graphqlError) Error() string {
unwrapped := e.err.Unwrap()
if unwrapped != nil {
var networkErr graphql.NetworkError
if errors.As(unwrapped, &networkErr) {
if networkErr.StatusCode() == 422 {
return networkErr.Body()
}
}
}
return e.err.Error()
}

// convertGraphqlError converts a graphql.Error or graphql.Errors into an error with a useful message.
// graphql.Error swallows important information, so we need to convert it to a more useful error type.
func convertGraphqlError(err error) error {
var gqlErrs graphql.Errors
if errors.As(err, &gqlErrs) {
ret := make(graphqlErrors, len(gqlErrs))
for i, e := range gqlErrs {
ret[i] = convertGraphqlError(e)
}
return ret
}

var gqlErr graphql.Error
if errors.As(err, &gqlErr) {
return graphqlError{gqlErr}
}

return err
}
33 changes: 26 additions & 7 deletions pkg/scraper/image.go
Original file line number Diff line number Diff line change
Expand Up @@ -122,13 +122,19 @@ func setGroupBackImage(ctx context.Context, client *http.Client, m *models.Scrap
return nil
}

func getImage(ctx context.Context, url string, client *http.Client, globalConfig GlobalConfig) (*string, error) {
type imageGetter struct {
client *http.Client
globalConfig GlobalConfig
requestModifier func(req *http.Request)
}

func (i *imageGetter) getImage(ctx context.Context, url string) (*string, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return nil, err
}

userAgent := globalConfig.GetScraperUserAgent()
userAgent := i.globalConfig.GetScraperUserAgent()
if userAgent != "" {
req.Header.Set("User-Agent", userAgent)
}
Expand All @@ -140,7 +146,11 @@ func getImage(ctx context.Context, url string, client *http.Client, globalConfig
req.Header.Set("Referer", req.URL.Scheme+"://"+req.Host+"/")
}

resp, err := client.Do(req)
if i.requestModifier != nil {
i.requestModifier(req)
}

resp, err := i.client.Do(req)

if err != nil {
return nil, err
Expand All @@ -167,10 +177,19 @@ func getImage(ctx context.Context, url string, client *http.Client, globalConfig
return &img, nil
}

func getStashPerformerImage(ctx context.Context, stashURL string, performerID string, client *http.Client, globalConfig GlobalConfig) (*string, error) {
return getImage(ctx, stashURL+"/performer/"+performerID+"/image", client, globalConfig)
func getImage(ctx context.Context, url string, client *http.Client, globalConfig GlobalConfig) (*string, error) {
g := imageGetter{
client: client,
globalConfig: globalConfig,
}

return g.getImage(ctx, url)
}

func getStashPerformerImage(ctx context.Context, stashURL string, performerID string, imageGetter imageGetter) (*string, error) {
return imageGetter.getImage(ctx, stashURL+"/performer/"+performerID+"/image")
}

func getStashSceneImage(ctx context.Context, stashURL string, sceneID string, client *http.Client, globalConfig GlobalConfig) (*string, error) {
return getImage(ctx, stashURL+"/scene/"+sceneID+"/screenshot", client, globalConfig)
func getStashSceneImage(ctx context.Context, stashURL string, sceneID string, imageGetter imageGetter) (*string, error) {
return imageGetter.getImage(ctx, stashURL+"/scene/"+sceneID+"/screenshot")
}
Loading

0 comments on commit 5f690d9

Please sign in to comment.