diff --git a/.gitpod.dockerfile b/.gitpod.dockerfile
index 1c3f74c3e..292dddf90 100644
--- a/.gitpod.dockerfile
+++ b/.gitpod.dockerfile
@@ -11,7 +11,7 @@ RUN export PATH=$(echo "$PATH" | sed -e 's|:/workspace/go/bin||' -e 's|:/home/gi
ENV PATH=$GOROOT/bin:$GOPATH/bin:$PATH
RUN go install -v \
- github.com/cosmtrek/air@latest && \
+ github.com/air-verse/air@latest && \
sudo rm -rf $GOPATH/src && \
sudo rm -rf $GOPATH/pkg
# user Go packages
diff --git a/.gitpod.yml b/.gitpod.yml
index 9a99bf403..762737219 100644
--- a/.gitpod.yml
+++ b/.gitpod.yml
@@ -2,7 +2,7 @@ image:
file: .gitpod.dockerfile
tasks:
- name: Continuous Build
- command: yarn config set ignore-engines true && yarn global add concurrently && go install github.com/cosmtrek/air@latest && cd /workspace/xbvr && go generate && go get && yarn && yarn dev
+ command: yarn config set ignore-engines true && yarn global add concurrently && go install github.com/air-verse/air@latest && cd /workspace/xbvr && go generate && go get && yarn && yarn dev
ports:
- port: 9999
onOpen: open-preview
diff --git a/README.md b/README.md
index bd4c2f7c7..2e7be8c5d 100644
--- a/README.md
+++ b/README.md
@@ -151,4 +151,4 @@ esc - closes details pane
| `--web_port` | XBVR_WEB_PORT | Int| override default Web Page port 9999|
| `--ws_addr` | XBVR_WS_ADDR | String| override default Websocket address from the default 0.0.0.0:9998|
| `--db_connection_pool_size` | DB_CONNECTION_POOL_SIZE | Int| sets the connection pool size for mariadb databases|
-| `--concurrent_scrapers` | CONCURRENT_SCRAPERS | String| set the number of scrapers that run concurrently default 9999|
+| `--concurrent_scrapers` | CONCURRENT_SCRAPERS | Int| set the number of scrapers that run concurrently default 9999|
diff --git a/go.mod b/go.mod
index 3bc515ef5..1987dc434 100644
--- a/go.mod
+++ b/go.mod
@@ -9,7 +9,7 @@ require (
github.com/anacrolix/ffprobe v1.1.0
github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de
github.com/avast/retry-go/v4 v4.6.0
- github.com/blevesearch/bleve/v2 v2.4.0
+ github.com/blevesearch/bleve/v2 v2.4.1
github.com/bregydoc/gtranslate v0.0.0-20200913051839-1bd07f6c1fc5
github.com/creasty/defaults v1.7.0
github.com/darwayne/go-timecode v1.1.0
@@ -17,12 +17,12 @@ require (
github.com/djherbis/times v1.6.0
github.com/dustin/go-humanize v1.0.1
github.com/emicklei/go-restful-openapi/v2 v2.10.2
- github.com/emicklei/go-restful/v3 v3.12.0
+ github.com/emicklei/go-restful/v3 v3.12.1
github.com/gammazero/nexus/v3 v3.2.2
github.com/getlantern/systray v1.2.2
github.com/go-openapi/spec v0.21.0
github.com/go-resty/resty/v2 v2.13.1
- github.com/go-test/deep v1.1.0
+ github.com/go-test/deep v1.1.1
github.com/gocolly/colly/v2 v2.1.0
github.com/gorilla/mux v1.8.1
github.com/gosimple/slug v1.14.0
@@ -46,44 +46,44 @@ require (
github.com/peterbourgon/diskv v2.0.1+incompatible
github.com/pkg/errors v0.9.1
github.com/putdotio/go-putio v1.7.1
- github.com/robertkrimen/otto v0.3.0
+ github.com/robertkrimen/otto v0.4.0
github.com/robfig/cron/v3 v3.0.1
- github.com/rs/cors v1.10.1
+ github.com/rs/cors v1.11.0
github.com/sirupsen/logrus v1.9.3
github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966
github.com/thoas/go-funk v0.9.3
github.com/tidwall/gjson v1.17.1
github.com/x-cray/logrus-prefixed-formatter v0.5.2
github.com/xo/dburl v0.21.1
- golang.org/x/crypto v0.23.0
- golang.org/x/net v0.25.0
- golang.org/x/oauth2 v0.20.0
- golang.org/x/sys v0.20.0
+ golang.org/x/crypto v0.25.0
+ golang.org/x/net v0.27.0
+ golang.org/x/oauth2 v0.21.0
+ golang.org/x/sys v0.22.0
golang.org/x/text v0.16.0
gopkg.in/gormigrate.v1 v1.6.0
willnorris.com/go/imageproxy v0.11.3-0.20240601234520-572ad2db78ed
)
require (
- github.com/blevesearch/go-faiss v1.0.13 // indirect
- github.com/blevesearch/zapx/v16 v16.0.12 // indirect
+ github.com/blevesearch/go-faiss v1.0.19 // indirect
+ github.com/blevesearch/zapx/v16 v16.1.4 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
require (
- github.com/RoaringBitmap/roaring v1.2.3 // indirect
+ github.com/RoaringBitmap/roaring v1.9.3 // indirect
github.com/andybalholm/cascadia v1.3.2 // indirect
github.com/antchfx/htmlquery v1.2.3 // indirect
github.com/antchfx/xmlquery v1.3.1 // indirect
github.com/antchfx/xpath v1.1.10 // indirect
github.com/beorn7/perks v1.0.1 // indirect
- github.com/bits-and-blooms/bitset v1.2.0 // indirect
- github.com/blevesearch/bleve_index_api v1.1.6
+ github.com/bits-and-blooms/bitset v1.12.0 // indirect
+ github.com/blevesearch/bleve_index_api v1.1.9
github.com/blevesearch/geo v0.1.20 // indirect
github.com/blevesearch/go-porterstemmer v1.0.3 // indirect
github.com/blevesearch/gtreap v0.1.1 // indirect
github.com/blevesearch/mmap-go v1.0.4 // indirect
- github.com/blevesearch/scorch_segment_api/v2 v2.2.9 // indirect
+ github.com/blevesearch/scorch_segment_api/v2 v2.2.14 // indirect
github.com/blevesearch/segment v0.9.1 // indirect
github.com/blevesearch/snowballstem v0.9.0 // indirect
github.com/blevesearch/upsidedown_store_api v1.0.2 // indirect
@@ -148,7 +148,7 @@ require (
github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect
go.etcd.io/bbolt v1.3.7 // indirect
golang.org/x/image v0.18.0 // indirect
- golang.org/x/term v0.20.0 // indirect
+ golang.org/x/term v0.22.0 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/protobuf v1.34.1 // indirect
gopkg.in/sourcemap.v1 v1.0.5 // indirect
diff --git a/go.sum b/go.sum
index 2eb937d26..bec0f80f0 100644
--- a/go.sum
+++ b/go.sum
@@ -7,8 +7,8 @@ github.com/ProtonMail/go-appdir v1.1.0/go.mod h1:3d8Y9F5mbEUjrYbcJ3rcDxcWbqbttF+
github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc=
github.com/PuerkitoBio/goquery v1.9.2 h1:4/wZksC3KgkQw7SQgkKotmKljk0M6V8TUvA8Wb4yPeE=
github.com/PuerkitoBio/goquery v1.9.2/go.mod h1:GHPCaP0ODyyxqcNoFGYlAprUFH81NuRPd0GX3Zu2Mvk=
-github.com/RoaringBitmap/roaring v1.2.3 h1:yqreLINqIrX22ErkKI0vY47/ivtJr6n+kMhVOVmhWBY=
-github.com/RoaringBitmap/roaring v1.2.3/go.mod h1:plvDsJQpxOC5bw8LRteu/MLWHsHez/3y6cubLI4/1yE=
+github.com/RoaringBitmap/roaring v1.9.3 h1:t4EbC5qQwnisr5PrP9nt0IRhRTb9gMUgQF4t4S2OByM=
+github.com/RoaringBitmap/roaring v1.9.3/go.mod h1:6AXUsoIEzDTFFQCe1RbGA6uFONMhvejWj5rqITANK90=
github.com/abbot/go-http-auth v0.4.0 h1:QjmvZ5gSC7jm3Zg54DqWE/T5m1t2AfDu6QlXJT0EVT0=
github.com/abbot/go-http-auth v0.4.0/go.mod h1:Cz6ARTIzApMJDzh5bRMSUou6UMSp0IEXg9km/ci7TJM=
github.com/anacrolix/envpprof v1.0.0 h1:AwZ+mBP4rQ5f7JSsrsN3h7M2xDW/xSE66IPVOqlnuUc=
@@ -36,24 +36,24 @@ github.com/avast/retry-go/v4 v4.6.0 h1:K9xNA+KeB8HHc2aWFuLb25Offp+0iVRXEvFx8IinR
github.com/avast/retry-go/v4 v4.6.0/go.mod h1:gvWlPhBVsvBbLkVGDg/KwvBv0bEkCOLRRSHKIr2PyOE=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
-github.com/bits-and-blooms/bitset v1.2.0 h1:Kn4yilvwNtMACtf1eYDlG8H77R07mZSPbMjLyS07ChA=
-github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA=
-github.com/blevesearch/bleve/v2 v2.4.0 h1:2xyg+Wv60CFHYccXc+moGxbL+8QKT/dZK09AewHgKsg=
-github.com/blevesearch/bleve/v2 v2.4.0/go.mod h1:IhQHoFAbHgWKYavb9rQgQEJJVMuY99cKdQ0wPpst2aY=
-github.com/blevesearch/bleve_index_api v1.1.6 h1:orkqDFCBuNU2oHW9hN2YEJmet+TE9orml3FCGbl1cKk=
-github.com/blevesearch/bleve_index_api v1.1.6/go.mod h1:PbcwjIcRmjhGbkS/lJCpfgVSMROV6TRubGGAODaK1W8=
+github.com/bits-and-blooms/bitset v1.12.0 h1:U/q1fAF7xXRhFCrhROzIfffYnu+dlS38vCZtmFVPHmA=
+github.com/bits-and-blooms/bitset v1.12.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
+github.com/blevesearch/bleve/v2 v2.4.1 h1:8QWqsifq693mN3h6cSigKqkKUsUfv5hu0FDgz/4bFuA=
+github.com/blevesearch/bleve/v2 v2.4.1/go.mod h1:Ezmvsouspi+uVwnDzjIsCeUIT0WuBKlicP5JZnExWzo=
+github.com/blevesearch/bleve_index_api v1.1.9 h1:Cpq0Lp3As0Gfk3+PmcoNDRKeI50C5yuFNpj0YlN/bOE=
+github.com/blevesearch/bleve_index_api v1.1.9/go.mod h1:PbcwjIcRmjhGbkS/lJCpfgVSMROV6TRubGGAODaK1W8=
github.com/blevesearch/geo v0.1.20 h1:paaSpu2Ewh/tn5DKn/FB5SzvH0EWupxHEIwbCk/QPqM=
github.com/blevesearch/geo v0.1.20/go.mod h1:DVG2QjwHNMFmjo+ZgzrIq2sfCh6rIHzy9d9d0B59I6w=
-github.com/blevesearch/go-faiss v1.0.13 h1:zfFs7ZYD0NqXVSY37j0JZjZT1BhE9AE4peJfcx/NB4A=
-github.com/blevesearch/go-faiss v1.0.13/go.mod h1:jrxHrbl42X/RnDPI+wBoZU8joxxuRwedrxqswQ3xfU8=
+github.com/blevesearch/go-faiss v1.0.19 h1:UKoP8hS7DVsVSRRloNJb4qPfe2UQ99pP4D3oXd23g2A=
+github.com/blevesearch/go-faiss v1.0.19/go.mod h1:jrxHrbl42X/RnDPI+wBoZU8joxxuRwedrxqswQ3xfU8=
github.com/blevesearch/go-porterstemmer v1.0.3 h1:GtmsqID0aZdCSNiY8SkuPJ12pD4jI+DdXTAn4YRcHCo=
github.com/blevesearch/go-porterstemmer v1.0.3/go.mod h1:angGc5Ht+k2xhJdZi511LtmxuEf0OVpvUUNrwmM1P7M=
github.com/blevesearch/gtreap v0.1.1 h1:2JWigFrzDMR+42WGIN/V2p0cUvn4UP3C4Q5nmaZGW8Y=
github.com/blevesearch/gtreap v0.1.1/go.mod h1:QaQyDRAT51sotthUWAH4Sj08awFSSWzgYICSZ3w0tYk=
github.com/blevesearch/mmap-go v1.0.4 h1:OVhDhT5B/M1HNPpYPBKIEJaD0F3Si+CrEKULGCDPWmc=
github.com/blevesearch/mmap-go v1.0.4/go.mod h1:EWmEAOmdAS9z/pi/+Toxu99DnsbhG1TIxUoRmJw/pSs=
-github.com/blevesearch/scorch_segment_api/v2 v2.2.9 h1:3nBaSBRFokjE4FtPW3eUDgcAu3KphBg1GP07zy/6Uyk=
-github.com/blevesearch/scorch_segment_api/v2 v2.2.9/go.mod h1:ckbeb7knyOOvAdZinn/ASbB7EA3HoagnJkmEV3J7+sg=
+github.com/blevesearch/scorch_segment_api/v2 v2.2.14 h1:fgMLMpGWR7u2TdRm7XSZVWhPvMAcdYHh25Lq1fQ6Fjo=
+github.com/blevesearch/scorch_segment_api/v2 v2.2.14/go.mod h1:B7+a7vfpY4NsjuTkpv/eY7RZ91Xr90VaJzT2t7upZN8=
github.com/blevesearch/segment v0.9.1 h1:+dThDy+Lvgj5JMxhmOVlgFfkUtZV2kw49xax4+jTfSU=
github.com/blevesearch/segment v0.9.1/go.mod h1:zN21iLm7+GnBHWTao9I+Au/7MBiL8pPFtJBJTsk6kQw=
github.com/blevesearch/snowballstem v0.9.0 h1:lMQ189YspGP6sXvZQ4WZ+MLawfV8wOmPoD/iWeNXm8s=
@@ -72,8 +72,8 @@ github.com/blevesearch/zapx/v14 v14.3.10 h1:SG6xlsL+W6YjhX5N3aEiL/2tcWh3DO75Bnz7
github.com/blevesearch/zapx/v14 v14.3.10/go.mod h1:qqyuR0u230jN1yMmE4FIAuCxmahRQEOehF78m6oTgns=
github.com/blevesearch/zapx/v15 v15.3.13 h1:6EkfaZiPlAxqXz0neniq35my6S48QI94W/wyhnpDHHQ=
github.com/blevesearch/zapx/v15 v15.3.13/go.mod h1:Turk/TNRKj9es7ZpKK95PS7f6D44Y7fAFy8F4LXQtGg=
-github.com/blevesearch/zapx/v16 v16.0.12 h1:Uccxvjmn+hQ6ywQP+wIiTpdq9LnAviGoryJOmGwAo/I=
-github.com/blevesearch/zapx/v16 v16.0.12/go.mod h1:MYnOshRfSm4C4drxx1LGRI+MVFByykJ2anDY1fxdk9Q=
+github.com/blevesearch/zapx/v16 v16.1.4 h1:TBQfG77g2UUXwfjOVcEtB9pXkg6JBmGXkeZKI67+TiA=
+github.com/blevesearch/zapx/v16 v16.1.4/go.mod h1:+Q+Z89Iv7ewhdX2jyE6Qs/RUnN4tZuokaQ0xvTaFmx8=
github.com/bradfitz/iter v0.0.0-20140124041915-454541ec3da2 h1:1B/+1BcRhOMG1KH/YhNIU8OppSWk5d/NGyfRla88CuY=
github.com/bradfitz/iter v0.0.0-20140124041915-454541ec3da2/go.mod h1:PyRFw1Lt2wKX4ZVSQ2mk+PeDa1rxyObEDlApuIsUKuo=
github.com/bregydoc/gtranslate v0.0.0-20200913051839-1bd07f6c1fc5 h1:fpVDaadW68V+6vqxJHU9jrW0/z1i2MQYJZyk9w2cjpw=
@@ -105,8 +105,8 @@ github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+m
github.com/emicklei/go-restful-openapi/v2 v2.10.2 h1:RfxWvGmASIwVoZIEncvXLi5HxYQ0S8rNBkPresDMt1c=
github.com/emicklei/go-restful-openapi/v2 v2.10.2/go.mod h1:4CTuOXHFg3jkvCpnXN+Wkw5prVUnP8hIACssJTYorWo=
github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
-github.com/emicklei/go-restful/v3 v3.12.0 h1:y2DdzBAURM29NFF94q6RaY4vjIH1rtwDapwQtU84iWk=
-github.com/emicklei/go-restful/v3 v3.12.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
+github.com/emicklei/go-restful/v3 v3.12.1 h1:PJMDIM/ak7btuL8Ex0iYET9hxM3CI2sjZtzpL63nKAU=
+github.com/emicklei/go-restful/v3 v3.12.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5 h1:Yzb9+7DPaBjB8zlTR87/ElzFsnQfuHnVUVqpZZIcV5Y=
@@ -151,8 +151,8 @@ github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gG
github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
-github.com/go-test/deep v1.1.0 h1:WOcxcdHcvdgThNXjw0t76K42FXTU7HpNQWHpA2HHNlg=
-github.com/go-test/deep v1.1.0/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE=
+github.com/go-test/deep v1.1.1 h1:0r/53hagsehfO4bzD2Pgr/+RgHqhmf+k1Bpse2cTu1U=
+github.com/go-test/deep v1.1.1/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE=
github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
github.com/gocolly/colly v1.2.0/go.mod h1:Hof5T3ZswNVsOHYmba1u03W65HDWgpV5HifSuueE0EA=
@@ -322,14 +322,14 @@ github.com/prometheus/procfs v0.14.0/go.mod h1:XL+Iwz8k8ZabyZfMFHPiilCniixqQarAy
github.com/putdotio/go-putio v1.7.1 h1:316PpOMO2a7H73foRxlpHmekeLso07et26Z00YlwQ2A=
github.com/putdotio/go-putio v1.7.1/go.mod h1:QhjpLhn3La/ea4FeJlp1qsiaFZDC0EIO8VUe8VEKMV0=
github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
-github.com/robertkrimen/otto v0.3.0 h1:5RI+8860NSxvXywDY9ddF5HcPw0puRsd8EgbXV0oqRE=
-github.com/robertkrimen/otto v0.3.0/go.mod h1:uW9yN1CYflmUQYvAMS0m+ZiNo3dMzRUDQJX0jWbzgxw=
+github.com/robertkrimen/otto v0.4.0 h1:/c0GRrK1XDPcgIasAsnlpBT5DelIeB9U/Z/JCQsgr7E=
+github.com/robertkrimen/otto v0.4.0/go.mod h1:uW9yN1CYflmUQYvAMS0m+ZiNo3dMzRUDQJX0jWbzgxw=
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M=
github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA=
-github.com/rs/cors v1.10.1 h1:L0uuZVXIKlI1SShY2nhFfo44TYvDPQ1w4oFkUJNfhyo=
-github.com/rs/cors v1.10.1/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU=
+github.com/rs/cors v1.11.0 h1:0B9GE/r9Bc2UxRMMtymBkHTenPkHDv0CW4Y98GBY+po=
+github.com/rs/cors v1.11.0/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU=
github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd h1:CmH9+J6ZSsIjUK3dcGsnCnO41eRBOnY12zwkn5qVwgc=
github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk=
github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca h1:NugYot0LIVPxTvN8n+Kvkn6TrbMyxQiuvKdEwFdR9vI=
@@ -385,8 +385,9 @@ golang.org/x/crypto v0.0.0-20191205180655-e7c4368fe9dd/go.mod h1:LzIPMQfyMNhhGPh
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
-golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI=
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
+golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30=
+golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/image v0.18.0 h1:jGzIakQa/ZXI1I0Fxvaa9W7yP25TqT6cHIHn+6CqvSQ=
@@ -417,12 +418,13 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
-golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac=
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
+golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys=
+golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.2.0/go.mod h1:Cwn6afJ8jrQwYMxQDTpISoXmXW9I6qF6vDeuuoX3Ibs=
-golang.org/x/oauth2 v0.20.0 h1:4mQdhULixXKP1rwYBW0vAijoXnkTG0BLCDRzfe1idMo=
-golang.org/x/oauth2 v0.20.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
+golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs=
+golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -447,8 +449,9 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
+golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
@@ -456,8 +459,9 @@ golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
-golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw=
golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
+golang.org/x/term v0.22.0 h1:BbsgPEJULsl2fV/AT3v15Mjva5yXKQDyKf+TbDz7QJk=
+golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
diff --git a/package.json b/package.json
index 5e7d39fd7..6b3cf781f 100644
--- a/package.json
+++ b/package.json
@@ -35,8 +35,8 @@
"wampy": "6.4.2"
},
"devDependencies": {
- "@babel/core": "7.24.7",
- "@babel/eslint-parser": "7.24.7",
+ "@babel/core": "7.24.9",
+ "@babel/eslint-parser": "7.24.8",
"@vue/cli-plugin-babel": "5.0.8",
"@vue/cli-plugin-eslint": "5.0.8",
"@vue/cli-service": "5.0.8",
@@ -46,18 +46,18 @@
"eslint": "8.57.0",
"eslint-plugin-import": "2.29.1",
"eslint-plugin-n": "14.0.0",
- "eslint-plugin-promise": "6.1.1",
+ "eslint-plugin-promise": "6.4.0",
"eslint-plugin-standard": "4.1.0",
- "eslint-plugin-vue": "9.25.0",
+ "eslint-plugin-vue": "9.27.0",
"less": "4.2.0",
"less-loader": "12.2.0",
- "sass": "1.76.0",
+ "sass": "1.77.8",
"sass-loader": "14.2.1",
"simple-progress-webpack-plugin": "2.0.0",
"vue-cli-plugin-i18n": "2.3.2",
"vue-i18n-extract": "2.0.7",
"vue-template-compiler": "2.7.16",
- "webpack": "5.91.0"
+ "webpack": "5.93.0"
},
"eslintConfig": {
"root": true,
diff --git a/pkg/api/actors.go b/pkg/api/actors.go
index 2a28b4d3c..39ae8b39c 100644
--- a/pkg/api/actors.go
+++ b/pkg/api/actors.go
@@ -11,6 +11,7 @@ import (
restfulspec "github.com/emicklei/go-restful-openapi/v2"
"github.com/emicklei/go-restful/v3"
"github.com/xbapps/xbvr/pkg/models"
+ "github.com/xbapps/xbvr/pkg/scrape"
)
type ResponseGetActors struct {
@@ -194,6 +195,15 @@ func (i ActorResource) getFilters(req *restful.Request, resp *restful.Response)
outAttributes = append(outAttributes, "Breast Type "+r.Result)
}
+ db.Table("actors").
+ Where("IFNULL(gender,'') <> ''").
+ Select("distinct gender as result").
+ Order("gender").
+ Find(&results)
+ for _, r := range results {
+ outAttributes = append(outAttributes, "Gender "+r.Result)
+ }
+
resp.WriteHeaderAndEntity(http.StatusOK, ResponseGetActorFilters{
Attributes: outAttributes,
Cast: outCast,
@@ -731,7 +741,7 @@ func (i ActorResource) editActorExtRefs(req *restful.Request, resp *restful.Resp
// add new links
for _, url := range urls {
var extref models.ExternalReference
- extref.FindExternalUrl(extref.DetermineActorScraperByUrl(url), url)
+ commonDb.Preload("XbvrLinks").Where(&models.ExternalReference{ExternalURL: url}).First(&extref)
if extref.ID == 0 {
// create new extref + link
extref.ExternalSource = extref.DetermineActorScraperByUrl(url)
@@ -746,6 +756,9 @@ func (i ActorResource) editActorExtRefs(req *restful.Request, resp *restful.Resp
ExternalReferenceID: extref.ID, ExternalSource: extref.ExternalSource, ExternalId: extref.ExternalId, MatchType: 0})
extref.Save()
models.AddActionActor(actor.ID, "edit_actor", "add", "external_reference_link", url)
+ if extref.ExternalSource == "stashdb performer" {
+ scrape.RefreshPerformer(extref.ExternalId)
+ }
} else {
// external reference exists, but check it is linked to this actor
found := false
@@ -761,6 +774,9 @@ func (i ActorResource) editActorExtRefs(req *restful.Request, resp *restful.Resp
ExternalReferenceID: extref.ID, ExternalSource: extref.ExternalSource, ExternalId: extref.ExternalId, MatchType: 0}
newLink.Save()
models.AddActionActor(actor.ID, "edit_actor", "add", "external_reference_link", url)
+ if extref.ExternalSource == "stashdb performer" {
+ scrape.RefreshPerformer(extref.ExternalId)
+ }
}
}
}
diff --git a/pkg/api/deovr.go b/pkg/api/deovr.go
index 1678f0cb4..4a1a2cb5f 100644
--- a/pkg/api/deovr.go
+++ b/pkg/api/deovr.go
@@ -419,17 +419,19 @@ func (i DeoVRResource) getDeoScene(req *restful.Request, resp *restful.Response)
var deoScriptFiles []DeoSceneScriptFile
var scriptFiles []models.File
- scriptFiles, err = scene.GetScriptFiles()
+ scriptFiles, err = scene.GetScriptFilesSorted(config.Config.Interfaces.Players.ScriptSortSeq)
if err != nil {
log.Error(err)
return
}
for _, file := range scriptFiles {
- deoScriptFiles = append(deoScriptFiles, DeoSceneScriptFile{
- Title: file.Filename,
- URL: fmt.Sprintf("%v/api/dms/file/%v", session.DeoRequestHost, file.ID),
- })
+ if strings.HasSuffix(file.Filename, ".funscript") {
+ deoScriptFiles = append(deoScriptFiles, DeoSceneScriptFile{
+ Title: file.Filename,
+ URL: fmt.Sprintf("%v/api/dms/file/%v", session.DeoRequestHost, file.ID),
+ })
+ }
}
var deoHSPFiles []DeoSceneHSPFile
diff --git a/pkg/api/heresphere.go b/pkg/api/heresphere.go
index d072255dc..4037393b5 100644
--- a/pkg/api/heresphere.go
+++ b/pkg/api/heresphere.go
@@ -567,7 +567,7 @@ func (i HeresphereResource) getHeresphereScene(req *restful.Request, resp *restf
var heresphereSubtitlesFiles []HeresphereSubtitles
var subtitlesFiles []models.File
- subtitlesFiles, err = scene.GetSubtitlesFiles()
+ subtitlesFiles, err = scene.GetSubtitlesFilesSorted(config.Config.Interfaces.Players.SubtitleSortSeq)
if err != nil {
log.Error(err)
return
diff --git a/pkg/api/options.go b/pkg/api/options.go
index 0e918ed3d..543c670a1 100644
--- a/pkg/api/options.go
+++ b/pkg/api/options.go
@@ -102,6 +102,7 @@ type RequestSaveOptionsDeoVR struct {
MultitrackCuepoints bool `json:"multitrack_cuepoints"`
VideoSortSeq string `json:"video_sort_seq"`
ScriptSortSeq string `json:"script_sort_seq"`
+ SubtitleSortSeq string `json:"subtitle_sort_seq"`
MultitrackCastCuepoints bool `json:"multitrack_cast_cuepoints"`
RetainNonHSPCuepoints bool `json:"retain_non_hsp_cuepoints"`
}
@@ -536,6 +537,7 @@ func (i ConfigResource) saveOptionsDeoVR(req *restful.Request, resp *restful.Res
config.Config.Interfaces.Heresphere.MultitrackCuepoints = r.MultitrackCuepoints
config.Config.Interfaces.Players.VideoSortSeq = r.VideoSortSeq
config.Config.Interfaces.Players.ScriptSortSeq = r.ScriptSortSeq
+ config.Config.Interfaces.Players.SubtitleSortSeq = r.SubtitleSortSeq
config.Config.Interfaces.Heresphere.MultitrackCastCuepoints = r.MultitrackCastCuepoints
config.Config.Interfaces.Heresphere.RetainNonHSPCuepoints = r.RetainNonHSPCuepoints
if r.Password != config.Config.Interfaces.DeoVR.Password && r.Password != "" {
diff --git a/pkg/config/config.go b/pkg/config/config.go
index f96f3b5f3..a7a77fe6b 100644
--- a/pkg/config/config.go
+++ b/pkg/config/config.go
@@ -97,8 +97,9 @@ type ObjectConfig struct {
RetainNonHSPCuepoints bool `default:"true" json:"retain_non_hsp_cuepoints"`
} `json:"heresphere"`
Players struct {
- VideoSortSeq string `default:"" json:"video_sort_seq"`
- ScriptSortSeq string `default:"" json:"script_sort_seq"`
+ VideoSortSeq string `default:"" json:"video_sort_seq"`
+ ScriptSortSeq string `default:"" json:"script_sort_seq"`
+ SubtitleSortSeq string `default:"" json:"subtitle_sort_seq"`
} `json:"players"`
} `json:"interfaces"`
Library struct {
diff --git a/pkg/externalreference/stashdb.go b/pkg/externalreference/stashdb.go
index 8918f3cdd..ac1e7bc23 100644
--- a/pkg/externalreference/stashdb.go
+++ b/pkg/externalreference/stashdb.go
@@ -249,7 +249,8 @@ func checkMatchedScenes() {
// if len(ref.XbvrLinks) == 0 {
for _, xbvrActor := range xbvrScene.Cast {
- if strings.EqualFold(strings.TrimSpace(simplifyName(xbvrActor.Name)), strings.TrimSpace(simplifyName(performer.Performer.Name))) {
+ if strings.EqualFold(strings.TrimSpace(simplifyName(xbvrActor.Name)), strings.TrimSpace(simplifyName(performer.Performer.Name))) ||
+ strings.EqualFold(strings.TrimSpace(simplifyName(xbvrActor.Name)), strings.TrimSpace(simplifyName(performer.As))) {
// check if actor already matched
exists := false
for _, link := range ref.XbvrLinks {
diff --git a/pkg/migrations/migrations.go b/pkg/migrations/migrations.go
index 66f48ba2a..8b8419b11 100644
--- a/pkg/migrations/migrations.go
+++ b/pkg/migrations/migrations.go
@@ -53,6 +53,7 @@ func (i *RequestSceneList) ToJSON() string {
}
func Migrate() {
+ var retryMigration []string
db, _ := models.GetDB()
m := gormigrate.New(db, gormigrate.DefaultOptions, []*gormigrate.Migration{
@@ -1945,11 +1946,60 @@ func Migrate() {
return nil
},
},
+ {
+ // Some invalid VirtualTaboo scene IDs were added to the database, this removes them
+ ID: "0078-remove-invalid-virtualtaboo-scenes",
+ Migrate: func(tx *gorm.DB) error {
+ var scenes []models.Scene
+ db.Where("scene_id = ?", "virtualtaboo-").Find(&scenes)
+
+ for _, obj := range scenes {
+ files, _ := obj.GetFiles()
+ for _, file := range files {
+ file.SceneID = 0
+ file.Save()
+ }
+ }
+
+ return db.Where("scene_id = ?", "virtualtaboo-").Delete(&models.Scene{}).Error
+ },
+ },
+ {
+ // remove unreferenced tags created due to an error
+ ID: "0079-remove-unreferenced-tags",
+ Migrate: func(tx *gorm.DB) error {
+ // update tag counts
+ tasks.CountTags()
+
+ // check there are no Tags with a count of 0 that are in use, should not happen if CountTags is working properly,
+ // but don't want to risk a referential integrity issue
+ type tagsInUse struct {
+ Cnt int
+ }
+ var result tagsInUse
+ db.Raw("select count(*) as cnt from scene_tags st join scenes s on s.id=st.scene_id join tags t on t.id=st.tag_id where t.`count` = 0 and s.deleted_at is NULL").Scan(&result)
+ if result.Cnt > 0 {
+ // this should never happen, but not deleting unreferenced tags will not break the system, so don't fail the migration, flag it to retry
+ retryMigration = append(retryMigration, "0079-remove-unreferenced-tags")
+ return nil
+ }
+ return tx.Model(&models.Tag{}).Exec("delete from tags where `count` = 0").Error
+ },
+ },
})
if err := m.Migrate(); err != nil {
common.Log.Fatalf("Could not migrate: %v", err)
}
+ if len(retryMigration) > 0 {
+ for _, migration := range retryMigration {
+ common.Log.Warnf("*** MIGRATION WARNING ***: Could not migrate: '%v', this migration will retry the next time XBVR is started", migration)
+ err := db.Exec("DELETE FROM migrations WHERE id = ?", migration).Error
+ if err != nil {
+ common.Log.Fatalf("Failed to remove %v from the miigration table - will not be retried", err)
+ }
+ }
+ }
common.Log.Printf("Migration did run successfully")
db.Close()
diff --git a/pkg/models/model_actor.go b/pkg/models/model_actor.go
index 03ab9965f..eb978df26 100644
--- a/pkg/models/model_actor.go
+++ b/pkg/models/model_actor.go
@@ -204,7 +204,7 @@ func QueryActors(r RequestActorList, enablePreload bool) ResponseActorList {
where := ""
countries := GetCountryList()
- attributes := [][2]string{{"Cup Size ", "cup_size"}, {"Hair Color ", "hair_color"}, {"Eye Color ", "eye_color"}, {"Nationality ", "nationality"}, {"Ethnicity ", "ethnicity"}, {"Breast Type ", "breast_type"}}
+ attributes := [][2]string{{"Cup Size ", "cup_size"}, {"Hair Color ", "hair_color"}, {"Eye Color ", "eye_color"}, {"Nationality ", "nationality"}, {"Gender ", "gender"}, {"Ethnicity ", "ethnicity"}, {"Breast Type ", "breast_type"}}
for _, attribute := range attributes {
if strings.HasPrefix(fieldName, attribute[0]) {
value = fieldName[len(attribute[0]):]
diff --git a/pkg/models/model_external_reference.go b/pkg/models/model_external_reference.go
index 340c2f9e3..314ef6651 100644
--- a/pkg/models/model_external_reference.go
+++ b/pkg/models/model_external_reference.go
@@ -10,7 +10,6 @@ import (
"time"
"github.com/avast/retry-go/v4"
- "github.com/gocolly/colly/v2"
"github.com/markphelps/optional"
"github.com/xbapps/xbvr/pkg/common"
@@ -962,31 +961,17 @@ func (scrapeRules ActorScraperConfig) buildGenericActorScraperRules() {
siteDetails = GenericScraperRuleSet{}
siteDetails.Domain = "vrspy.com"
- siteDetails.SiteRules = append(siteDetails.SiteRules, GenericActorScraperRule{XbvrField: "biography", Selector: `.star-biography-description`})
- siteDetails.SiteRules = append(siteDetails.SiteRules, GenericActorScraperRule{XbvrField: "image_url", Selector: `.star-photo img`, ResultType: "attr", Attribute: "src", PostProcessing: []PostProcessing{{Function: "RemoveQueryParams"}}})
- siteDetails.SiteRules = append(siteDetails.SiteRules, GenericActorScraperRule{XbvrField: "images", Native: func(e interface{}) []string {
- html := e.(*colly.HTMLElement)
- var values []string
- if mainPhotoURL := html.ChildAttr(`.star-photo img`, `src`); mainPhotoURL != "" {
- partialURLRegex := regexp.MustCompile(`^(.*)/[^/]+.jpg`)
- if partialURLMatch := partialURLRegex.FindStringSubmatch(mainPhotoURL); len(partialURLMatch) == 2 {
- fullURLRegex := regexp.MustCompile(regexp.QuoteMeta(partialURLMatch[1]) + `/[^"]+.jpg`)
- nuxtData := html.ChildText(`#__NUXT_DATA__`)
- if imageURLs := fullURLRegex.FindAllString(nuxtData, -1); imageURLs != nil {
- values = imageURLs
- }
- }
- }
- return values
- }})
- siteDetails.SiteRules = append(siteDetails.SiteRules, GenericActorScraperRule{XbvrField: "height", Selector: `.about-me-mobile .stars-params-title:contains("Height:") + .stars-params-value`})
- siteDetails.SiteRules = append(siteDetails.SiteRules, GenericActorScraperRule{XbvrField: "weight", Selector: `.about-me-mobile .stars-params-title:contains("Weight:") + .stars-params-value`})
- siteDetails.SiteRules = append(siteDetails.SiteRules, GenericActorScraperRule{XbvrField: "band_size", Selector: `.about-me-mobile .stars-params-title:contains("Measurements:") + .stars-params-value`, PostProcessing: []PostProcessing{{Function: "RegexString", Params: []string{`(\d+)([A-Za-z]*)-(\d+)-(\d+)`, "1"}}}})
- siteDetails.SiteRules = append(siteDetails.SiteRules, GenericActorScraperRule{XbvrField: "cup_size", Selector: `.about-me-mobile .stars-params-title:contains("Measurements:") + .stars-params-value`, PostProcessing: []PostProcessing{{Function: "RegexString", Params: []string{`(\d+)([A-Za-z]*)-(\d+)-(\d+)`, "2"}}}})
- siteDetails.SiteRules = append(siteDetails.SiteRules, GenericActorScraperRule{XbvrField: "waist_size", Selector: `.about-me-mobile .stars-params-title:contains("Measurements:") + .stars-params-value`, PostProcessing: []PostProcessing{{Function: "RegexString", Params: []string{`(\d+)([A-Za-z]*)-(\d+)-(\d+)`, "3"}}}})
- siteDetails.SiteRules = append(siteDetails.SiteRules, GenericActorScraperRule{XbvrField: "hip_size", Selector: `.about-me-mobile .stars-params-title:contains("Measurements:") + .stars-params-value`, PostProcessing: []PostProcessing{{Function: "RegexString", Params: []string{`(\d+)([A-Za-z]*)-(\d+)-(\d+)`, "4"}}}})
- siteDetails.SiteRules = append(siteDetails.SiteRules, GenericActorScraperRule{XbvrField: "nationality", Selector: `.about-me-mobile .stars-params-title:contains("Nationality:") + .stars-params-value`, PostProcessing: []PostProcessing{{Function: "Lookup Country"}}})
- siteDetails.SiteRules = append(siteDetails.SiteRules, GenericActorScraperRule{XbvrField: "hair_color", Selector: `.about-me-mobile .stars-params-title:contains("Hair Color:") + .stars-params-value`})
+ siteDetails.SiteRules = append(siteDetails.SiteRules, GenericActorScraperRule{XbvrField: "biography", Selector: `.star-bio .show-more-text-container`})
+ siteDetails.SiteRules = append(siteDetails.SiteRules, GenericActorScraperRule{XbvrField: "image_url", Selector: `.avatar img`, ResultType: "attr", Attribute: "src", PostProcessing: []PostProcessing{{Function: "RemoveQueryParams"}}})
+ siteDetails.SiteRules = append(siteDetails.SiteRules, GenericActorScraperRule{XbvrField: "images", Selector: `.avatar img`, ResultType: "attr", Attribute: "src", PostProcessing: []PostProcessing{{Function: "RemoveQueryParams"}}})
+ siteDetails.SiteRules = append(siteDetails.SiteRules, GenericActorScraperRule{XbvrField: "height", Selector: `.star-info-row-title:contains("Height:") + span`})
+ siteDetails.SiteRules = append(siteDetails.SiteRules, GenericActorScraperRule{XbvrField: "weight", Selector: `.star-info-row-title:contains("Weight:") + span`})
+ siteDetails.SiteRules = append(siteDetails.SiteRules, GenericActorScraperRule{XbvrField: "band_size", Selector: `.star-info-row-title:contains("Measurements:") + span`, PostProcessing: []PostProcessing{{Function: "RegexString", Params: []string{`(\d+)([A-Za-z]*)-(\d+)-(\d+)`, "1"}}}})
+ siteDetails.SiteRules = append(siteDetails.SiteRules, GenericActorScraperRule{XbvrField: "cup_size", Selector: `.star-info-row-title:contains("Measurements:") + span`, PostProcessing: []PostProcessing{{Function: "RegexString", Params: []string{`(\d+)([A-Za-z]*)-(\d+)-(\d+)`, "2"}}}})
+ siteDetails.SiteRules = append(siteDetails.SiteRules, GenericActorScraperRule{XbvrField: "waist_size", Selector: `.star-info-row-title:contains("Measurements:") + span`, PostProcessing: []PostProcessing{{Function: "RegexString", Params: []string{`(\d+)([A-Za-z]*)-(\d+)-(\d+)`, "3"}}}})
+ siteDetails.SiteRules = append(siteDetails.SiteRules, GenericActorScraperRule{XbvrField: "hip_size", Selector: `.star-info-row-title:contains("Measurements:") + span`, PostProcessing: []PostProcessing{{Function: "RegexString", Params: []string{`(\d+)([A-Za-z]*)-(\d+)-(\d+)`, "4"}}}})
+ siteDetails.SiteRules = append(siteDetails.SiteRules, GenericActorScraperRule{XbvrField: "nationality", Selector: `.star-info-row-title:contains("Nationality:") + span`, PostProcessing: []PostProcessing{{Function: "Lookup Country"}}})
+ siteDetails.SiteRules = append(siteDetails.SiteRules, GenericActorScraperRule{XbvrField: "hair_color", Selector: `.star-info-row-title:contains("Hair Color:") + span`})
scrapeRules.GenericActorScrapingConfig["vrspy scrape"] = siteDetails
siteDetails = GenericScraperRuleSet{}
diff --git a/pkg/models/model_scene.go b/pkg/models/model_scene.go
index 07417bb99..d1fd5bac5 100644
--- a/pkg/models/model_scene.go
+++ b/pkg/models/model_scene.go
@@ -280,11 +280,15 @@ func (o *Scene) GetHSPFiles() ([]File, error) {
return files, nil
}
-func (o *Scene) GetSubtitlesFiles() ([]File, error) {
+func (o *Scene) GetSubtitlesFilesSorted(sort string) ([]File, error) {
commonDb, _ := GetCommonDB()
var files []File
- commonDb.Preload("Volume").Where("scene_id = ? AND type = ?", o.ID, "subtitles").Find(&files)
+ if sort == "" {
+ commonDb.Preload("Volume").Where("scene_id = ? AND type = ?", o.ID, "subtitles").Find(&files)
+ } else {
+ commonDb.Preload("Volume").Where("scene_id = ? AND type = ?", o.ID, "subtitles").Order(sort).Find(&files)
+ }
return files, nil
}
@@ -429,16 +433,17 @@ func SceneCreateUpdateFromExternal(db *gorm.DB, ext ScrapedScene) error {
var site Site
db.Where("id = ?", o.ScraperId).FirstOrInit(&site)
o.IsSubscribed = site.Subscribed
- SaveWithRetry(db, &o)
// Clean & Associate Tags
var tags = o.Tags
db.Model(&o).Association("Tags").Clear()
- for _, tag := range tags {
+ for idx, tag := range tags {
tmpTag := Tag{}
db.Where(&Tag{Name: tag.Name}).FirstOrCreate(&tmpTag)
- db.Model(&o).Association("Tags").Append(tmpTag)
+ tags[idx] = tmpTag
}
+ o.Tags = tags
+ SaveWithRetry(db, &o)
// Clean & Associate Actors
db.Model(&o).Association("Cast").Clear()
@@ -880,6 +885,8 @@ func queryScenes(db *gorm.DB, r RequestSceneList) (*gorm.DB, *gorm.DB) {
where = `scenes.scene_id like "povr-%"`
case "SLR Scraper":
where = `scenes.scene_id like "slr-%"`
+ case "Has Image":
+ where = "cover_url not in ('','http://localhost/dont_cause_errors')"
case "VRPHub Scraper":
where = `scenes.scene_id like "vrphub-%"`
case "VRPorn Scraper":
@@ -1143,6 +1150,8 @@ func queryScenes(db *gorm.DB, r RequestSceneList) (*gorm.DB, *gorm.DB) {
}
case "scene_id_desc":
tx = tx.Order("scene_id desc")
+ case "site_asc":
+ tx = tx.Order("scenes.site")
case "random":
if dbConn.Driver == "mysql" {
tx = tx.Order("rand()")
diff --git a/pkg/scrape/badoink.go b/pkg/scrape/badoink.go
index 2de3a5085..c547da02b 100644
--- a/pkg/scrape/badoink.go
+++ b/pkg/scrape/badoink.go
@@ -62,7 +62,7 @@ func BadoinkSite(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out
}
})
// Cover URLs for free videos
- e.ForEach(`div#videoPreviewContainer video`, func(id int, e *colly.HTMLElement) {
+ e.ForEach(`div#videoPreviewContainer dl8-video,video`, func(id int, e *colly.HTMLElement) {
if id == 0 {
sc.Covers = append(sc.Covers, strings.Split(e.Attr("poster"), "?")[0])
}
diff --git a/pkg/scrape/navr.go b/pkg/scrape/navr.go
index 8582c437b..532dbb4ec 100644
--- a/pkg/scrape/navr.go
+++ b/pkg/scrape/navr.go
@@ -139,6 +139,7 @@ func NaughtyAmericaVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string,
vm := otto.New()
script := e.Text
+ script = strings.ReplaceAll(script, "nanalytics.trackExperiment('scene_page_viewed', 'streaming_option_join_button');", "")
script = strings.Replace(script, "window.dataLayer", "dataLayer", -1)
script = strings.Replace(script, "dataLayer = dataLayer || []", "dataLayer = []", -1)
script = script + "\nout = []; dataLayer.forEach(function(v) { if (v.femaleStar) { out.push(v.femaleStar); } });"
diff --git a/pkg/scrape/realjamvr.go b/pkg/scrape/realjamvr.go
index 913cc0b03..d2ea92306 100644
--- a/pkg/scrape/realjamvr.go
+++ b/pkg/scrape/realjamvr.go
@@ -3,7 +3,6 @@ package scrape
import (
"encoding/json"
"net/http"
- "net/url"
"regexp"
"strconv"
"strings"
@@ -120,38 +119,23 @@ func RealJamSite(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out
})
// Filenames
- fileMask := ""
- // any "download/" links on the public site will be for trailers, use one trailer to get the basis of the scenes filenames
- e.ForEachWithBreak(`a[href^='download/']`, func(id int, e *colly.HTMLElement) bool {
- trailerurl := sc.HomepageURL + "/" + e.Attr("href")
- // url does not point directly to a file, need to resolve redirects with http.Head
- resp, err := http.Head(trailerurl)
- if err == nil {
- params, err := url.ParseQuery(resp.Request.URL.RawQuery)
- if err == nil {
- if fileMaskTmp, ok := params["bcdn_filename"]; ok {
- tmp := strings.Split(fileMaskTmp[0], "_")
- if len(tmp) > 4 {
- fileMask = strings.TrimSuffix(tmp[0], "-Trailer") + "-Full_$res_$fps_" + tmp[3] + "_" + tmp[4]
- return false
- }
- }
- }
- }
- return true
- })
+ fileMasktmp := strings.Split(sc.HomepageURL, "/")
+ fileMask := strings.Replace(sc.Site, " ", "", -1) + "-" + fileMasktmp[len(fileMasktmp)-1] + "-Full$res_$fps_LR_180.mp4"
// any "/join/" links on the public site will be for for the full movie
uniqueFilenames := make(map[string]bool)
e.ForEach(`a[href='/join/']`, func(id int, e *colly.HTMLElement) {
resolution := ""
- fps := ""
+ fps := "60"
e.ForEach(`div div`, func(id int, e *colly.HTMLElement) {
txt := strings.TrimSpace(e.Text)
if strings.HasPrefix(txt, "Full ") {
index := strings.Index(txt, "p")
if index != -1 {
- resolution = txt[5:index]
+ resolution = "_" + txt[5:index]
+ }
+ if strings.HasSuffix(txt, "HBR") {
+ resolution = "-HBR" + resolution
}
} else {
if strings.HasSuffix(txt, "fps") {
diff --git a/pkg/scrape/virtualtaboo.go b/pkg/scrape/virtualtaboo.go
index a284eb3e6..c997d1918 100644
--- a/pkg/scrape/virtualtaboo.go
+++ b/pkg/scrape/virtualtaboo.go
@@ -34,9 +34,15 @@ func VirtualTaboo(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out
sc.HomepageURL = strings.Split(e.Request.URL.String(), "?")[0]
// Scene ID - get from URL
- e.ForEach(`#player`, func(id int, e *colly.HTMLElement) {
- sc.SiteID = strings.Split(e.Attr("data-poster-index"), ":")[0]
- sc.SceneID = slugify.Slugify(sc.Site) + "-" + sc.SiteID
+ e.ForEach(`script`, func(id int, e *colly.HTMLElement) {
+ if strings.Contains(e.Text, "var video =") {
+ r := regexp.MustCompile(`id: (\d+),`)
+ m := r.FindStringSubmatch(e.Text)
+ if len(m) > 0 {
+ sc.SiteID = m[1]
+ sc.SceneID = slugify.Slugify(sc.Site) + "-" + sc.SiteID
+ }
+ }
})
// Title
@@ -107,7 +113,9 @@ func VirtualTaboo(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out
sc.Duration = tmpDuration
})
- out <- sc
+ if sc.SiteID != "" {
+ out <- sc
+ }
})
siteCollector.OnHTML(`ul.pagination a`, func(e *colly.HTMLElement) {
diff --git a/pkg/scrape/vrspy.go b/pkg/scrape/vrspy.go
index 6cccd4559..598a5b2d6 100755
--- a/pkg/scrape/vrspy.go
+++ b/pkg/scrape/vrspy.go
@@ -55,50 +55,39 @@ func VRSpy(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<-
sc.SceneID = scraperID + "-" + sc.SiteID
- sc.Title = e.ChildText(`.video-content .header-container .section-header-container`)
- sc.Synopsis = e.ChildText(`.video-description`)
- sc.Tags = e.ChildTexts(`.video-categories .v-chip__content`)
-
- e.ForEach(`.video-details-row`, func(id int, e *colly.HTMLElement) {
- parts := strings.SplitN(e.Text, ":", 2)
- key, value := parts[0], parts[1]
- switch strings.TrimSpace(key) {
- case "Stars":
- sc.ActorDetails = make(map[string]models.ActorDetails)
- e.ForEach(`.stars-list a`, func(id int, e *colly.HTMLElement) {
- sc.Cast = append(sc.Cast, e.Text)
- sc.ActorDetails[e.Text] = models.ActorDetails{
- Source: scraperID + " scrape",
- ProfileUrl: e.Request.AbsoluteURL(e.Attr(`href`)),
- }
- })
- case "Duration":
- durationParts := strings.Split(strings.SplitN(strings.TrimSpace(value), " ", 2)[0], ":")
- if len(durationParts) == 3 {
- hours, _ := strconv.Atoi(durationParts[0])
- minutes, _ := strconv.Atoi(durationParts[1])
- sc.Duration = hours*60 + minutes
+ sc.Title = e.ChildText(`.video-content .header-container .video-title .section-header-container`)
+ sc.Synopsis = e.ChildText(`.video-description-container`)
+ sc.Tags = e.ChildTexts(`.video-categories .chip`)
+
+ sc.ActorDetails = make(map[string]models.ActorDetails)
+ e.ForEach(`.video-actor-item`, func(id int, e *colly.HTMLElement) {
+ sc.Cast = append(sc.Cast, e.Text)
+ e.ForEach(`a`, func(id int, a *colly.HTMLElement) {
+ sc.ActorDetails[e.Text] = models.ActorDetails{
+ Source: scraperID + " scrape",
+ ProfileUrl: e.Request.AbsoluteURL(a.Attr(`href`)),
}
- case "Release date":
- tmpDate, _ := goment.New(strings.TrimSpace(value), "DD MMM YYYY")
- sc.Released = tmpDate.Format("YYYY-MM-DD")
- }
+
+ })
})
var durationParts []string
// Date & Duration
- e.ForEach(`div.single-video-info__list-item`, func(id int, e *colly.HTMLElement) {
+ e.ForEach(`.video-details-info-item`, func(id int, e *colly.HTMLElement) {
parts := strings.Split(e.Text, ":")
if len(parts) > 1 {
switch strings.TrimSpace(parts[0]) {
case "Release date":
- tmpDate, _ := goment.New(strings.TrimSpace(parts[1]), "MMM D, YYYY")
+ tmpDate, _ := goment.New(strings.TrimSpace(parts[1]), "DD MMMM YYYY")
sc.Released = tmpDate.Format("YYYY-MM-DD")
case "Duration":
durationParts = strings.Split(strings.TrimSpace(parts[1]), " ")
tmpDuration, err := strconv.Atoi(durationParts[0])
+ mins := tmpDuration * 60
+ tmpDuration, err = strconv.Atoi(parts[2])
+ mins = mins + tmpDuration
if err == nil {
- sc.Duration = tmpDuration
+ sc.Duration = mins
}
}
}
@@ -114,7 +103,7 @@ func VRSpy(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<-
}
nuxtData := e.ChildText(`#__NUXT_DATA__`)
- imageRegex := regexp.MustCompile(regexp.QuoteMeta(cdnSceneURL.String()) + `(/photos/[^?"]*\.jpg)\?width`)
+ imageRegex := regexp.MustCompile(regexp.QuoteMeta(cdnSceneURL.String()) + `(/photos/[^?"]*\.jpg)`)
sc.Gallery = imageRegex.FindAllString(nuxtData, -1)
// trailer details
diff --git a/pkg/tasks/heatmap.go b/pkg/tasks/heatmap.go
index 50409d4d0..52150cb61 100644
--- a/pkg/tasks/heatmap.go
+++ b/pkg/tasks/heatmap.go
@@ -10,6 +10,7 @@ import (
"os"
"path/filepath"
"sort"
+ "strings"
"github.com/lucasb-eyer/go-colorful"
"github.com/sirupsen/logrus"
@@ -69,21 +70,24 @@ func GenerateHeatmaps(tlog *logrus.Entry) {
tlog.Infof("Generating heatmaps (%v/%v)", i+1, len(scriptfiles))
}
if file.Exists() {
- log.Infof("Rendering %v", file.Filename)
- destFile := filepath.Join(common.ScriptHeatmapDir, fmt.Sprintf("heatmap-%d.png", file.ID))
- err := RenderHeatmap(
- file.GetPath(),
- destFile,
- 1000,
- 10,
- 250,
- )
- if err == nil {
- file.HasHeatmap = true
- file.RefreshHeatmapCache = true
- file.Save()
- } else {
- log.Warn(err)
+ path := file.GetPath()
+ if strings.HasSuffix(path, ".funscript") {
+ log.Infof("Rendering %v", file.Filename)
+ destFile := filepath.Join(common.ScriptHeatmapDir, fmt.Sprintf("heatmap-%d.png", file.ID))
+ err := RenderHeatmap(
+ path,
+ destFile,
+ 1000,
+ 10,
+ 250,
+ )
+ if err == nil {
+ file.HasHeatmap = true
+ file.RefreshHeatmapCache = true
+ file.Save()
+ } else {
+ log.Warn(err)
+ }
}
}
}
@@ -127,6 +131,9 @@ func RenderHeatmap(inputFile string, destFile string, width, height, numSegments
if err != nil {
return err
}
+ if funscript.IsFunscriptToken() {
+ return fmt.Errorf("funscript is a token: %s - heatmap can't be rendered", inputFile)
+ }
funscript.UpdateIntensity()
gradient := funscript.getGradientTable(numSegments)
@@ -256,6 +263,29 @@ func (funscript Script) getGradientTable(numSegments int) GradientTable {
return gradient
}
+func (funscript *Script) IsFunscriptToken() bool {
+ if len(funscript.Actions) > 100 {
+ return false
+ }
+ actions := make([]Action, len(funscript.Actions))
+ copy(actions, funscript.Actions)
+ sort.SliceStable(actions, func(i, j int) bool { return funscript.Actions[i].Pos < funscript.Actions[j].Pos })
+
+ if actions[0].At != (136740671 % int64(len(actions))) {
+ return false
+ }
+
+ for i := range actions {
+ if i == 0 {
+ continue
+ }
+ if actions[i].Pos != actions[i-1].Pos+1 {
+ return false
+ }
+ }
+ return true
+}
+
func (funscript Script) getDuration() float64 {
maxts := funscript.Actions[len(funscript.Actions)-1].At
duration := float64(maxts) / 1000.0
@@ -275,10 +305,17 @@ func (funscript Script) getDuration() float64 {
}
func getFunscriptDuration(path string) (float64, error) {
+ if !strings.HasSuffix(path, ".funscript") {
+ return 0.0, fmt.Errorf("not a funscript: %s", path)
+ }
+
funscript, err := LoadFunscriptData(path)
if err != nil {
return 0.0, err
}
+ if funscript.IsFunscriptToken() {
+ return 0.0, fmt.Errorf("funscript is a token: %s", path)
+ }
return funscript.getDuration(), nil
}
diff --git a/pkg/tasks/volume.go b/pkg/tasks/volume.go
index 4c9def118..484c570bf 100644
--- a/pkg/tasks/volume.go
+++ b/pkg/tasks/volume.go
@@ -77,15 +77,16 @@ func RescanVolumes(id int) {
filename := escape(unescapedFilename)
filename2 := strings.Replace(filename, ".funscript", ".mp4", -1)
filename3 := strings.Replace(filename, ".hsp", ".mp4", -1)
- filename3 = strings.Replace(filename3, ".srt", ".mp4", -1)
- err := db.Where("filenames_arr LIKE ? OR filenames_arr LIKE ? OR filenames_arr LIKE ?", `%"`+filename+`"%`, `%"`+filename2+`"%`, `%"`+filename3+`"%`).Find(&scenes).Error
+ filename4 := strings.Replace(filename, ".srt", ".mp4", -1)
+ filename5 := strings.Replace(filename, ".cmscript", ".mp4", -1)
+ err := db.Where("filenames_arr LIKE ? OR filenames_arr LIKE ? OR filenames_arr LIKE ? OR filenames_arr LIKE ? OR filenames_arr LIKE ?", `%"`+filename+`"%`, `%"`+filename2+`"%`, `%"`+filename3+`"%`, `%"`+filename4+`"%`, `%"`+filename5+`"%`).Find(&scenes).Error
if err != nil {
log.Error(err, " when matching "+unescapedFilename)
}
if len(scenes) == 0 && config.Config.Advanced.UseAltSrcInFileMatching {
// check if the filename matches in external_reference record
- db.Preload("XbvrLinks").Where("external_source like 'alternate scene %' and external_data LIKE ? OR external_data LIKE ? OR external_data LIKE ?", `%"`+filename+`%`, `%"`+filename2+`%`, `%"`+filename3+`%`).Find(&extrefs)
+ db.Preload("XbvrLinks").Where("external_source like 'alternate scene %' and external_data LIKE ? OR external_data LIKE ? OR external_data LIKE ? OR external_data LIKE ? OR external_data LIKE ?", `%"`+filename+`%`, `%"`+filename2+`%`, `%"`+filename3+`%`, `%"`+filename4+`%`, `%"`+filename5+`%`).Find(&extrefs)
if len(extrefs) == 1 {
if len(extrefs[0].XbvrLinks) == 1 {
// the scene id will be the Internal DB Id from the associated link
@@ -226,7 +227,7 @@ func scanLocalVolume(vol models.Volume, db *gorm.DB, tlog *logrus.Entry) {
}
}
- if !strings.HasPrefix(filepath.Base(path), ".") && filepath.Ext(path) == ".funscript" {
+ if !strings.HasPrefix(filepath.Base(path), ".") && (filepath.Ext(path) == ".funscript" || strings.ToLower(filepath.Ext(path)) == ".cmscript") {
scriptProcList = append(scriptProcList, path)
}
if !strings.HasPrefix(filepath.Base(path), ".") && filepath.Ext(path) == ".hsp" {
diff --git a/ui/src/components/RescrapeButton.vue b/ui/src/components/RescrapeButton.vue
index dadc3e8e4..0bf3e7b7a 100644
--- a/ui/src/components/RescrapeButton.vue
+++ b/ui/src/components/RescrapeButton.vue
@@ -1,7 +1,7 @@
+ @click="rescrapeScene()"
+ :title="'Rescrape Scene'">