Skip to content

Commit

Permalink
Merge branch 'xbapps:master' into util
Browse files Browse the repository at this point in the history
  • Loading branch information
@theRealKLH
theRealKLH authored Sep 1, 2024
2 parents 54d377b + 943283e commit 64a492b
Show file tree
Hide file tree
Showing 13 changed files with 194 additions and 70 deletions.
34 changes: 18 additions & 16 deletions pkg/api/options.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,22 +43,23 @@ type VersionCheckResponse struct {
}

type RequestSaveOptionsWeb struct {
TagSort string `json:"tagSort"`
SceneHidden bool `json:"sceneHidden"`
SceneWatchlist bool `json:"sceneWatchlist"`
SceneFavourite bool `json:"sceneFavourite"`
SceneWishlist bool `json:"sceneWishlist"`
SceneWatched bool `json:"sceneWatched"`
SceneEdit bool `json:"sceneEdit"`
SceneDuration bool `json:"sceneDuration"`
SceneCuepoint bool `json:"sceneCuepoint"`
ShowHspFile bool `json:"showHspFile"`
ShowSubtitlesFile bool `json:"showSubtitlesFile"`
SceneTrailerlist bool `json:"sceneTrailerlist"`
ShowScriptHeatmap bool `json:"showScriptHeatmap"`
ShowAllHeatmaps bool `json:"showAllHeatmaps"`
UpdateCheck bool `json:"updateCheck"`
IsAvailOpacity int `json:"isAvailOpacity"`
TagSort string `json:"tagSort"`
SceneHidden bool `json:"sceneHidden"`
SceneWatchlist bool `json:"sceneWatchlist"`
SceneFavourite bool `json:"sceneFavourite"`
SceneWishlist bool `json:"sceneWishlist"`
SceneWatched bool `json:"sceneWatched"`
SceneEdit bool `json:"sceneEdit"`
SceneDuration bool `json:"sceneDuration"`
SceneCuepoint bool `json:"sceneCuepoint"`
ShowHspFile bool `json:"showHspFile"`
ShowSubtitlesFile bool `json:"showSubtitlesFile"`
SceneTrailerlist bool `json:"sceneTrailerlist"`
ShowScriptHeatmap bool `json:"showScriptHeatmap"`
ShowAllHeatmaps bool `json:"showAllHeatmaps"`
ShowOpenInNewWindow bool `json:"showOpenInNewWindow"`
UpdateCheck bool `json:"updateCheck"`
IsAvailOpacity int `json:"isAvailOpacity"`
}

type RequestSaveOptionsAdvanced struct {
Expand Down Expand Up @@ -470,6 +471,7 @@ func (i ConfigResource) saveOptionsWeb(req *restful.Request, resp *restful.Respo
config.Config.Web.SceneTrailerlist = r.SceneTrailerlist
config.Config.Web.ShowScriptHeatmap = r.ShowScriptHeatmap
config.Config.Web.ShowAllHeatmaps = r.ShowAllHeatmaps
config.Config.Web.ShowOpenInNewWindow = r.ShowOpenInNewWindow
config.Config.Web.UpdateCheck = r.UpdateCheck
config.Config.Web.IsAvailOpacity = r.IsAvailOpacity
config.SaveConfig()
Expand Down
33 changes: 17 additions & 16 deletions pkg/config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,22 +30,23 @@ type ObjectConfig struct {
Password string `default:"" json:"password"`
} `json:"security"`
Web struct {
TagSort string `default:"by-tag-count" json:"tagSort"`
SceneHidden bool `default:"true" json:"sceneHidden"`
SceneWatchlist bool `default:"true" json:"sceneWatchlist"`
SceneFavourite bool `default:"true" json:"sceneFavourite"`
SceneWishlist bool `default:"true" json:"sceneWishlist"`
SceneWatched bool `default:"false" json:"sceneWatched"`
SceneEdit bool `default:"false" json:"sceneEdit"`
SceneDuration bool `default:"false" json:"sceneDuration"`
SceneCuepoint bool `default:"true" json:"sceneCuepoint"`
ShowHspFile bool `default:"true" json:"showHspFile"`
ShowSubtitlesFile bool `default:"true" json:"showSubtitlesFile"`
SceneTrailerlist bool `default:"true" json:"sceneTrailerlist"`
ShowScriptHeatmap bool `default:"true" json:"showScriptHeatmap"`
ShowAllHeatmaps bool `default:"false" json:"showAllHeatmaps"`
UpdateCheck bool `default:"true" json:"updateCheck"`
IsAvailOpacity int `default:"40" json:"isAvailOpacity"`
TagSort string `default:"by-tag-count" json:"tagSort"`
SceneHidden bool `default:"true" json:"sceneHidden"`
SceneWatchlist bool `default:"true" json:"sceneWatchlist"`
SceneFavourite bool `default:"true" json:"sceneFavourite"`
SceneWishlist bool `default:"true" json:"sceneWishlist"`
SceneWatched bool `default:"false" json:"sceneWatched"`
SceneEdit bool `default:"false" json:"sceneEdit"`
SceneDuration bool `default:"false" json:"sceneDuration"`
SceneCuepoint bool `default:"true" json:"sceneCuepoint"`
ShowHspFile bool `default:"true" json:"showHspFile"`
ShowSubtitlesFile bool `default:"true" json:"showSubtitlesFile"`
SceneTrailerlist bool `default:"true" json:"sceneTrailerlist"`
ShowScriptHeatmap bool `default:"true" json:"showScriptHeatmap"`
ShowAllHeatmaps bool `default:"false" json:"showAllHeatmaps"`
ShowOpenInNewWindow bool `default:"true" json:"showOpenInNewWindow"`
UpdateCheck bool `default:"true" json:"updateCheck"`
IsAvailOpacity int `default:"40" json:"isAvailOpacity"`
} `json:"web"`
Advanced struct {
ShowInternalSceneId bool `default:"false" json:"showInternalSceneId"`
Expand Down
33 changes: 17 additions & 16 deletions pkg/config/state.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,22 +12,23 @@ type ObjectState struct {
BoundIP []string `json:"bound_ip"`
} `json:"server"`
Web struct {
TagSort string `json:"tagSort"`
SceneHidden bool `json:"sceneHidden"`
SceneWatchlist bool `json:"sceneWatchlist"`
SceneFavourite bool `json:"sceneFavourite"`
SceneWishlist bool `json:"sceneWishlist"`
SceneWatched bool `json:"sceneWatched"`
SceneEdit bool `json:"sceneEdit"`
SceneDuration bool `json:"sceneDuration"`
SceneCuepoint bool `json:"sceneCuepoint"`
ShowHspFile bool `json:"showHspFile"`
ShowSubtitlesFile bool `json:"showSubtitlesFile"`
SceneTrailerlist bool `json:"sceneTrailerlist"`
ShowScriptHeatmap bool `json:"showScriptHeatmap"`
ShowAllHeatmaps bool `json:"showAllHeatmaps"`
UpdateCheck bool `json:"updateCheck"`
IsAvailOpacity int `json:"isAvailOpacity"`
TagSort string `json:"tagSort"`
SceneHidden bool `json:"sceneHidden"`
SceneWatchlist bool `json:"sceneWatchlist"`
SceneFavourite bool `json:"sceneFavourite"`
SceneWishlist bool `json:"sceneWishlist"`
SceneWatched bool `json:"sceneWatched"`
SceneEdit bool `json:"sceneEdit"`
SceneDuration bool `json:"sceneDuration"`
SceneCuepoint bool `json:"sceneCuepoint"`
ShowHspFile bool `json:"showHspFile"`
ShowSubtitlesFile bool `json:"showSubtitlesFile"`
SceneTrailerlist bool `json:"sceneTrailerlist"`
ShowScriptHeatmap bool `json:"showScriptHeatmap"`
ShowAllHeatmaps bool `json:"showAllHeatmaps"`
ShowOpenInNewWindow bool `json:"showOpenInNewWindow"`
UpdateCheck bool `json:"updateCheck"`
IsAvailOpacity int `json:"isAvailOpacity"`
} `json:"web"`
DLNA struct {
Running bool `json:"running"`
Expand Down
4 changes: 2 additions & 2 deletions pkg/scrape/povr.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,8 +124,8 @@ func POVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<-
siteCollector.OnHTML(`div.thumbnail-wrap div.thumbnail a.thumbnail__link`, func(e *colly.HTMLElement) {
sceneURL := e.Request.AbsoluteURL(e.Attr("href"))

// If scene exists in database, or the slternate source exists, there's no need to scrape
if !funk.ContainsString(knownScenes, sceneURL) && !strings.Contains(sceneURL, "/join") {
// If scene exists in database, or the slternate source exists, there's no need to scrape. Also make sure we only grab valid scene links in the vr-porn directory
if !funk.ContainsString(knownScenes, sceneURL) && strings.Contains(sceneURL, "/vr-porn/") && !strings.Contains(sceneURL, "/join") {
WaitBeforeVisit("povr.com", sceneCollector.Visit, sceneURL)
}
})
Expand Down
28 changes: 28 additions & 0 deletions pkg/scrape/sinsvr.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package scrape

import (
"encoding/json"
"net/http"
"regexp"
"strconv"
"strings"
Expand Down Expand Up @@ -66,15 +67,18 @@ func SinsVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<
sc.ActorDetails = make(map[string]models.ActorDetails)
e.ForEach(`.video-detail__specs div.cell`, func(id int, e *colly.HTMLElement) {
c := strings.TrimSpace(e.Text)

// Cast
if strings.Contains(c, "Starring") {
e.ForEach(`.cell a`, func(id int, e *colly.HTMLElement) {

cast := strings.Split(e.Text, ",")
sc.Cast = append(sc.Cast, cast...)
if len(cast) > 1 {
sc.ActorDetails[strings.TrimSpace(e.Text)] = models.ActorDetails{Source: sc.ScraperID + " scrape", ProfileUrl: e.Request.AbsoluteURL(e.Attr("href"))}
}
sc.ActorDetails[strings.TrimSpace(e.Text)] = models.ActorDetails{Source: sc.ScraperID + " scrape", ProfileUrl: e.Request.AbsoluteURL(e.Attr("href"))}

})
} else {
// Released - Date Oct 19, 2019
Expand All @@ -85,6 +89,30 @@ func SinsVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<
}
})

// Fallback incase SinsVR forgets to add the actor to the scene. This uses the trailer URL which contains the models in the scene.
if len(sc.Cast) == 0 {
trailerURL := e.ChildAttr("div.video-player-container__player source", "src")
//The cast is the first part of the trailer file name in the URL
re := regexp.MustCompile(`https:\/\/public\.xsinsvr\.com\/video\/.+\/(?P<cast>[A-Za-z_-]+)_trailer`)
r := re.FindStringSubmatch(trailerURL)
castIndex := re.SubexpIndex("cast")
//sinsVR uses _ for whitespace and - to separate cast members
cast := strings.Split(strings.ReplaceAll(r[castIndex], "_", " "), "-")
sc.Cast = append(sc.Cast, cast...)

// This should result in the correct model url for sinsVR but occasionally sins has yet to create the model url and will result in a 404
for _, name := range cast {
profileUrl := `https://xsinsvr.com/model/` + strings.ToLower(strings.ReplaceAll(name, " ", "-"))
profileUrlResp, err := http.Head(profileUrl)
if err != nil {
log.Errorf("Method Head Failed on profileUrlResp %s with error %s", profileUrlResp, err)
} else if profileUrlResp.StatusCode == 200 { //The url is not valid don't bother adding it to the ActorDetails
sc.ActorDetails[name] = models.ActorDetails{Source: sc.ScraperID + " scrape", ProfileUrl: profileUrl}
}
defer profileUrlResp.Body.Close()
}
}

// Duration
durationText := e.ChildText(`div.video-player-container__info div.tn-video-props span`)
for _, regex := range durationRegexes {
Expand Down
14 changes: 9 additions & 5 deletions pkg/scrape/slrstudios.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -151,21 +151,25 @@ func SexLikeReal(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out
if !isTransScene {
appCover := gjson.Get(JsonMetadataA, "thumbnailUrl").String()
desktopCover := strings.Replace(gjson.Get(JsonMetadataA, "thumbnailUrl").String(), "app", "desktop", -1)
desktopCresp, _ := http.Head(desktopCover)
if desktopCresp.StatusCode == 200 {
desktopCresp, err := http.Head(desktopCover)
if err != nil {
log.Errorf("Method Head Failed on desktopCover %s with error %s", desktopCover, err)
} else if desktopCresp.StatusCode == 200 {
coverURL := desktopCover
sc.Covers = append(sc.Covers, coverURL)
} else {
appCresp, _ := http.Head(appCover)
if appCresp.StatusCode == 200 {
appCresp, err := http.Head(appCover)
if err != nil {
log.Errorf("Method Head Failed on appCover %s with error %s", appCover, err)
} else if appCresp.StatusCode == 200 {
coverURL := appCover
sc.Covers = append(sc.Covers, coverURL)
defer appCresp.Body.Close()
} else {
e.ForEach(`link[as="image"]`, func(id int, e *colly.HTMLElement) {
sc.Covers = append(sc.Covers, e.Request.AbsoluteURL(e.Attr("href")))
})
}
defer appCresp.Body.Close()
}
defer desktopCresp.Body.Close()
} else {
Expand Down
1 change: 1 addition & 0 deletions pkg/scrape/vrallure.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ func VRAllure(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out cha
sc.Studio = "VRAllure"
sc.Site = siteID
sc.HomepageURL = strings.Split(e.Request.URL.String(), "?")[0]
sc.MembersUrl = strings.Replace(sc.HomepageURL, "https://vrallure.com/scenes/", "https://ma.vrallure.com/scene/", 1)

// Scene ID - get from URL
tmp := strings.Split(sc.HomepageURL, "/")
Expand Down
2 changes: 1 addition & 1 deletion pkg/scrape/vrspy.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ func VRSpy(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<-
sc.Site = siteID
sc.HomepageURL = e.Request.URL.String()

ogimage := e.ChildAttr(`meta[property="og:image"]`, "content")
ogimage := e.ChildAttr(`meta[property="og:image"][content*="cover.jpg"]`, "content")
if ogimage != "" {
ogimageURL, err := url.Parse(ogimage)
if err == nil {
Expand Down
20 changes: 15 additions & 5 deletions pkg/scrape/wetvr.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package scrape

import (
"encoding/json"
"fmt"
"strings"
"sync"
"time"
Expand All @@ -21,6 +22,7 @@ func WetVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<-

sceneCollector := createCollector("wetvr.com")
siteCollector := createCollector("wetvr.com")
pageCnt := 1

sceneCollector.OnHTML(`div#trailer_player`, func(e *colly.HTMLElement) {
sc := models.ScrapedScene{}
Expand Down Expand Up @@ -89,10 +91,17 @@ func WetVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<-
out <- sc
})

siteCollector.OnHTML(`ul a.page-link`, func(e *colly.HTMLElement) {
//This p with Latest only exists on pages container sceneURLs. If the index page doesn't have this we have reached the end of sceneURLs
// Stop search for more siteURLs
siteCollector.OnHTML(`div:has(p:contains("Latest"))`, func(e *colly.HTMLElement) {
// Check to make sure we aren't getting multiple firings from pages we have already incremented the count on
if !limitScraping {
pageURL := e.Request.AbsoluteURL(e.Attr("href"))
siteCollector.Visit(pageURL)
if e.Request.URL.String() == `https://wetvr.com/?page=`+fmt.Sprint(pageCnt) {
pageCnt += 1

pageURL := e.Request.AbsoluteURL(`https://wetvr.com/?page=` + fmt.Sprint(pageCnt))
siteCollector.Visit(pageURL)
}
}
})

Expand Down Expand Up @@ -133,8 +142,9 @@ func WetVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<-
ctx.Put("scene-date", "")
}
sceneCollector.Request("GET", singleSceneURL, nil, ctx, nil)
} else {
siteCollector.Visit("https://wetvr.com/")
} else if pageCnt == 1 {
// Only visit page 1 on start up of scraper
siteCollector.Visit("https://wetvr.com/?page=1")
}

if updateSite {
Expand Down
5 changes: 4 additions & 1 deletion ui/src/store/optionsWeb.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ const state = {
isAvailOpacity: 40,
showScriptHeatmap: false,
showAllHeatmaps: false,
showOpenInNewWindow: true,
updateCheck: true
}
}
Expand Down Expand Up @@ -46,7 +47,8 @@ const actions = {
state.web.showScriptHeatmap = data.config.web.showScriptHeatmap
state.web.showAllHeatmaps = data.config.web.showAllHeatmaps
state.web.updateCheck = data.config.web.updateCheck
state.web.isAvailOpacity = data.config.web.isAvailOpacity
state.web.isAvailOpacity = data.config.web.isAvailOpacity
state.web.showOpenInNewWindow = data.config.web.showOpenInNewWindow
state.loading = false
})
},
Expand All @@ -71,6 +73,7 @@ const actions = {
state.web.showAllHeatmaps = data.showAllHeatmaps
state.web.updateCheck = data.updateCheck
state.web.isAvailOpacity = data.isAvailOpacity
state.web.showOpenInNewWindow = data.showOpenInNewWindow
state.loading = false
})
}
Expand Down
21 changes: 20 additions & 1 deletion ui/src/views/actors/ActorDetails.vue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -151,7 +151,10 @@
{{ actor.biography }}
</b-message>
</b-tab-item>
<b-tab-item :label="`Scenes (${actor.scenes.length})`">
<b-tab-item>
<template #header>
Scenes ({{ actor.scenes.length }}) <a v-if="showOpenInNewWindow" :href='getCastScenesUrl([actor.name])' target="_blank" style="padding-left: 0.1em; border-bottom-style: none;"><b-icon pack="mdi" icon="open-in-new" size="is-small" style="background-color: hsl(0, 0%, 100%);"></b-icon></a>
</template>
<div v-show="activeTab == 1" :class="['columns', 'is-multiline', actor.scenes.length > 6 ? 'scroll' : '']">
<div :class="['column', 'is-multiline', 'is-one-third']"
v-for="(scene, idx) in actor.scenes" :key="idx" class="image-wrapper">
Expand Down Expand Up @@ -312,6 +315,9 @@ export default {
showEdit () {
return this.$store.state.overlay.actoredit.show
},
showOpenInNewWindow () {
return this.$store.state.optionsWeb.web.showOpenInNewWindow
},
},
mounted () {
ky.get('/api/actor/countrylist')
Expand Down Expand Up @@ -595,6 +601,19 @@ export default {
}
return true
},
getCastScenesUrl(actor) {
let newfilters = Object.assign({}, this.$store.state.sceneList.filters);
console.log(newfilters)
newfilters.cast = actor;
newfilters.dlState = "any"
newfilters.isAvailable=null
newfilters.isAccessible=null
console.log(newfilters)
return this.$router.resolve({
name: 'scenes',
query: { q: Buffer.from(JSON.stringify(newfilters)).toString('base64') }
}).href
},
refreshScraper(url){
if (url.includes('stashdb')) {
this.$store.state.actorList.isLoading = true
Expand Down
13 changes: 13 additions & 0 deletions ui/src/views/options/sections/InterfaceWeb.vue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,11 @@
show All Heatmaps
</b-switch>
</b-field>
<b-field>
<b-switch v-model="openInNewWindow" type="is-dark">
show Open Tag in New Window
</b-switch>
</b-field>
<b-field label="Opacity of unavailable scenes">
<div class="columns">
<div class="column is-two-thirds">
Expand Down Expand Up @@ -241,6 +246,14 @@ export default {
this.$store.state.optionsWeb.web.showSubtitlesFile = value
}
},
openInNewWindow: {
get () {
return this.$store.state.optionsWeb.web.showOpenInNewWindow
},
set (value) {
this.$store.state.optionsWeb.web.showOpenInNewWindow = value
}
},
isAvailOpacity: {
get () {
if (this.$store.state.optionsWeb.web.isAvailOpacity == undefined) {
Expand Down
Loading

0 comments on commit 64a492b

Please sign in to comment.