Skip to content

Commit

Permalink
Merge branch 'Single_DB_open_for_scrape_functions_&_max_pool_size' in…
Browse files Browse the repository at this point in the history 
…to CoomonDbTest
  • Loading branch information
@toshski
toshski committed Jan 9, 2024
2 parents 3295daa + 33b250f commit 384d29f
Show file tree
Hide file tree
Showing 14 changed files with 96 additions and 109 deletions.
3 changes: 1 addition & 2 deletions pkg/api/actors.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -710,8 +710,7 @@ func (i ActorResource) editActorExtRefs(req *restful.Request, resp *restful.Resp

var links []models.ExternalReferenceLink

db, _ := models.GetDB()
defer db.Close()
db, _ := models.GetCommonDB()

// find any links that were removed
db.Preload("ExternalReference").Where("internal_table = 'actors' and internal_db_id = ?", id).Find(&links)
Expand Down
19 changes: 10 additions & 9 deletions pkg/common/common.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,15 +12,16 @@ var (
)

type EnvConfigSpec struct {
Debug bool `envconfig:"DEBUG" default:"false"`
DebugRequests bool `envconfig:"DEBUG_REQUESTS" default:"false"`
DebugSQL bool `envconfig:"DEBUG_SQL" default:"false"`
DebugWS bool `envconfig:"DEBUG_WS" default:"false"`
UIUsername string `envconfig:"UI_USERNAME" required:"false"`
UIPassword string `envconfig:"UI_PASSWORD" required:"false"`
DatabaseURL string `envconfig:"DATABASE_URL" required:"false" default:""`
WsAddr string `envconfig:"XBVR_WS_ADDR" required:"false" default:""`
WebPort int `envconfig:"XBVR_WEB_PORT" required:"false" default:"0"`
Debug bool `envconfig:"DEBUG" default:"false"`
DebugRequests bool `envconfig:"DEBUG_REQUESTS" default:"false"`
DebugSQL bool `envconfig:"DEBUG_SQL" default:"false"`
DebugWS bool `envconfig:"DEBUG_WS" default:"false"`
UIUsername string `envconfig:"UI_USERNAME" required:"false"`
UIPassword string `envconfig:"UI_PASSWORD" required:"false"`
DatabaseURL string `envconfig:"DATABASE_URL" required:"false" default:""`
WsAddr string `envconfig:"XBVR_WS_ADDR" required:"false" default:""`
WebPort int `envconfig:"XBVR_WEB_PORT" required:"false" default:"0"`
DBConnectionPoolSize int `envconfig:"DB_CONNECTION_POOL_SIZE" required:"false" default:"0"`
}

var EnvConfig EnvConfigSpec
Expand Down
7 changes: 7 additions & 0 deletions pkg/common/paths.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ var ScriptHeatmapDir string
var MyFilesDir string
var DownloadDir string
var WebPort int
var DBConnectionPoolSize int

func DirSize(path string) (int64, error) {
var size int64
Expand Down Expand Up @@ -51,6 +52,7 @@ func InitPaths() {
databaseurl := flag.String("database_url", "", "Optional: override default database path")
web_port := flag.Int("web_port", 0, "Optional: override default Web Page port 9999")
ws_addr := flag.String("ws_addr", "", "Optional: override default Websocket address from the default 0.0.0.0:9998")
dB_connection_pool_size := flag.Int("dB_connection_pool_size", 0, "Optional: sets a limit to the number of db connections while scraping")

flag.Parse()

Expand Down Expand Up @@ -113,6 +115,11 @@ func InitPaths() {
WsAddr = EnvConfig.WsAddr
}
}
if *dB_connection_pool_size != 0 {
DBConnectionPoolSize = *dB_connection_pool_size
} else {
DBConnectionPoolSize = EnvConfig.DBConnectionPoolSize
}

_ = os.MkdirAll(AppDir, os.ModePerm)
_ = os.MkdirAll(ImgDir, os.ModePerm)
Expand Down
33 changes: 33 additions & 0 deletions pkg/models/db.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import (
var log = &common.Log
var dbConn *dburl.URL
var supportedDB = []string{"mysql", "sqlite3"}
var commonConnection *gorm.DB

func parseDBConnString() {
var err error
Expand Down Expand Up @@ -78,6 +79,37 @@ func GetDB() (*gorm.DB, error) {
return db, nil
}

func GetCommonDB() (*gorm.DB, error) {
if common.EnvConfig.DebugSQL {
log.Debug("Getting Common DB handle from ", common.GetCallerFunctionName())
}

var err error

if commonConnection != nil {
return commonConnection, nil
}
err = retry.Do(
func() error {
commonConnection, err = gorm.Open(dbConn.Driver, dbConn.DSN)
commonConnection.LogMode(common.EnvConfig.DebugSQL)
if common.DBConnectionPoolSize > 0 {
commonConnection.DB().SetMaxOpenConns(common.DBConnectionPoolSize)
}
if err != nil {
return err
}
return nil
},
)

if err != nil {
log.Fatal("Failed to connect to database ", err)
}

return commonConnection, nil
}

// Lock functions

func CreateLock(lock string) {
Expand Down Expand Up @@ -127,4 +159,5 @@ func init() {
common.InitPaths()
common.InitLogging()
parseDBConnString()
GetCommonDB()
}
24 changes: 8 additions & 16 deletions pkg/models/model_actor.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,8 +97,7 @@ type ActorLink struct {
}

func (i *Actor) Save() error {
db, _ := GetDB()
defer db.Close()
db, _ := GetCommonDB()

var err error = retry.Do(
func() error {
Expand All @@ -118,8 +117,7 @@ func (i *Actor) Save() error {
}

func (i *Actor) CountActorTags() {
db, _ := GetDB()
defer db.Close()
db, _ := GetCommonDB()

type CountResults struct {
ID int
Expand Down Expand Up @@ -170,8 +168,7 @@ func QueryActors(r RequestActorList, enablePreload bool) ResponseActorList {
limit := r.Limit.OrElse(100)
offset := r.Offset.OrElse(0)

db, _ := GetDB()
defer db.Close()
db, _ := GetCommonDB()

var actors []Actor
tx := db.Model(&actors)
Expand Down Expand Up @@ -504,8 +501,7 @@ func QueryActors(r RequestActorList, enablePreload bool) ResponseActorList {
}

func (o *Actor) GetIfExist(id string) error {
db, _ := GetDB()
defer db.Close()
db, _ := GetCommonDB()

return db.
Preload("Scenes", func(db *gorm.DB) *gorm.DB {
Expand All @@ -515,8 +511,7 @@ func (o *Actor) GetIfExist(id string) error {
}

func (o *Actor) GetIfExistByPK(id uint) error {
db, _ := GetDB()
defer db.Close()
db, _ := GetCommonDB()

return db.
Preload("Scenes", func(db *gorm.DB) *gorm.DB {
Expand All @@ -526,8 +521,7 @@ func (o *Actor) GetIfExistByPK(id uint) error {
}

func (o *Actor) GetIfExistByPKWithSceneAvg(id uint) error {
db, _ := GetDB()
defer db.Close()
db, _ := GetCommonDB()

tx := db.Model(&Actor{})
tx = tx.Select(`actors.*,
Expand Down Expand Up @@ -631,17 +625,15 @@ func addToStringArray(inputArray string, newValue string) (string, bool) {

func (a *Actor) CheckForSetImage() bool {
// check if the field was deleted by the user,
db, _ := GetDB()
defer db.Close()
db, _ := GetCommonDB()
var action ActionActor
db.Where("source = 'edit_actor' and actor_id = ? and changed_column = 'image_url' and action_type = 'setimage'", a.ID).Order("ID desc").First(&action)
return action.ID != 0
}

func (a *Actor) CheckForUserDeletes(fieldName string, newValue string) bool {
// check if the field was deleted by the user,
db, _ := GetDB()
defer db.Close()
db, _ := GetCommonDB()
var action ActionActor
db.Where("source = 'edit_actor' and actor_id = ? and changed_column = ? and new_value = ?", a.ID, fieldName, newValue).Order("ID desc").First(&action)
if action.ID != 0 && action.ActionType == "delete" {
Expand Down
12 changes: 4 additions & 8 deletions pkg/models/model_aka.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,7 @@ type Aka struct {
}

func (i *Aka) Save() error {
db, _ := GetDB()
defer db.Close()
db, _ := GetCommonDB()

err := retry.Do(
func() error {
Expand All @@ -41,17 +40,15 @@ func (i *Aka) Save() error {
}

func (o *Aka) GetIfExistByPK(id uint) error {
db, _ := GetDB()
defer db.Close()
db, _ := GetCommonDB()

return db.
Preload("Actors").
Where(&Aka{ID: id}).First(o).Error
}

func (o *Aka) UpdateAkaSceneCastRecords() {
db, _ := GetDB()
defer db.Close()
db, _ := GetCommonDB()

// Queries to update the scene_cast table for the aka actor are comlex but fast.
// Significating faster than iterating through the results of multiple simpler queries.
Expand Down Expand Up @@ -95,8 +92,7 @@ func (o *Aka) UpdateAkaSceneCastRecords() {
}

func (o *Aka) RefreshAkaActorNames() {
db, _ := GetDB()
defer db.Close()
db, _ := GetCommonDB()

type SortedList struct {
AkaActorId uint
Expand Down
36 changes: 12 additions & 24 deletions pkg/models/model_external_reference.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,29 +90,25 @@ type SceneMatchRule struct {
}

func (o *ExternalReference) GetIfExist(id uint) error {
db, _ := GetDB()
defer db.Close()
db, _ := GetCommonDB()

return db.Preload("XbvrLinks").Where(&ExternalReference{ID: id}).First(o).Error
}

func (o *ExternalReference) FindExternalUrl(externalSource string, externalUrl string) error {
db, _ := GetDB()
defer db.Close()
db, _ := GetCommonDB()

return db.Preload("XbvrLinks").Where(&ExternalReference{ExternalSource: externalSource, ExternalURL: externalUrl}).First(o).Error
}

func (o *ExternalReference) FindExternalId(externalSource string, externalId string) error {
db, _ := GetDB()
defer db.Close()
db, _ := GetCommonDB()

return db.Preload("XbvrLinks").Where(&ExternalReference{ExternalSource: externalSource, ExternalId: externalId}).First(o).Error
}

func (o *ExternalReference) Save() {
db, _ := GetDB()
defer db.Close()
db, _ := GetCommonDB()

err := retry.Do(
func() error {
Expand All @@ -129,14 +125,12 @@ func (o *ExternalReference) Save() {
}

func (o *ExternalReference) Delete() {
db, _ := GetDB()
db, _ := GetCommonDB()
db.Delete(&o)
db.Close()
}

func (o *ExternalReference) AddUpdateWithUrl() {
db, _ := GetDB()
defer db.Close()
db, _ := GetCommonDB()

existingRef := ExternalReference{ExternalSource: o.ExternalSource, ExternalURL: o.ExternalURL}
existingRef.FindExternalUrl(o.ExternalSource, o.ExternalURL)
Expand Down Expand Up @@ -166,8 +160,7 @@ func (o *ExternalReference) AddUpdateWithUrl() {
}

func (o *ExternalReference) AddUpdateWithId() {
db, _ := GetDB()
defer db.Close()
db, _ := GetCommonDB()

existingRef := ExternalReference{ExternalSource: o.ExternalSource, ExternalId: o.ExternalId}
existingRef.FindExternalId(o.ExternalSource, o.ExternalId)
Expand Down Expand Up @@ -197,8 +190,7 @@ func (o *ExternalReference) AddUpdateWithId() {
}

func (o *ExternalReferenceLink) Save() {
db, _ := GetDB()
defer db.Close()
db, _ := GetCommonDB()

err := retry.Do(
func() error {
Expand All @@ -215,8 +207,7 @@ func (o *ExternalReferenceLink) Save() {
}

func (o *ExternalReferenceLink) Find(externalSource string, internalName string) error {
db, _ := GetDB()
defer db.Close()
db, _ := GetCommonDB()

return db.Where(&ExternalReferenceLink{ExternalSource: externalSource, InternalNameId: internalName}).First(o).Error
}
Expand Down Expand Up @@ -264,8 +255,7 @@ func (o *ExternalReference) DetermineActorScraperByUrl(url string) string {
}

func (o *ExternalReference) DetermineActorScraperBySiteId(siteId string) string {
db, _ := GetDB()
defer db.Close()
db, _ := GetCommonDB()

var site Site
db.Where("id = ?", siteId).First(&site)
Expand Down Expand Up @@ -304,8 +294,7 @@ func (config ActorScraperConfig) loadActorScraperRules() {
}

func (scrapeRules ActorScraperConfig) buildGenericActorScraperRules() {
db, _ := GetDB()
defer db.Close()
db, _ := GetCommonDB()
var sites []Site

// To understand the regex used, sign up to chat.openai.com and just ask something like Explain (.*, )?(.*)$
Expand Down Expand Up @@ -1080,8 +1069,7 @@ func (scrapeRules ActorScraperConfig) getCustomRules() {
}

func (scrapeRules ActorScraperConfig) getSiteUrlMatchingRules() {
db, _ := GetDB()
defer db.Close()
db, _ := GetCommonDB()

var sites []Site

Expand Down
Loading

0 comments on commit 384d29f

Please sign in to comment.