diff --git a/go.mod b/go.mod index e9ed2304..730e7072 100644 --- a/go.mod +++ b/go.mod @@ -27,7 +27,7 @@ require ( github.com/libp2p/go-libp2p v0.36.3 github.com/libp2p/go-libp2p-kad-dht v0.26.1 github.com/libp2p/go-libp2p-pubsub v0.12.0 - github.com/masa-finance/masa-twitter-scraper v0.0.0-20240910224030-76a02c878bd6 + github.com/masa-finance/masa-twitter-scraper v0.0.1 github.com/multiformats/go-multiaddr v0.13.0 github.com/multiformats/go-multihash v0.2.3 github.com/ollama/ollama v0.3.11 diff --git a/go.sum b/go.sum index cf81c962..faeb6a6f 100644 --- a/go.sum +++ b/go.sum @@ -451,8 +451,8 @@ github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd h1:br0buuQ854V8u83wA0rVZ8ttrq5CpaPZdvrK0LP2lOk= github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd/go.mod h1:QuCEs1Nt24+FYQEqAAncTDPJIuGs+LxK1MCiFL25pMU= -github.com/masa-finance/masa-twitter-scraper v0.0.0-20240910224030-76a02c878bd6 h1:m+WxxpznTnJtqnI0zogkhWEsVF+Dd1UP/lG3ZijDwkg= -github.com/masa-finance/masa-twitter-scraper v0.0.0-20240910224030-76a02c878bd6/go.mod h1:Q6UBbAZsaBNDZ4hZQr8lvUUDmkJgQAhHiYWJcj0E+BM= +github.com/masa-finance/masa-twitter-scraper v0.0.1 h1:n6GQZXmTCWqRr3c9m2wzKOjJfHDaRinLRmXUt10S/8U= +github.com/masa-finance/masa-twitter-scraper v0.0.1/go.mod h1:Q6UBbAZsaBNDZ4hZQr8lvUUDmkJgQAhHiYWJcj0E+BM= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= diff --git a/pkg/event/config.go b/pkg/event/config.go index cedb57f9..6ac80bf6 100644 --- a/pkg/event/config.go +++ b/pkg/event/config.go @@ -7,7 +7,7 @@ const ( APIVersion = "v1" // DefaultBaseURL is the default URL for the external API - DefaultBaseURL = "http://127.0.0.1:8081" + DefaultBaseURL = "https://test.protocol-api.masa.ai" // DefaultHTTPTimeout is the default timeout for HTTP requests DefaultHTTPTimeout = 10 * time.Second diff --git a/pkg/scrapers/twitter/cookies.go b/pkg/scrapers/twitter/cookies.go index 9450c20c..aef042e0 100644 --- a/pkg/scrapers/twitter/cookies.go +++ b/pkg/scrapers/twitter/cookies.go @@ -19,6 +19,12 @@ func SaveCookies(scraper *twitterscraper.Scraper, filePath string) error { if err != nil { return fmt.Errorf("error saving cookies to file: %v", err) } + + // Load the saved cookies back into the scraper + if err := LoadCookies(scraper, filePath); err != nil { + return fmt.Errorf("error loading saved cookies: %v", err) + } + return nil } diff --git a/pkg/scrapers/twitter/tweets.go b/pkg/scrapers/twitter/tweets.go index 21c0b717..a4061095 100644 --- a/pkg/scrapers/twitter/tweets.go +++ b/pkg/scrapers/twitter/tweets.go @@ -5,6 +5,7 @@ import ( "fmt" "path/filepath" "strings" + "time" _ "github.com/lib/pq" @@ -40,6 +41,8 @@ func auth() *twitterscraper.Scraper { password := appConfig.TwitterPassword twoFACode := appConfig.Twitter2FaCode + time.Sleep(500 * time.Millisecond) + var err error if twoFACode != "" { err = Login(scraper, username, password, twoFACode) @@ -52,6 +55,8 @@ func auth() *twitterscraper.Scraper { return nil } + time.Sleep(500 * time.Millisecond) + if err = SaveCookies(scraper, cookieFilePath); err != nil { logrus.WithError(err).Error("[-] Failed to save cookies") }