feat (twitter) scraper and user agent hot fix (#564)

* feat: update twitter scraper & add event API endpoint * feat: update to 500ms timeout and load cookies if not available
masa-finance · Sep 26, 2024 · 540191f · 540191f
1 parent 4940215
commit 540191f
Show file tree

Hide file tree

Showing 5 changed files with 15 additions and 4 deletions.
diff --git a/go.mod b/go.mod
@@ -27,7 +27,7 @@ require (
 	github.com/libp2p/go-libp2p v0.36.3
 	github.com/libp2p/go-libp2p-kad-dht v0.26.1
 	github.com/libp2p/go-libp2p-pubsub v0.12.0
-	github.com/masa-finance/masa-twitter-scraper v0.0.0-20240910224030-76a02c878bd6
+	github.com/masa-finance/masa-twitter-scraper v0.0.1
 	github.com/multiformats/go-multiaddr v0.13.0
 	github.com/multiformats/go-multihash v0.2.3
 	github.com/ollama/ollama v0.3.11

diff --git a/go.sum b/go.sum
@@ -451,8 +451,8 @@ github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0
 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
 github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd h1:br0buuQ854V8u83wA0rVZ8ttrq5CpaPZdvrK0LP2lOk=
 github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd/go.mod h1:QuCEs1Nt24+FYQEqAAncTDPJIuGs+LxK1MCiFL25pMU=
-github.com/masa-finance/masa-twitter-scraper v0.0.0-20240910224030-76a02c878bd6 h1:m+WxxpznTnJtqnI0zogkhWEsVF+Dd1UP/lG3ZijDwkg=
-github.com/masa-finance/masa-twitter-scraper v0.0.0-20240910224030-76a02c878bd6/go.mod h1:Q6UBbAZsaBNDZ4hZQr8lvUUDmkJgQAhHiYWJcj0E+BM=
+github.com/masa-finance/masa-twitter-scraper v0.0.1 h1:n6GQZXmTCWqRr3c9m2wzKOjJfHDaRinLRmXUt10S/8U=
+github.com/masa-finance/masa-twitter-scraper v0.0.1/go.mod h1:Q6UBbAZsaBNDZ4hZQr8lvUUDmkJgQAhHiYWJcj0E+BM=
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
 github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
 github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=

diff --git a/pkg/event/config.go b/pkg/event/config.go
@@ -7,7 +7,7 @@ const (
 	APIVersion = "v1"
 
 	// DefaultBaseURL is the default URL for the external API
-	DefaultBaseURL = "http://127.0.0.1:8081"
+	DefaultBaseURL = "https://test.protocol-api.masa.ai"
 
 	// DefaultHTTPTimeout is the default timeout for HTTP requests
 	DefaultHTTPTimeout = 10 * time.Second

diff --git a/pkg/scrapers/twitter/cookies.go b/pkg/scrapers/twitter/cookies.go
@@ -19,6 +19,12 @@ func SaveCookies(scraper *twitterscraper.Scraper, filePath string) error {
 	if err != nil {
 		return fmt.Errorf("error saving cookies to file: %v", err)
 	}
+
+	// Load the saved cookies back into the scraper
+	if err := LoadCookies(scraper, filePath); err != nil {
+		return fmt.Errorf("error loading saved cookies: %v", err)
+	}
+
 	return nil
 }
 

diff --git a/pkg/scrapers/twitter/tweets.go b/pkg/scrapers/twitter/tweets.go
@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"path/filepath"
 	"strings"
+	"time"
 
 	_ "github.com/lib/pq"
 
@@ -40,6 +41,8 @@ func auth() *twitterscraper.Scraper {
 	password := appConfig.TwitterPassword
 	twoFACode := appConfig.Twitter2FaCode
 
+	time.Sleep(500 * time.Millisecond)
+
 	var err error
 	if twoFACode != "" {
 		err = Login(scraper, username, password, twoFACode)
@@ -52,6 +55,8 @@ func auth() *twitterscraper.Scraper {
 		return nil
 	}
 
+	time.Sleep(500 * time.Millisecond)
+
 	if err = SaveCookies(scraper, cookieFilePath); err != nil {
 		logrus.WithError(err).Error("[-] Failed to save cookies")
 	}