fix(twitter): update package and follower response type to Profile

go.mod

@@ -18,6 +18,7 @@ require (
 	github.com/google/uuid v1.6.0
 	github.com/gotd/contrib v0.20.0
 	github.com/gotd/td v0.110.1
+	github.com/imperatrona/twitter-scraper v0.0.15
 	github.com/ipfs/go-cid v0.4.1
 	github.com/ipfs/go-datastore v0.6.0
 	github.com/ipfs/go-ds-leveldb v0.5.0
@@ -26,7 +27,7 @@ require (
 	github.com/libp2p/go-libp2p v0.36.3
 	github.com/libp2p/go-libp2p-kad-dht v0.26.1
 	github.com/libp2p/go-libp2p-pubsub v0.12.0
-	github.com/masa-finance/masa-twitter-scraper v0.0.1
+	github.com/masa-finance/masa-twitter-scraper v0.0.0-20241126070507-4938b90b44f4
 	github.com/multiformats/go-multiaddr v0.13.0
 	github.com/multiformats/go-multihash v0.2.3
 	github.com/onsi/ginkgo/v2 v2.20.2
@@ -42,6 +43,7 @@ require (
 )
 
 require (
+	github.com/AlexEidt/Vidio v1.5.1 // indirect
 	github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 // indirect
 	github.com/KyleBanks/depth v1.2.1 // indirect
 	github.com/Microsoft/go-winio v0.6.2 // indirect

go.sum

@@ -7,6 +7,8 @@ dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBr
 dmitri.shuralyov.com/service/change v0.0.0-20181023043359-a85b471d5412/go.mod h1:a1inKt/atXimZ4Mv927x+r7UpyzRUf4emIoiiSC2TN4=
 dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D6DFvNNtx+9ybjezNCa8XF0xaYcETyp6rHWU=
 git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg=
+github.com/AlexEidt/Vidio v1.5.1 h1:tovwvtgQagUz1vifiL9OeWkg1fP/XUzFazFKh7tFtaE=
+github.com/AlexEidt/Vidio v1.5.1/go.mod h1:djhIMnWMqPrC3X6nB6ymGX6uWWlgw+VayYGKE1bNwmI=
 github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 h1:cTp8I5+VIoKjsnZuH8vjyaysT/ses3EvZeaV/1UkF2M=
 github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
@@ -332,6 +334,8 @@ github.com/holiman/uint256 v1.3.1/go.mod h1:EOMSn4q6Nyt9P6efbI3bueV4e1b3dGlUCXei
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/huin/goupnp v1.3.0 h1:UvLUlWDNpoUdYzb2TCn+MuTWtcjXKSza2n6CBdQ0xXc=
 github.com/huin/goupnp v1.3.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8=
+github.com/imperatrona/twitter-scraper v0.0.15 h1:8BcaDK+pD4pPNgcV2UloHgnaIt47wCQIsaIDeGnM6rI=
+github.com/imperatrona/twitter-scraper v0.0.15/go.mod h1:38MY3g/h4V7Xl4HbW9lnkL8S3YiFZenBFv86hN57RG8=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/ipfs/boxo v0.21.0 h1:XpGXb+TQQ0IUdYaeAxGzWjSs6ow/Lce148A/2IbRDVE=
 github.com/ipfs/boxo v0.21.0/go.mod h1:NmweAYeY1USOaJJxouy7DLr/Y5M8UBSsCI2KRivO+TY=
@@ -449,8 +453,8 @@ github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0
 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
 github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd h1:br0buuQ854V8u83wA0rVZ8ttrq5CpaPZdvrK0LP2lOk=
 github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd/go.mod h1:QuCEs1Nt24+FYQEqAAncTDPJIuGs+LxK1MCiFL25pMU=
-github.com/masa-finance/masa-twitter-scraper v0.0.1 h1:n6GQZXmTCWqRr3c9m2wzKOjJfHDaRinLRmXUt10S/8U=
-github.com/masa-finance/masa-twitter-scraper v0.0.1/go.mod h1:Q6UBbAZsaBNDZ4hZQr8lvUUDmkJgQAhHiYWJcj0E+BM=
+github.com/masa-finance/masa-twitter-scraper v0.0.0-20241126070507-4938b90b44f4 h1:8E8Zp1raj1xIyT+zw4bp5kQJeofcmoJqo6f+4B88DCk=
+github.com/masa-finance/masa-twitter-scraper v0.0.0-20241126070507-4938b90b44f4/go.mod h1:Q6UBbAZsaBNDZ4hZQr8lvUUDmkJgQAhHiYWJcj0E+BM=
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
 github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
 github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=

pkg/scrapers/twitter/cookies.go

@@ -7,32 +7,66 @@ import (
 	"os"
 	"path/filepath"
 
-	twitterscraper "github.com/masa-finance/masa-twitter-scraper"
+	twitterscraper "github.com/imperatrona/twitter-scraper"
+
+	"github.com/sirupsen/logrus"
 )
 
 func SaveCookies(scraper *twitterscraper.Scraper, account *TwitterAccount, baseDir string) error {
+	logrus.Debugf("Saving cookies for user %s", account.Username)
 	cookieFile := filepath.Join(baseDir, fmt.Sprintf("%s_twitter_cookies.json", account.Username))
 	cookies := scraper.GetCookies()
+	logrus.Debugf("Got %d cookies to save", len(cookies))
+
 	data, err := json.Marshal(cookies)
 	if err != nil {
 		return fmt.Errorf("error marshaling cookies: %v", err)
 	}
+
+	logrus.Debugf("Writing cookies to file: %s", cookieFile)
 	if err = os.WriteFile(cookieFile, data, 0644); err != nil {
 		return fmt.Errorf("error saving cookies: %v", err)
 	}
+	logrus.Debug("Successfully saved cookies")
 	return nil
 }
 
 func LoadCookies(scraper *twitterscraper.Scraper, account *TwitterAccount, baseDir string) error {
+	logrus.Debugf("Loading cookies for user %s", account.Username)
 	cookieFile := filepath.Join(baseDir, fmt.Sprintf("%s_twitter_cookies.json", account.Username))
+
+	logrus.Debugf("Reading cookie file: %s", cookieFile)
 	data, err := os.ReadFile(cookieFile)
 	if err != nil {
 		return fmt.Errorf("error reading cookies: %v", err)
 	}
+
 	var cookies []*http.Cookie
 	if err = json.Unmarshal(data, &cookies); err != nil {
 		return fmt.Errorf("error unmarshaling cookies: %v", err)
 	}
+	logrus.Debugf("Loaded %d cookies from file", len(cookies))
+
+	// Verify critical cookies are present
+	var hasAuthToken, hasCSRFToken bool
+	for _, cookie := range cookies {
+		if cookie.Name == "auth_token" {
+			hasAuthToken = true
+			logrus.Debug("Found auth_token cookie")
+		}
+		if cookie.Name == "ct0" {
+			hasCSRFToken = true
+			logrus.Debug("Found CSRF token cookie")
+		}
+	}
+
+	if !hasAuthToken || !hasCSRFToken {
+		logrus.Debug("Missing critical authentication cookies")
+		return fmt.Errorf("missing critical authentication cookies")
+	}
+
+	logrus.Debug("Setting cookies in scraper")
 	scraper.SetCookies(cookies)
+	logrus.Debug("Successfully loaded and set cookies")
 	return nil
 }

pkg/scrapers/twitter/followers.go

@@ -3,11 +3,12 @@ package twitter
 import (
 	"fmt"
 
-	twitterscraper "github.com/masa-finance/masa-twitter-scraper"
+	twitterscraper "github.com/imperatrona/twitter-scraper"
+
 	"github.com/sirupsen/logrus"
 )
 
-func ScrapeFollowersForProfile(baseDir string, username string, count int) ([]twitterscraper.Legacy, error) {
+func ScrapeFollowersForProfile(baseDir string, username string, count int) ([]*twitterscraper.Profile, error) {
 	scraper, account, err := getAuthenticatedScraper(baseDir)
 	if err != nil {
 		return nil, err

pkg/scrapers/twitter/profile.go

@@ -1,7 +1,7 @@
 package twitter
 
 import (
-	twitterscraper "github.com/masa-finance/masa-twitter-scraper"
+	twitterscraper "github.com/imperatrona/twitter-scraper"
 )
 
 func ScrapeTweetsProfile(baseDir string, username string) (twitterscraper.Profile, error) {

pkg/scrapers/twitter/scraper.go

@@ -1,7 +1,7 @@
 package twitter
 
 import (
-	twitterscraper "github.com/masa-finance/masa-twitter-scraper"
+	twitterscraper "github.com/imperatrona/twitter-scraper"
 )
 
 type Scraper struct {

pkg/scrapers/twitter/tweets.go

@@ -3,7 +3,7 @@ package twitter
 import (
 	"context"
 
-	twitterscraper "github.com/masa-finance/masa-twitter-scraper"
+	twitterscraper "github.com/imperatrona/twitter-scraper"
 )
 
 type TweetResult struct {