diff --git a/pkg/scrapers/twitter/auth.go b/pkg/scrapers/twitter/auth.go index a6be7767..47163aeb 100644 --- a/pkg/scrapers/twitter/auth.go +++ b/pkg/scrapers/twitter/auth.go @@ -17,14 +17,14 @@ func NewScraper(account *TwitterAccount, cookieDir string) *Scraper { } } - ShortSleep() + RandomSleep() if err := scraper.Login(account.Username, account.Password, account.TwoFACode); err != nil { logrus.WithError(err).Warnf("Login failed for %s", account.Username) return nil } - ShortSleep() + RandomSleep() if err := SaveCookies(scraper.Scraper, account, cookieDir); err != nil { logrus.WithError(err).Errorf("Failed to save cookies for %s", account.Username) diff --git a/pkg/scrapers/twitter/config.go b/pkg/scrapers/twitter/config.go index a0bfdf98..42b69a88 100644 --- a/pkg/scrapers/twitter/config.go +++ b/pkg/scrapers/twitter/config.go @@ -1,16 +1,30 @@ package twitter import ( + "math/rand" "time" + + "github.com/sirupsen/logrus" ) const ( - ShortSleepDuration = 20 * time.Millisecond - RateLimitDuration = 15 * time.Minute + minSleepDuration = 500 * time.Millisecond + maxSleepDuration = 2 * time.Second + RateLimitDuration = 15 * time.Minute +) + +var ( + rng *rand.Rand ) -func ShortSleep() { - time.Sleep(ShortSleepDuration) +func init() { + rng = rand.New(rand.NewSource(time.Now().UnixNano())) +} + +func RandomSleep() { + duration := minSleepDuration + time.Duration(rng.Int63n(int64(maxSleepDuration-minSleepDuration))) + logrus.Debugf("Sleeping for %v", duration) + time.Sleep(duration) } func GetRateLimitDuration() time.Duration {