Improve Twitter API rate limit error handling and propagation

- Modified ScrapeTweetsByQuery to immediately return rate limit errors
- Updated TwitterQueryHandler to properly propagate scraper errors
- Adjusted handleWorkResponse to correctly handle and return errors to the client
- Ensured rate limit errors are logged and returned with appropriate HTTP status codes
- Improved error message clarity for better debugging and user feedback

This commit enhances the system's ability to detect, log, and respond to Twitter API rate limit errors, providing clearer feedback to both developers and end-users when such limits are encountered.

go.mod

@@ -4,7 +4,6 @@ go 1.22.0
 
 require (
 	github.com/cenkalti/backoff/v4 v4.3.0
-	github.com/chyeh/pubip v0.0.0-20170203095919-b7e679cf541c
 	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0
 	github.com/dgraph-io/badger v1.6.2
 	github.com/ethereum/go-ethereum v1.14.8
@@ -128,7 +127,6 @@ require (
 	github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect
 	github.com/jbenet/goprocess v0.1.4 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
-	github.com/jpillora/backoff v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/kennygrant/sanitize v1.2.4 // indirect
 	github.com/klauspost/compress v1.17.9 // indirect

go.sum

@@ -70,8 +70,6 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UF
 github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 h1:SKI1/fuSdodxmNNyVBR8d7X/HuLnRpvvFO0AgyQk764=
 github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927/go.mod h1:h/aW8ynjgkuj+NQRlZcDbAbM1ORAbXjXX77sX7T289U=
-github.com/chyeh/pubip v0.0.0-20170203095919-b7e679cf541c h1:++BhWlmSX+n8m3O4gPfy3S4PTZ0TMzH6nelerBLPUng=
-github.com/chyeh/pubip v0.0.0-20170203095919-b7e679cf541c/go.mod h1:C7ma6h458jTWT65mXC58L1Q6hnEtr0unur8cMc0UEXM=
 github.com/cilium/ebpf v0.2.0/go.mod h1:To2CFviqOWL/M0gIMsvSMlqe7em/l1ALkX1PyjrX2Qs=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y=
@@ -372,8 +370,6 @@ github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
 github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
-github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA=
-github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=

pkg/api/handlers_data.go

@@ -26,7 +26,7 @@ import (
 	"github.com/masa-finance/masa-oracle/pkg/scrapers/discord"
 	"github.com/masa-finance/masa-oracle/pkg/scrapers/telegram"
 	"github.com/masa-finance/masa-oracle/pkg/workers"
-	"github.com/masa-finance/masa-oracle/pkg/workers/types"
+	data_types "github.com/masa-finance/masa-oracle/pkg/workers/types"
 )
 
 type LLMChat struct {
@@ -122,25 +122,22 @@ func handleWorkResponse(c *gin.Context, responseCh chan data_types.WorkResponse,
 		select {
 		case response := <-responseCh:
 			if response.Error != "" {
-				c.JSON(http.StatusExpectationFailed, response)
+				logrus.Errorf("[+] Work error: %s", response.Error)
+				if strings.Contains(response.Error, "Rate limit exceeded") {
+					c.JSON(http.StatusTooManyRequests, gin.H{"error": "Twitter API rate limit exceeded"})
+				} else {
+					c.JSON(http.StatusInternalServerError, gin.H{"error": response.Error})
+				}
 				wg.Done()
 				return
 			}
-			if data, ok := response.Data.(string); ok && IsBase64(data) {
-				decodedData, err := base64.StdEncoding.DecodeString(response.Data.(string))
-				if err != nil {
-					c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to decode base64 data"})
-					return
-				}
-				var jsonData map[string]interface{}
-				err = json.Unmarshal(decodedData, &jsonData)
-				if err != nil {
-					c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to parse JSON data"})
-					return
-				}
-				response.Data = jsonData
+
+			if response.Data == nil {
+				c.JSON(http.StatusNotFound, gin.H{"error": "No data returned"})
+				wg.Done()
+				return
 			}
-			response.WorkRequest = nil
+
 			c.JSON(http.StatusOK, response)
 			wg.Done()
 			return

pkg/scrapers/twitter/tweets.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"path/filepath"
+	"strings"
 
 	_ "github.com/lib/pq"
 
@@ -128,6 +129,7 @@ func ScrapeTweetsForSentiment(query string, count int, model string) (string, st
 func ScrapeTweetsByQuery(query string, count int) ([]*TweetResult, error) {
 	scraper := auth()
 	var tweets []*TweetResult
+	var lastError error
 
 	if scraper == nil {
 		return nil, fmt.Errorf("there was an error authenticating with your Twitter credentials")
@@ -138,23 +140,19 @@ func ScrapeTweetsByQuery(query string, count int) ([]*TweetResult, error) {
 
 	// Perform the search with the specified query and count
 	for tweetResult := range scraper.SearchTweets(context.Background(), query, count) {
-		var tweet TweetResult
 		if tweetResult.Error != nil {
-			tweet = TweetResult{
-				Tweet: nil,
-				Error: tweetResult.Error,
-			}
-		} else {
-			tweet = TweetResult{
-				Tweet: &tweetResult.Tweet,
-				Error: nil,
+			lastError = tweetResult.Error
+			logrus.Warnf("[+] Error encountered while scraping tweet: %v", tweetResult.Error)
+			if strings.Contains(tweetResult.Error.Error(), "Rate limit exceeded") {
+				return nil, fmt.Errorf("Twitter API rate limit exceeded (429 error)")
 			}
+			continue
 		}
-		tweets = append(tweets, &tweet)
+		tweets = append(tweets, &TweetResult{Tweet: &tweetResult.Tweet, Error: nil})
 	}
 
-	if len(tweets) == 0 {
-		return nil, fmt.Errorf("no tweets found for the given query")
+	if len(tweets) == 0 && lastError != nil {
+		return nil, lastError
 	}
 
 	return tweets, nil

pkg/workers/handlers/twitter.go

@@ -6,7 +6,7 @@ import (
 	"github.com/sirupsen/logrus"
 
 	"github.com/masa-finance/masa-oracle/pkg/scrapers/twitter"
-	"github.com/masa-finance/masa-oracle/pkg/workers/types"
+	data_types "github.com/masa-finance/masa-oracle/pkg/workers/types"
 )
 
 type TwitterQueryHandler struct{}
@@ -16,17 +16,25 @@ type TwitterSentimentHandler struct{}
 type TwitterTrendsHandler struct{}
 
 func (h *TwitterQueryHandler) HandleWork(data []byte) data_types.WorkResponse {
-	logrus.Infof("[+] TwitterQueryHandler %s", data)
+	logrus.Infof("[+] TwitterQueryHandler input: %s", data)
 	dataMap, err := JsonBytesToMap(data)
 	if err != nil {
+		logrus.Errorf("[+] TwitterQueryHandler error parsing data: %v", err)
 		return data_types.WorkResponse{Error: fmt.Sprintf("unable to parse twitter query data: %v", err)}
 	}
 	count := int(dataMap["count"].(float64))
 	query := dataMap["query"].(string)
+
+	logrus.Infof("[+] Scraping tweets for query: %s, count: %d", query, count)
+
 	resp, err := twitter.ScrapeTweetsByQuery(query, count)
 	if err != nil {
-		return data_types.WorkResponse{Error: fmt.Sprintf("unable to get twitter query: %v", err)}
+		logrus.Errorf("[+] TwitterQueryHandler error scraping tweets: %v", err)
+		return data_types.WorkResponse{Error: err.Error()}
 	}
+
+	logrus.Infof("[+] TwitterQueryHandler response: %d tweets found", len(resp))
+
 	return data_types.WorkResponse{Data: resp}
 }
 

pkg/workers/worker_manager.go

@@ -15,6 +15,7 @@ import (
 
 	masa "github.com/masa-finance/masa-oracle/pkg"
 	"github.com/masa-finance/masa-oracle/pkg/config"
+	"github.com/masa-finance/masa-oracle/pkg/scrapers/twitter"
 	"github.com/masa-finance/masa-oracle/pkg/workers/handlers"
 	data_types "github.com/masa-finance/masa-oracle/pkg/workers/types"
 )
@@ -214,13 +215,29 @@ func (whm *WorkHandlerManager) ExecuteWork(workRequest data_types.WorkRequest) (
 	go func() {
 		startTime := time.Now()
 		workResponse := handler.HandleWork(workRequest.Data)
-		if workResponse.Error == "" {
-			duration := time.Since(startTime)
-			whm.mu.Lock()
-			handlerInfo := whm.handlers[workRequest.WorkType]
-			handlerInfo.CallCount++
-			handlerInfo.TotalRuntime += duration
-			whm.mu.Unlock()
+		duration := time.Since(startTime)
+		whm.mu.Lock()
+		handlerInfo := whm.handlers[workRequest.WorkType]
+		handlerInfo.CallCount++
+		handlerInfo.TotalRuntime += duration
+		whm.mu.Unlock()
+
+		if workResponse.Error != "" {
+			logrus.Errorf("[+] Work error for %s: %s", workRequest.WorkType, workResponse.Error)
+		} else if workResponse.Data == nil {
+			logrus.Warnf("[+] Work response for %s: No data returned", workRequest.WorkType)
+		} else {
+			switch data := workResponse.Data.(type) {
+			case []*twitter.TweetResult:
+				logrus.Infof("[+] Work response for %s: %d tweets returned", workRequest.WorkType, len(data))
+				if len(data) > 0 && data[0].Tweet != nil {
+					tweet := data[0].Tweet
+					logrus.Infof("[+] First tweet: ID: %s, Text: %s, Author: %s, CreatedAt: %s",
+						tweet.ID, tweet.Text, tweet.Username, tweet.TimeParsed)
+				}
+			default:
+				logrus.Infof("[+] Work response for %s: Data successfully returned (type: %T)", workRequest.WorkType, workResponse.Data)
+			}
 		}
 		responseChan <- workResponse
 	}()