diff --git a/.github/workflows/build-goreleaser-test.yml b/.github/workflows/build-goreleaser-test.yml new file mode 100644 index 00000000..761f013d --- /dev/null +++ b/.github/workflows/build-goreleaser-test.yml @@ -0,0 +1,25 @@ +name: goreleaser-build-test + +on: + push: + pull_request: + +jobs: + goreleaser: # Add this new job for GoReleaser + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: 1.22 + - name: Run GoReleaser + uses: goreleaser/goreleaser-action@v6 + with: + version: v2.1.0 + args: build --clean --snapshot + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/goreleaser.yml b/.github/workflows/goreleaser.yml new file mode 100644 index 00000000..dccaa4f8 --- /dev/null +++ b/.github/workflows/goreleaser.yml @@ -0,0 +1,30 @@ +name: goreleaser + +on: + push: + tags: + - 'v*' # Add this line to trigger the workflow on tag pushes that match 'v*' + +permissions: + id-token: write + contents: read + +jobs: + goreleaser: # Add this new job for GoReleaser + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: 1.22 + - name: Run GoReleaser + uses: goreleaser/goreleaser-action@v6 + with: + version: v2.1.0 + args: release --clean + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/static.yml b/.github/workflows/static.yml new file mode 100644 index 00000000..1bc752c8 --- /dev/null +++ b/.github/workflows/static.yml @@ -0,0 +1,70 @@ +name: static check +on: pull_request + +jobs: + imports: + name: Imports + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@master + - name: check + uses: danhunsaker/golang-github-actions@v1.3.0 + with: + run: imports + token: ${{ secrets.GITHUB_TOKEN }} + + errcheck: + name: Errcheck + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@master + - name: check + uses: danhunsaker/golang-github-actions@v1.3.0 + with: + run: errcheck + token: ${{ secrets.GITHUB_TOKEN }} + + lint: + name: Lint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@master + - name: check + uses: danhunsaker/golang-github-actions@v1.3.0 + with: + run: lint + token: ${{ secrets.GITHUB_TOKEN }} + + shadow: + name: Shadow + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@master + - name: check + uses: danhunsaker/golang-github-actions@v1.3.0 + with: + run: shadow + token: ${{ secrets.GITHUB_TOKEN }} + + staticcheck: + name: StaticCheck + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@master + - name: check + uses: danhunsaker/golang-github-actions@v1.3.0 + with: + run: staticcheck + token: ${{ secrets.GITHUB_TOKEN }} + + sec: + name: Sec + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@master + - name: check + uses: danhunsaker/golang-github-actions@v1.3.0 + with: + run: sec + token: ${{ secrets.GITHUB_TOKEN }} + flags: "-exclude=G104" diff --git a/.gitignore b/.gitignore index 3fc6bb2b..eaa12583 100644 --- a/.gitignore +++ b/.gitignore @@ -69,3 +69,6 @@ docs/api-reference.md transcription.txt snippets.txt .env copy + +# Build result of goreleaser +dist/ diff --git a/.goreleaser.yml b/.goreleaser.yml new file mode 100644 index 00000000..230d00e2 --- /dev/null +++ b/.goreleaser.yml @@ -0,0 +1,38 @@ +# Make sure to check the documentation at http://goreleaser.com +version: 2 +builds: + - main: ./cmd/masa-node/main.go + ldflags: + - -w -s + - -X github.com/masa-finance/masa-oracle/internal.Version={{.Tag}} + - -X github.com/masa-finance/masa-oracle/internal.Commit={{.Commit}} + env: + - CGO_ENABLED=0 + goos: + - linux + - windows + - darwin + - freebsd + goarch: + - amd64 + - arm + - arm64 +source: + enabled: true + name_template: '{{ .ProjectName }}-{{ .Tag }}-source' +archives: + # Default template uses underscores instead of - + - name_template: >- + {{ .ProjectName }}-{{ .Tag }}- + {{- if eq .Os "freebsd" }}FreeBSD + {{- else }}{{- title .Os }}{{end}}- + {{- if eq .Arch "amd64" }}x86_64 + {{- else if eq .Arch "386" }}i386 + {{- else }}{{ .Arch }}{{end}} + {{- if .Arm }}v{{ .Arm }}{{ end }} +checksum: + name_template: '{{ .ProjectName }}-{{ .Tag }}-checksums.txt' +snapshot: + name_template: "{{ .Tag }}-next" +changelog: + use: github-native \ No newline at end of file diff --git a/Makefile b/Makefile index 29b9ba4c..117222e3 100644 --- a/Makefile +++ b/Makefile @@ -4,8 +4,8 @@ print-version: @echo "Version: ${VERSION}" build: - @go build -v -ldflags "-X github.com/masa-finance/masa-oracle/pkg/config.Version=${VERSION}" -o ./bin/masa-node ./cmd/masa-node - @go build -v -ldflags "-X github.com/masa-finance/masa-oracle/pkg/config.Version=${VERSION}" -o ./bin/masa-node-cli ./cmd/masa-node-cli + @go build -v -ldflags "-X github.com/masa-finance/masa-oracle/internal/versioning.ApplicationVersion=${VERSION}" -o ./bin/masa-node ./cmd/masa-node + @go build -v -ldflags "-X github.com/masa-finance/masa-oracle/internal/versioning.ApplicationVersion=${VERSION}" -o ./bin/masa-node-cli ./cmd/masa-node-cli install: @sh ./node_install.sh diff --git a/cmd/masa-node-cli/components.go b/cmd/masa-node-cli/components.go index dca4b4f3..e43a3766 100644 --- a/cmd/masa-node-cli/components.go +++ b/cmd/masa-node-cli/components.go @@ -5,7 +5,8 @@ import ( "strings" "github.com/gdamore/tcell/v2" - "github.com/masa-finance/masa-oracle/pkg/config" + "github.com/masa-finance/masa-oracle/internal/versioning" + "github.com/rivo/tview" ) @@ -125,7 +126,7 @@ const ( navigation = `[yellow]use keys or mouse to navigate` ) -var version string = fmt.Sprintf("[green]%s", config.Version) +var version string = fmt.Sprintf(`[green]Application version: %s\n[green]Protocol Version: %s`, versioning.ApplicationVersion, versioning.ProtocolVersion) // Splash shows the app info func Splash() (content tview.Primitive) { diff --git a/cmd/masa-node/main.go b/cmd/masa-node/main.go index ea416574..ff934f8f 100644 --- a/cmd/masa-node/main.go +++ b/cmd/masa-node/main.go @@ -6,6 +6,7 @@ import ( "os/signal" "syscall" + "github.com/masa-finance/masa-oracle/internal/versioning" "github.com/masa-finance/masa-oracle/pkg/workers" "github.com/sirupsen/logrus" @@ -21,7 +22,7 @@ import ( func main() { if len(os.Args) > 1 && os.Args[1] == "--version" { - logrus.Infof("Masa Oracle Node Version: %s\n", config.Version) + logrus.Infof("Masa Oracle Node Version: %s\nMasa Oracle Protocol verison: %s", versioning.ApplicationVersion, versioning.ProtocolVersion) os.Exit(0) } @@ -97,7 +98,7 @@ func main() { // and other peers can do work we only need to check this here // if this peer can or cannot scrape or write that is checked in other places if node.IsStaked { - go workers.MonitorWorkers(ctx, node) + node.Host.SetStreamHandler(config.ProtocolWithVersion(config.WorkerProtocol), workers.GetWorkHandlerManager().HandleWorkerStream) go masa.SubscribeToBlocks(ctx, node) go node.NodeTracker.ClearExpiredWorkerTimeouts() } @@ -135,7 +136,7 @@ func main() { multiAddr := node.GetMultiAddrs().String() // Get the multiaddress ipAddr := node.Host.Addrs()[0].String() // Get the IP address // Display the welcome message with the multiaddress and IP address - config.DisplayWelcomeMessage(multiAddr, ipAddr, keyManager.EthAddress, isStaked, isValidator, cfg.TwitterScraper, cfg.TelegramScraper, cfg.DiscordScraper, cfg.WebScraper, config.Version) + config.DisplayWelcomeMessage(multiAddr, ipAddr, keyManager.EthAddress, isStaked, isValidator, cfg.TwitterScraper, cfg.TelegramScraper, cfg.DiscordScraper, cfg.WebScraper, versioning.ApplicationVersion, versioning.ProtocolVersion) <-ctx.Done() } diff --git a/docs/docs.go b/docs/docs.go index 0fa4b234..94d35e82 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -4,6 +4,50 @@ package docs import "github.com/swaggo/swag" const docTemplate = `{ + "swagger": "2.0", + "info": { + "description": "{{escape .Description}}", + "title": "{{.Title}}", + "contact": { + "name": "Masa API Support", + "url": "https://masa.ai", + "email": "support@masa.ai" + }, + "license": { + "name": "MIT", + "url": "https://opensource.org/license/mit" + }, + "version": "{{.Version}}" + }, + "host": "{{.Host}}", + "basePath": "{{.BasePath}}", + "securityDefinitions": { + "Bearer": { + "type": "apiKey", + "name": "Authorization", + "in": "header" + } + }, + "security": [ + { + "Bearer": [] + } + ], + "paths": { + "/data/twitter/profile/{username}": { + "get": { + "description": "Retrieves tweets from a specific Twitter profile", + "consumes": ["application/json"], + "produces": ["application/json"], + "tags": ["Twitter"], + "summary": "Search Twitter Profile", + "parameters": [ + { + "type": "string", + "description": "Twitter Username", + "name": "username", + "in": "path", + "required": true "schemes": {{ marshal .Schemes }}, "swagger": "2.0", "info": { @@ -1105,6 +1149,10 @@ const docTemplate = `{ }, "phone_code_hash": { "type": "string" + }, + "password": { + "type": "string", + "description": "Optional password for two-factor authentication" } }, "required": ["phone_number", "code", "phone_code_hash"] @@ -1207,249 +1255,535 @@ const docTemplate = `{ "type": "string" } } - }, - "SuccessResponse": { - "type": "object", - "properties": { - "message": { - "type": "string" - } - } - }, - "WebDataRequest": { - "type": "object", - "properties": { - "query": { - "type": "string" - }, - "url": { - "type": "string" - }, - "depth": { - "type": "integer" - } - } - }, - "WebDataResponse": { - "type": "object", - "properties": { - "data": { - "type": "string" + ], + "responses": { + "200": { + "description": "List of tweets from the profile", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/Tweet" } + } + }, + "400": { + "description": "Invalid username or error fetching tweets", + "schema": { + "$ref": "#/definitions/ErrorResponse" + } } - }, - "SentimentAnalysisResponse": { - "type": "object", - "properties": { - "sentiment": { - "type": "string" - }, - "data": { - "type": "string" - } + }, + "security": [ + { + "Bearer": [] } - }, - "definitions": { - "ChatResponse": { - "type": "object", - "properties": { - "message": { - "type": "string" - } - } - }, - "DHTResponse": { - "type": "object", - "properties": { - "key": { - "type": "string" - }, - "value": { - "type": "string" - } - } + ] + } + }, + "/data/twitter/followers/{username}": { + "get": { + "description": "Retrieves followers from a specific Twitter profile.", + "consumes": ["application/json"], + "produces": ["application/json"], + "tags": ["Twitter"], + "summary": "Search Twitter Followers", + "parameters": [ + { + "name": "username", + "in": "path", + "description": "Twitter Username", + "required": true, + "type": "string" }, - "UserProfile": { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "username": { - "type": "string" - }, - "discriminator": { - "type": "string" - }, - "avatar": { - "type": "string" - } - } - }, - "ErrorResponse": { - "type": "object", - "properties": { - "error": { - "type": "string" - } - } - }, - "Tweet": { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "text": { - "type": "string" - }, - "created_at": { - "type": "string" - }, - "user": { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "name": { - "type": "string" - }, - "screen_name": { - "type": "string" - } - } - } + { + "name": "limit", + "in": "query", + "description": "The maximum number of followers to return", + "required": false, + "type": "integer", + "format": "int32", + "default": 20 + } + ], + "responses": { + "200": { + "description": "Array of profiles a user has as followers", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/Profile" } + } }, - "ChannelMessage": { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "channelID": { - "type": "string" - }, - "author": { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "username": { - "type": "string" - }, - "discriminator": { - "type": "string" - }, - "avatar": { - "type": "string" - } - } - }, - "content": { - "type": "string" - }, - "timestamp": { - "type": "string" - } - } - }, - "GuildChannel": { + "400": { + "description": "Invalid username or error fetching followers", + "schema": { + "$ref": "#/definitions/ErrorResponse" + } + } + } + } + }, + "/data/twitter/tweets/recent": { + "post": { + "description": "Retrieves recent tweets based on query parameters", + "consumes": ["application/json"], + "produces": ["application/json"], + "tags": ["Twitter"], + "summary": "Search recent tweets", + "parameters": [ + { + "in": "body", + "name": "body", + "description": "Search parameters", + "required": true, + "schema": { "type": "object", "properties": { - "id": { - "type": "string" - }, - "guildID": { - "type": "string" - }, - "name": { - "type": "string" + "query": { + "type": "string", + "description": "Search Query" }, - "type": { - "type": "integer" + "count": { + "type": "integer", + "description": "Number of tweets to return" } } - }, - "Guild": { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "name": { - "type": "string" - }, - "icon": { - "type": "string" - }, - "owner": { - "type": "boolean" - }, - "permissions": { - "type": "string" - } + } + } + ], + "responses": { + "200": { + "description": "List of recent tweets", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/Tweet" } - }, - "Trend": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "url": { - "type": "string" - }, - "tweet_volume": { - "type": "integer" - } + } + }, + "400": { + "description": "Invalid query or error fetching tweets", + "schema": { + "$ref": "#/definitions/ErrorResponse" + } + } + }, + "security": [ + { + "Bearer": [] + } + ] + } + }, + "/data/twitter/tweets/trends": { + "get": { + "description": "Retrieves the latest Twitter trending topics", + "consumes": ["application/json"], + "produces": ["application/json"], + "tags": ["Twitter"], + "summary": "Twitter Trends", + "responses": { + "200": { + "description": "List of trending topics", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/Trend" } + } }, - "SentimentAnalysisResponse": { - "type": "object", - "properties": { - "sentiment": { - "type": "string" - }, - "data": { - "type": "string" - } + "400": { + "description": "Error fetching Twitter trends", + "schema": { + "$ref": "#/definitions/ErrorResponse" + } + } + }, + "security": [ + { + "Bearer": [] + } + ] + } + }, + "/data/discord/profile/{userID}": { + "get": { + "description": "Retrieves a Discord user profile by user ID.", + "consumes": ["application/json"], + "produces": ["application/json"], + "tags": ["Discord"], + "summary": "Search Discord Profile", + "parameters": [ + { + "name": "userID", + "in": "path", + "description": "Discord User ID", + "required": true, + "type": "string" + } + ], + "responses": { + "200": { + "description": "Successfully retrieved Discord user profile", + "schema": { + "$ref": "#/definitions/UserProfile" + } + }, + "400": { + "description": "Invalid user ID or error fetching profile", + "schema": { + "$ref": "#/definitions/ErrorResponse" + } + } + }, + "security": [ + { + "Bearer": [] + } + ] + } + }, + "/data/discord/channels/{channelID}/messages": { + "get": { + "description": "Retrieves messages from a specified Discord channel.", + "consumes": ["application/json"], + "produces": ["application/json"], + "tags": ["Discord"], + "summary": "Get messages from a Discord channel", + "parameters": [ + { + "name": "channelID", + "in": "path", + "description": "Discord Channel ID", + "required": true, + "type": "string" + }, + { + "name": "limit", + "in": "query", + "description": "The maximum number of messages to return", + "required": false, + "type": "integer", + "format": "int32" + }, + { + "name": "before", + "in": "query", + "description": "A message ID to return messages posted before this message", + "required": false, + "type": "string" + } + ], + "responses": { + "200": { + "description": "Successfully retrieved messages from the Discord channel", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/ChannelMessage" } + } }, - "WebDataResponse": { - "type": "object", - "properties": { - "data": { - "type": "string" - } + "400": { + "description": "Invalid channel ID or error fetching messages", + "schema": { + "$ref": "#/definitions/ErrorResponse" + } + } + }, + "security": [ + { + "Bearer": [] + } + ] + } + }, + "/data/discord/guilds/{guildID}/channels": { + "get": { + "description": "Retrieves channels from a specified Discord guild.", + "tags": ["Discord"], + "summary": "Get channels from a Discord guild", + "parameters": [ + { + "name": "guildID", + "in": "path", + "description": "Discord Guild ID", + "required": true, + "type": "string" + } + ], + "responses": { + "200": { + "description": "Successfully retrieved channels from the Discord guild", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/GuildChannel" } + } }, - "LLMModelsResponse": { - "type": "object", - "properties": { - "models": { - "type": "array", - "items": { - "type": "string" - } - } + "400": { + "description": "Invalid guild ID or error fetching channels", + "schema": { + "$ref": "#/definitions/ErrorResponse" + } + } + }, + "security": [ + { + "Bearer": [] + } + ] + } + }, + "/data/discord/user/guilds": { + "get": { + "description": "Retrieves guilds from a specified Discord user.", + "tags": ["Discord"], + "summary": "Get guilds from a Discord user", + "responses": { + "200": { + "description": "Successfully retrieved guilds from the Discord user", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/UserGuild" } + } }, - "NodeDataResponse": { - "type": "object", - "properties": { - "peer_id": { - "type": "string" - }, - "data": { - "type": "string" - } + "400": { + "description": "Invalid user ID or error fetching guilds", + "schema": { + "$ref": "#/definitions/ErrorResponse" + } + } + }, + "security": [ + { + "Bearer": [] + } + ] + } + }, + "/data/discord/guilds/all": { + "get": { + "description": "Retrieves all guilds that all the Discord workers are apart of.", + "consumes": ["application/json"], + "produces": ["application/json"], + "tags": ["Discord"], + "summary": "Get all guilds", + "responses": { + "200": { + "description": "Successfully retrieved all guilds for the Discord user", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/Guild" } + } + }, + "400": { + "description": "Error fetching guilds or invalid access token", + "schema": { + "$ref": "#/definitions/ErrorResponse" + } } + }, + "security": [ + { + "Bearer": [] + } + ] + } + } + }, + "definitions": { + "UserProfile": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "username": { + "type": "string" + }, + "discriminator": { + "type": "string" + }, + "avatar": { + "type": "string" + } + } + }, + "SuccessResponse": { + "type": "object", + "properties": { + "message": { + "type": "string" + } + } + }, + "ErrorResponse": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + } + }, + "Tweet": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "text": { + "type": "string" + }, + "created_at": { + "type": "string" + }, + "user": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "screen_name": { + "type": "string" + } + } + } + } + }, + "ChannelMessage": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "channelID": { + "type": "string" + }, + "author": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "username": { + "type": "string" + }, + "discriminator": { + "type": "string" + }, + "avatar": { + "type": "string" + } + } + }, + "content": { + "type": "string" + }, + "timestamp": { + "type": "string" + } + } + }, + "GuildChannel": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "guildID": { + "type": "string" + }, + "name": { + "type": "string" + }, + "type": { + "type": "integer" + } + } + }, + "Guild": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "icon": { + "type": "string" + }, + "owner": { + "type": "boolean" + }, + "permissions": { + "type": "string" + } + } + }, + "Trend": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "url": { + "type": "string" + }, + "tweet_volume": { + "type": "integer" + } + } + }, + "Profile": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "username": { + "type": "string" + }, + "discriminator": { + "type": "string" + }, + "avatar": { + "type": "string" + } + } + }, + "UserGuild": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "icon": { + "type": "string" + }, + "owner": { + "type": "boolean" + }, + "permissions": { + "type": "string" + } } -}` + } + } + }` // SwaggerInfo holds exported Swagger Info so clients can modify it var SwaggerInfo = &swag.Spec{ diff --git a/docs/oracle-node/telegram-sentiment.md b/docs/oracle-node/telegram-sentiment.md index 8764ea15..5c399367 100644 --- a/docs/oracle-node/telegram-sentiment.md +++ b/docs/oracle-node/telegram-sentiment.md @@ -5,15 +5,15 @@ title: Telegram Sentiment ## Masa Node Telegram Sentiment Analysis Feature -The Masa Node introduces a powerful feature for analyzing the sentiment of telegram messages. This functionality leverages advanced language models to interpret the sentiment behind a collection of tweets, providing valuable insights into public perception and trends. +The Masa Node introduces a powerful feature for analyzing the sentiment of telegram messages. This functionality leverages advanced language models to interpret the sentiment behind a collection of Telegram messages, providing valuable insights into public perception and trends. ## Overview -The Telegram sentiment analysis feature is part of the broader capabilities of the Masa Node, designed to interact with Telegram messages data in a meaningful way. It uses state-of-the-art language models to evaluate the sentiment of tweets, categorizing them into positive, negative, or neutral sentiments. +The Telegram sentiment analysis feature is part of the broader capabilities of the Masa Node, designed to interact with Telegram messages data in a meaningful way. It uses state-of-the-art language models to evaluate the sentiment of Telegram messages, categorizing them into positive, negative, or neutral sentiments. ## How It Works -The sentiment analysis process involves fetching tweets based on specific queries, and then analyzing these tweets using selected language models. The system supports various models, including Claude and GPT variants, allowing for flexible and powerful sentiment analysis. +The sentiment analysis process involves fetching Telegram messages based on specific queries, and then analyzing these messages using selected language models. The system supports various models, including Claude and GPT variants, allowing for flexible and powerful sentiment analysis. ### Models @@ -64,7 +64,7 @@ const ( #### Masa cli or code integration -Tweets are fetched using the Twitter Scraper library, as seen in the [llmbridge](file:///Users/john/Projects/masa/masa-oracle/pkg/llmbridge/sentiment.go#) package. This process does not require Telegram API keys, making it accessible and straightforward. +Messages are fetched using the Telegram Scraper library, as seen in the [llmbridge](/masa-oracle/pkg/llmbridge/sentiment.go#) package. This process does not require Telegram API keys, making it accessible and straightforward. ```go func AnalyzeSentimentTelegram(messages []*tg.Message, model string, prompt string) (string, string, error) { @@ -72,4 +72,4 @@ func AnalyzeSentimentTelegram(messages []*tg.Message, model string, prompt strin ### Analyzing Sentiment -Once tweets are fetched, they are sent to the chosen language model for sentiment analysis. The system currently supports models prefixed with "claude-" and "gpt-", catering to a range of analysis needs. \ No newline at end of file +Once Telegram Messages are fetched, they are sent to the chosen language model for sentiment analysis. The system currently supports models prefixed with "claude-" and "gpt-", catering to a range of analysis needs. \ No newline at end of file diff --git a/docs/oracle-node/twitter-data.md b/docs/oracle-node/twitter-data.md index 4eba986d..a4c9a281 100644 --- a/docs/oracle-node/twitter-data.md +++ b/docs/oracle-node/twitter-data.md @@ -211,6 +211,18 @@ Example response: The Advanced Search feature allows users to perform more complex queries to filter tweets according to various criteria such as date ranges, specific users, hashtags, and more. Below you will find detailed information on how to construct advanced search queries. +### Exact Search +Search for tweets containing specific hashtags. + +**Syntax:** `"\"searchterm\"` + +**Example:** + +```bash +curl -X POST http://localhost:8080/api/v1/data/twitter/tweets/recent \ +-H "Content-Type: application/json" \ +-d '{"query": "\"masa\", "count": 10}' +``` ### Hashtag Search Search for tweets containing specific hashtags. diff --git a/docs/worker-node/telegram-worker.md b/docs/worker-node/telegram-worker.md index c9badcc4..323ac686 100644 --- a/docs/worker-node/telegram-worker.md +++ b/docs/worker-node/telegram-worker.md @@ -58,6 +58,8 @@ TELEGRAM_SCRAPER=true 1. Call the `/api/v1/auth/telegram/complete` endpoint with the code you received on your phone, your phone number, and the `phone_code_hash` from the previous step. +*Note* - If you have 2FA turned on, you will need to pass your 2FA password in the API call. + ### Verifying Node Configuration Ensure your node is correctly configured to handle Twitter data requests by checkint the initialization message: diff --git a/go.mod b/go.mod index adceeb32..bffce256 100644 --- a/go.mod +++ b/go.mod @@ -1,16 +1,12 @@ module github.com/masa-finance/masa-oracle go 1.22.0 - -toolchain go1.22.2 - require ( - github.com/asynkron/protoactor-go v0.0.0-20240413045429-76c172a71a16 github.com/cenkalti/backoff/v4 v4.3.0 github.com/chyeh/pubip v0.0.0-20170203095919-b7e679cf541c github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 github.com/dgraph-io/badger v1.6.2 - github.com/ethereum/go-ethereum v1.14.7 + github.com/ethereum/go-ethereum v1.14.8 github.com/fatih/color v1.17.0 github.com/gdamore/tcell/v2 v2.7.4 github.com/gin-contrib/cors v1.7.2 @@ -32,7 +28,7 @@ require ( github.com/masa-finance/masa-twitter-scraper v0.0.0-20240515201201-b83fa3597a31 github.com/multiformats/go-multiaddr v0.13.0 github.com/multiformats/go-multihash v0.2.3 - github.com/ollama/ollama v0.3.1 + github.com/ollama/ollama v0.3.6 github.com/rivo/tview v0.0.0-20240505185119-ed116790de0f github.com/sashabaranov/go-openai v1.27.0 github.com/sirupsen/logrus v1.9.3 @@ -42,7 +38,6 @@ require ( github.com/swaggo/files v1.0.1 github.com/swaggo/gin-swagger v1.6.0 github.com/swaggo/swag v1.16.3 - google.golang.org/protobuf v1.34.2 ) require ( @@ -50,17 +45,15 @@ require ( github.com/KyleBanks/depth v1.2.1 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/PuerkitoBio/goquery v1.9.1 // indirect - github.com/Workiva/go-datastructures v1.1.3 // indirect github.com/andybalholm/cascadia v1.3.2 // indirect github.com/antchfx/htmlquery v1.2.3 // indirect github.com/antchfx/xmlquery v1.3.1 // indirect github.com/antchfx/xpath v1.1.10 // indirect - github.com/asynkron/gofun v0.0.0-20220329210725-34fed760f4c2 // indirect github.com/benbjohnson/clock v1.3.5 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bits-and-blooms/bitset v1.13.0 // indirect github.com/blang/semver/v4 v4.0.0 // indirect - github.com/btcsuite/btcd/btcec/v2 v2.3.2 // indirect + github.com/btcsuite/btcd/btcec/v2 v2.3.4 // indirect github.com/btcsuite/btcd/chaincfg/chainhash v1.1.0 // indirect github.com/bytedance/sonic v1.11.6 // indirect github.com/bytedance/sonic/loader v0.1.1 // indirect @@ -82,7 +75,6 @@ require ( github.com/docker/go-units v0.5.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/elastic/gosigar v0.14.2 // indirect - github.com/emirpasic/gods v1.18.1 // indirect github.com/ethereum/c-kzg-4844 v1.0.0 // indirect github.com/flynn/noise v1.1.0 // indirect github.com/francoispqt/gojay v1.2.13 // indirect @@ -122,7 +114,7 @@ require ( github.com/hashicorp/golang-lru v1.0.2 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/hashicorp/hcl v1.0.0 // indirect - github.com/holiman/uint256 v1.3.0 // indirect + github.com/holiman/uint256 v1.3.1 // indirect github.com/huin/goupnp v1.3.0 // indirect github.com/ipfs/boxo v0.18.0 // indirect github.com/ipfs/go-log v1.0.5 // indirect @@ -152,8 +144,6 @@ require ( github.com/libp2p/go-reuseport v0.4.0 // indirect github.com/libp2p/go-yamux/v4 v4.0.1 // indirect github.com/libp2p/zeroconf/v2 v2.2.0 // indirect - github.com/lithammer/shortuuid/v4 v4.0.0 // indirect - github.com/lmittmann/tint v1.0.3 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mailru/easyjson v0.7.7 // indirect @@ -182,7 +172,6 @@ require ( github.com/onsi/ginkgo/v2 v2.16.0 // indirect github.com/opencontainers/runtime-spec v1.2.0 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect - github.com/orcaman/concurrent-map v1.0.0 // indirect github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/pion/datachannel v1.5.6 // indirect @@ -231,7 +220,6 @@ require ( github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.6.1 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect - github.com/twmb/murmur3 v1.1.8 // indirect github.com/ugorji/go/codec v1.2.12 // indirect github.com/urfave/cli/v2 v2.27.1 // indirect github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirect @@ -239,10 +227,7 @@ require ( github.com/yusufpapurcu/wmi v1.2.4 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/otel v1.28.0 // indirect - go.opentelemetry.io/otel/exporters/prometheus v0.44.0 // indirect go.opentelemetry.io/otel/metric v1.28.0 // indirect - go.opentelemetry.io/otel/sdk v1.28.0 // indirect - go.opentelemetry.io/otel/sdk/metric v1.28.0 // indirect go.opentelemetry.io/otel/trace v1.28.0 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/dig v1.17.1 // indirect @@ -262,8 +247,7 @@ require ( golang.org/x/tools v0.23.0 // indirect gonum.org/v1/gonum v0.15.0 // indirect google.golang.org/appengine v1.6.8 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240513163218-0867130af1f8 // indirect - google.golang.org/grpc v1.63.2 // indirect + google.golang.org/protobuf v1.34.2 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect lukechampine.com/blake3 v1.2.1 // indirect diff --git a/go.sum b/go.sum index 5dfa19d7..da083196 100644 --- a/go.sum +++ b/go.sum @@ -23,8 +23,6 @@ github.com/PuerkitoBio/goquery v1.9.1 h1:mTL6XjbJTZdpfL+Gwl5U2h1l9yEkJjhmlTeV9VP github.com/PuerkitoBio/goquery v1.9.1/go.mod h1:cW1n6TmIMDoORQU5IU/P1T3tGFunOeXEpGP2WHRwkbY= github.com/VictoriaMetrics/fastcache v1.12.2 h1:N0y9ASrJ0F6h0QaC3o6uJb3NIZ9VKLjCM7NQbSmF7WI= github.com/VictoriaMetrics/fastcache v1.12.2/go.mod h1:AmC+Nzz1+3G2eCPapF6UcsnkThDcMsQicp4xDukwJYI= -github.com/Workiva/go-datastructures v1.1.3 h1:LRdRrug9tEuKk7TGfz/sct5gjVj44G9pfqDt4qm7ghw= -github.com/Workiva/go-datastructures v1.1.3/go.mod h1:1yZL+zfsztete+ePzZz/Zb1/t5BnDuE2Ya2MMGhzP6A= github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= github.com/andybalholm/cascadia v1.2.0/go.mod h1:YCyR8vOZT9aZ1CHEd8ap0gMVm2aFgxBp0T0eFw1RUQY= github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss= @@ -40,10 +38,6 @@ github.com/antchfx/xpath v1.1.8/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNY github.com/antchfx/xpath v1.1.10 h1:cJ0pOvEdN/WvYXxvRrzQH9x5QWKpzHacYO8qzCcDYAg= github.com/antchfx/xpath v1.1.10/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= -github.com/asynkron/gofun v0.0.0-20220329210725-34fed760f4c2 h1:jEsFZ9d/ieJGVrx3fSPi8oe/qv21fRmyUL5cS3ZEn5A= -github.com/asynkron/gofun v0.0.0-20220329210725-34fed760f4c2/go.mod h1:5GMOSqaYxNWwuVRWyampTPJEntwz7Mj9J8v1a7gSU2E= -github.com/asynkron/protoactor-go v0.0.0-20240413045429-76c172a71a16 h1:WcgLv2PuooiG5+WmeJAaWevD5RZH3HMVxyTZX0xofJM= -github.com/asynkron/protoactor-go v0.0.0-20240413045429-76c172a71a16/go.mod h1:HTx47MGokOrouz8nrUmjyLLOVu+/kRNN6KKVG0XjQ3E= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz5o= @@ -56,8 +50,8 @@ github.com/bits-and-blooms/bitset v1.13.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6 github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g= -github.com/btcsuite/btcd/btcec/v2 v2.3.2 h1:5n0X6hX0Zk+6omWcihdYvdAlGf2DfasC0GMf7DClJ3U= -github.com/btcsuite/btcd/btcec/v2 v2.3.2/go.mod h1:zYzJ8etWJQIv1Ogk7OzpWjowwOdXY1W/17j2MW85J04= +github.com/btcsuite/btcd/btcec/v2 v2.3.4 h1:3EJjcN70HCu/mwqlUsGK8GcNVyLVxFDlWurTXGPFfiQ= +github.com/btcsuite/btcd/btcec/v2 v2.3.4/go.mod h1:zYzJ8etWJQIv1Ogk7OzpWjowwOdXY1W/17j2MW85J04= github.com/btcsuite/btcd/chaincfg/chainhash v1.1.0 h1:59Kx4K6lzOW5w6nFlA0v5+lk/6sjybR934QNHSJZPTQ= github.com/btcsuite/btcd/chaincfg/chainhash v1.1.0/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc= github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= @@ -149,22 +143,18 @@ github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+m github.com/elastic/gosigar v0.12.0/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs= github.com/elastic/gosigar v0.14.2 h1:Dg80n8cr90OZ7x+bAax/QjoW/XqTI11RmA79ZwIm9/4= github.com/elastic/gosigar v0.14.2/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs= -github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= -github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/ethereum/c-kzg-4844 v1.0.0 h1:0X1LBXxaEtYD9xsyj9B9ctQEZIpnvVDeoBx8aHEwTNA= github.com/ethereum/c-kzg-4844 v1.0.0/go.mod h1:VewdlzQmpT5QSrVhbBuGoCdFJkpaJlO1aQputP83wc0= -github.com/ethereum/go-ethereum v1.14.7 h1:EHpv3dE8evQmpVEQ/Ne2ahB06n2mQptdwqaMNhAT29g= -github.com/ethereum/go-ethereum v1.14.7/go.mod h1:Mq0biU2jbdmKSZoqOj29017ygFrMnB5/Rifwp980W4o= +github.com/ethereum/go-ethereum v1.14.8 h1:NgOWvXS+lauK+zFukEvi85UmmsS/OkV0N23UZ1VTIig= +github.com/ethereum/go-ethereum v1.14.8/go.mod h1:TJhyuDq0JDppAkFXgqjwpdlQApywnu/m10kFPxh8vvs= github.com/ethereum/go-verkle v0.1.1-0.20240306133620-7d920df305f0 h1:KrE8I4reeVvf7C1tm8elRjj4BdscTYzz/WAbYyf/JI4= github.com/ethereum/go-verkle v0.1.1-0.20240306133620-7d920df305f0/go.mod h1:D9AJLVXSyZQXJQVk8oh1EwjISE+sJTn2duYIZC0dy3w= github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4= github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI= -github.com/fjl/memsize v0.0.2 h1:27txuSD9or+NZlnOWdKUxeBzTAUkWCVh+4Gf2dWFOzA= -github.com/fjl/memsize v0.0.2/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/flynn/noise v1.1.0 h1:KjPQoQCEFdZDiP03phOvGi11+SVVhBG2wOWAorLsstg= github.com/flynn/noise v1.1.0/go.mod h1:xbMo+0i6+IGbYdJhF31t2eR1BIU0CYc12+BNAKwUTag= @@ -303,7 +293,6 @@ github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm4 github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -341,8 +330,8 @@ github.com/holiman/billy v0.0.0-20240216141850-2abb0c79d3c4 h1:X4egAf/gcS1zATw6w github.com/holiman/billy v0.0.0-20240216141850-2abb0c79d3c4/go.mod h1:5GuXa7vkL8u9FkFuWdVvfR5ix8hRB7DbOAaYULamFpc= github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZKVu0fao= github.com/holiman/bloomfilter/v2 v2.0.3/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iURXE7ZOP9L9hSkA= -github.com/holiman/uint256 v1.3.0 h1:4wdcm/tnd0xXdu7iS3ruNvxkWwrb4aeBQv19ayYn8F4= -github.com/holiman/uint256 v1.3.0/go.mod h1:EOMSn4q6Nyt9P6efbI3bueV4e1b3dGlUCXeiRV4ng7E= +github.com/holiman/uint256 v1.3.1 h1:JfTzmih28bittyHM8z360dCjIA9dbPIBlcTI6lmctQs= +github.com/holiman/uint256 v1.3.1/go.mod h1:EOMSn4q6Nyt9P6efbI3bueV4e1b3dGlUCXeiRV4ng7E= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huin/goupnp v1.3.0 h1:UvLUlWDNpoUdYzb2TCn+MuTWtcjXKSza2n6CBdQ0xXc= github.com/huin/goupnp v1.3.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= @@ -456,10 +445,6 @@ github.com/libp2p/go-yamux/v4 v4.0.1 h1:FfDR4S1wj6Bw2Pqbc8Uz7pCxeRBPbwsBbEdfwiCy github.com/libp2p/go-yamux/v4 v4.0.1/go.mod h1:NWjl8ZTLOGlozrXSOZ/HlfG++39iKNnM5wwmtQP1YB4= github.com/libp2p/zeroconf/v2 v2.2.0 h1:Cup06Jv6u81HLhIj1KasuNM/RHHrJ8T7wOTS4+Tv53Q= github.com/libp2p/zeroconf/v2 v2.2.0/go.mod h1:fuJqLnUwZTshS3U/bMRJ3+ow/v9oid1n0DmyYyNO1Xs= -github.com/lithammer/shortuuid/v4 v4.0.0 h1:QRbbVkfgNippHOS8PXDkti4NaWeyYfcBTHtw7k08o4c= -github.com/lithammer/shortuuid/v4 v4.0.0/go.mod h1:Zs8puNcrvf2rV9rTH51ZLLcj7ZXqQI3lv67aw4KiB1Y= -github.com/lmittmann/tint v1.0.3 h1:W5PHeA2D8bBJVvabNfQD/XW9HPLZK1XoPZH0cq8NouQ= -github.com/lmittmann/tint v1.0.3/go.mod h1:HIS3gSy7qNwGCj+5oRjAutErFBl4BzdQP6cJZ0NfMwE= github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= @@ -547,8 +532,8 @@ github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY= github.com/nxadm/tail v1.4.11/go.mod h1:OTaG3NK980DZzxbRq6lEuzgU+mug70nY11sMd4JXXHc= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= -github.com/ollama/ollama v0.3.1 h1:FvbhD9TxSB1F2xvQPFaGvYKLVxK9QJqfU+EUb3ftwkE= -github.com/ollama/ollama v0.3.1/go.mod h1:USAVO5xFaXAoVWJ0rkPYgCVhTxE/oJ81o7YGcJxvyp8= +github.com/ollama/ollama v0.3.6 h1:nA/N0AmjP327po5cZDGLqI40nl+aeei0pD0dLa92ypE= +github.com/ollama/ollama v0.3.6/go.mod h1:YrWoNkFnPOYsnDvsf/Ztb1wxU9/IXrNsQHqcxbY2r94= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= @@ -568,14 +553,11 @@ github.com/opencontainers/runtime-spec v1.2.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/ github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8= -github.com/orcaman/concurrent-map v1.0.0 h1:I/2A2XPCb4IuQWcQhBhSwGfiuybl/J0ev9HDbW65HOY= -github.com/orcaman/concurrent-map v1.0.0/go.mod h1:Lu3tH6HLW3feq74c2GC+jIMS/K2CFcDWnWD9XkenwhI= github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0= github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= -github.com/philhofer/fwd v1.1.1/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= github.com/pion/datachannel v1.5.6 h1:1IxKJntfSlYkpUj8LlYRSWpYiTTC02nUrOE8T3DqGeg= github.com/pion/datachannel v1.5.6/go.mod h1:1eKT6Q85pRnr2mHiWHxJwO50SfZRtWHTsNIVb/NfGW4= github.com/pion/dtls/v2 v2.2.7/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s= @@ -766,16 +748,12 @@ github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45 github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= github.com/temoto/robotstxt v1.1.1 h1:Gh8RCs8ouX3hRSxxK7B1mO5RFByQ4CmJZDwgom++JaA= github.com/temoto/robotstxt v1.1.1/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo= -github.com/tinylib/msgp v1.1.5/go.mod h1:eQsjooMTnV42mHu917E26IogZ2930nFyBQdofk10Udg= github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= -github.com/ttacon/chalk v0.0.0-20160626202418-22c06c80ed31/go.mod h1:onvgF043R+lC5RZ8IT9rBXDaEDnpnw/Cl+HFiw+v/7Q= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= -github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg= -github.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/tyler-smith/go-bip39 v1.1.0 h1:5eUemwrMargf3BSLRRCalXT93Ns6pQJIjYQN2nyfOP8= github.com/tyler-smith/go-bip39 v1.1.0/go.mod h1:gUYDtqQw1JS3ZJ8UWVcGTGqqr6YIN3CWg+kkNaLt55U= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= @@ -805,14 +783,8 @@ go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/otel v1.28.0 h1:/SqNcYk+idO0CxKEUOtKQClMK/MimZihKYMruSMViUo= go.opentelemetry.io/otel v1.28.0/go.mod h1:q68ijF8Fc8CnMHKyzqL6akLO46ePnjkgfIMIjUIX9z4= -go.opentelemetry.io/otel/exporters/prometheus v0.44.0 h1:08qeJgaPC0YEBu2PQMbqU3rogTlyzpjhCI2b58Yn00w= -go.opentelemetry.io/otel/exporters/prometheus v0.44.0/go.mod h1:ERL2uIeBtg4TxZdojHUwzZfIFlUIjZtxubT5p4h1Gjg= go.opentelemetry.io/otel/metric v1.28.0 h1:f0HGvSl1KRAU1DLgLGFjrwVyismPlnuU6JD6bOeuA5Q= go.opentelemetry.io/otel/metric v1.28.0/go.mod h1:Fb1eVBFZmLVTMb6PPohq3TO9IIhUisDsbJoL/+uQW4s= -go.opentelemetry.io/otel/sdk v1.28.0 h1:b9d7hIry8yZsgtbmM0DKyPWMMUMlK9NEKuIG4aBqWyE= -go.opentelemetry.io/otel/sdk v1.28.0/go.mod h1:oYj7ClPUA7Iw3m+r7GeEjz0qckQRJK2B8zjcZEfu7Pg= -go.opentelemetry.io/otel/sdk/metric v1.28.0 h1:OkuaKgKrgAbYrrY0t92c+cC+2F6hsFNnCQArXCKlg08= -go.opentelemetry.io/otel/sdk/metric v1.28.0/go.mod h1:cWPjykihLAPvXKi4iZc1dpER3Jdq2Z0YLse3moQUCpg= go.opentelemetry.io/otel/trace v1.28.0 h1:GhQ9cUuQGmNDd5BTCP2dAvv75RdMxEfTmYejp+lkx9g= go.opentelemetry.io/otel/trace v1.28.0/go.mod h1:jPyXzNPg6da9+38HEwElrQiHlVMTnVfM3/yv2OlIHaI= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= @@ -1031,7 +1003,6 @@ golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20201022035929-9cf592e881e9/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= @@ -1061,8 +1032,6 @@ google.golang.org/genproto v0.0.0-20181202183823-bd91e49a0898/go.mod h1:7Ep/1NZk google.golang.org/genproto v0.0.0-20190306203927-b5d61aea6440/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240513163218-0867130af1f8 h1:mxSlqyb8ZAHsYDCfiXN1EDdNTdvjUJSLY+OnAUtYNYA= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240513163218-0867130af1f8/go.mod h1:I7Y+G38R2bu5j1aLzfFmQfTcU/WnFuqDwLZAbvKTKpM= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= @@ -1071,8 +1040,6 @@ google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyac google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.63.2 h1:MUeiw1B2maTVZthpU5xvASfTh3LDbxHd6IJ6QQVU+xM= -google.golang.org/grpc v1.63.2/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= diff --git a/internal/versioning/version.go b/internal/versioning/version.go new file mode 100644 index 00000000..4e9d3286 --- /dev/null +++ b/internal/versioning/version.go @@ -0,0 +1,9 @@ +package versioning + +var ( + ApplicationVersion string + + // XXX: Bump this value only when there are protocol changes that makes the oracle + // incompatible between version! + ProtocolVersion = `v0.6.0` +) diff --git a/pkg/api/config.go b/pkg/api/config.go new file mode 100644 index 00000000..4ae6f362 --- /dev/null +++ b/pkg/api/config.go @@ -0,0 +1,22 @@ +package api + +import "time" + +// APIConfig contains configuration settings for the API +type APIConfig struct { + WorkerResponseTimeout time.Duration + // Add other API-specific configuration fields here +} + +var DefaultConfig = APIConfig{ + WorkerResponseTimeout: 60 * time.Second, + // Set default values for other fields here +} + +// LoadConfig loads the API configuration +// This can be expanded later to load from environment variables or a file +func LoadConfig() (*APIConfig, error) { + // For now, we'll just return the default config + config := DefaultConfig + return &config, nil +} diff --git a/pkg/api/handlers_data.go b/pkg/api/handlers_data.go index d53d4fc4..c7c3950e 100644 --- a/pkg/api/handlers_data.go +++ b/pkg/api/handlers_data.go @@ -26,6 +26,7 @@ import ( "github.com/masa-finance/masa-oracle/pkg/scrapers/discord" "github.com/masa-finance/masa-oracle/pkg/scrapers/telegram" "github.com/masa-finance/masa-oracle/pkg/workers" + "github.com/masa-finance/masa-oracle/pkg/workers/types" ) type LLMChat struct { @@ -42,7 +43,42 @@ func IsBase64(s string) bool { return err == nil } -// publishWorkRequest sends a work request to the PubSubManager for processing by a worker. +// SendWorkRequest sends a work request to a worker for processing. +// It marshals the request details into JSON and sends it over a libp2p stream. +// It is currently re-using the response channel map for this; however, it could be a simple synchronous call +// in which case the worker handlers would be responseible for preparing the data to be sent back to the client +// +// Parameters: +// - api: The API instance containing the Node and PubSubManager. +// - requestID: A unique identifier for the request. +// - workType: The type of work to be performed by the worker. +// - bodyBytes: The request body in byte slice format. +// +// Returns: +// - error: An error object if the request could not be sent or processed, otherwise nil. +func SendWorkRequest(api *API, requestID string, workType data_types.WorkerType, bodyBytes []byte, wg *sync.WaitGroup) error { + request := data_types.WorkRequest{ + WorkType: workType, + RequestId: requestID, + Data: bodyBytes, + } + response := workers.GetWorkHandlerManager().DistributeWork(api.Node, request) + responseChannel, exists := workers.GetResponseChannelMap().Get(requestID) + if !exists { + return fmt.Errorf("response channel not found") + } + select { + case responseChannel <- response: + wg.Add(1) + // Successfully sent JSON response to the response channel + default: + // Log an error if the channel is blocking for debugging purposes + logrus.Errorf("response channel is blocking for request ID: %s", requestID) + } + return nil +} + +// SendWorkRequest sends a work request to the PubSubManager for processing by a worker. // It marshals the request details into JSON and publishes it to the configured topic. // // Parameters: @@ -53,7 +89,7 @@ func IsBase64(s string) bool { // // Returns: // - error: An error object if the request could not be published, otherwise nil. -func publishWorkRequest(api *API, requestID string, request workers.WorkerType, bodyBytes []byte) error { +func publishWorkRequest(api *API, requestID string, request data_types.WorkerType, bodyBytes []byte) error { workRequest := map[string]string{ "request": string(request), "request_id": requestID, @@ -74,18 +110,24 @@ func publishWorkRequest(api *API, requestID string, request workers.WorkerType, // Parameters: // - c: The gin.Context object, which provides the context for the HTTP request. // - responseCh: A channel that receives the worker's response as a byte slice. -func handleWorkResponse(c *gin.Context, responseCh chan []byte) { +func handleWorkResponse(c *gin.Context, responseCh chan data_types.WorkResponse, wg *sync.WaitGroup) { + cfg, err := LoadConfig() + if err != nil { + logrus.Errorf("Failed to load API cfg: %v", err) + c.JSON(http.StatusInternalServerError, gin.H{"error": "Internal server error"}) + return + } + for { select { case response := <-responseCh: - var result map[string]interface{} - if err := json.Unmarshal(response, &result); err != nil { - c.JSON(http.StatusExpectationFailed, gin.H{"error": err.Error()}) + if response.Error != "" { + c.JSON(http.StatusExpectationFailed, response) + wg.Done() return } - - if data, ok := result["data"].(string); ok && IsBase64(data) { - decodedData, err := base64.StdEncoding.DecodeString(result["data"].(string)) + if data, ok := response.Data.(string); ok && IsBase64(data) { + decodedData, err := base64.StdEncoding.DecodeString(response.Data.(string)) if err != nil { c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to decode base64 data"}) return @@ -96,14 +138,14 @@ func handleWorkResponse(c *gin.Context, responseCh chan []byte) { c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to parse JSON data"}) return } - result["data"] = jsonData + response.Data = jsonData } - - c.JSON(http.StatusOK, result) + response.WorkRequest = nil + c.JSON(http.StatusOK, response) + wg.Done() return - // teslashibe: adjust to timeout after 10 seconds for performance testing - case <-time.After(10 * time.Second): - c.JSON(http.StatusGatewayTimeout, gin.H{"error": "Request timed out, check that port 4001 TCP inbound is open."}) + case <-time.After(cfg.WorkerResponseTimeout): + c.JSON(http.StatusGatewayTimeout, gin.H{"error": "Request timed out in API layer"}) return case <-c.Done(): return @@ -186,14 +228,16 @@ func (api *API) SearchTweetsAndAnalyzeSentiment() gin.HandlerFunc { c.JSON(http.StatusBadRequest, gin.H{"error": wErr.Error()}) } requestID := uuid.New().String() - responseCh := pubsub2.GetResponseChannelMap().CreateChannel(requestID) - defer pubsub2.GetResponseChannelMap().Delete(requestID) - wErr = publishWorkRequest(api, requestID, workers.WORKER.TwitterSentiment, bodyBytes) + responseCh := workers.GetResponseChannelMap().CreateChannel(requestID) + wg := &sync.WaitGroup{} + defer workers.GetResponseChannelMap().Delete(requestID) + go handleWorkResponse(c, responseCh, wg) + + wErr = SendWorkRequest(api, requestID, data_types.TwitterSentiment, bodyBytes, wg) if wErr != nil { c.JSON(http.StatusBadRequest, gin.H{"error": wErr.Error()}) } - handleWorkResponse(c, responseCh) - // worker handler implementation + wg.Wait() } } @@ -236,14 +280,18 @@ func (api *API) SearchDiscordMessagesAndAnalyzeSentiment() gin.HandlerFunc { return } requestID := uuid.New().String() - responseCh := pubsub2.GetResponseChannelMap().CreateChannel(requestID) - defer pubsub2.GetResponseChannelMap().Delete(requestID) - wErr = publishWorkRequest(api, requestID, workers.WORKER.DiscordSentiment, bodyBytes) + responseCh := workers.GetResponseChannelMap().CreateChannel(requestID) + wg := &sync.WaitGroup{} + defer workers.GetResponseChannelMap().Delete(requestID) + go handleWorkResponse(c, responseCh, wg) + + wErr = SendWorkRequest(api, requestID, data_types.DiscordSentiment, bodyBytes, wg) if wErr != nil { c.JSON(http.StatusBadRequest, gin.H{"error": wErr.Error()}) return } - handleWorkResponse(c, responseCh) + wg.Wait() + } } @@ -286,14 +334,17 @@ func (api *API) SearchTelegramMessagesAndAnalyzeSentiment() gin.HandlerFunc { return } requestID := uuid.New().String() - responseCh := pubsub2.GetResponseChannelMap().CreateChannel(requestID) - defer pubsub2.GetResponseChannelMap().Delete(requestID) - wErr = publishWorkRequest(api, requestID, workers.WORKER.TelegramSentiment, bodyBytes) + responseCh := workers.GetResponseChannelMap().CreateChannel(requestID) + wg := &sync.WaitGroup{} + defer workers.GetResponseChannelMap().Delete(requestID) + go handleWorkResponse(c, responseCh, wg) + + wErr = SendWorkRequest(api, requestID, data_types.TelegramSentiment, bodyBytes, wg) if wErr != nil { c.JSON(http.StatusBadRequest, gin.H{"error": wErr.Error()}) return } - handleWorkResponse(c, responseCh) + wg.Wait() } } @@ -338,15 +389,18 @@ func (api *API) SearchWebAndAnalyzeSentiment() gin.HandlerFunc { if wErr != nil { c.JSON(http.StatusBadRequest, gin.H{"error": wErr.Error()}) } + requestID := uuid.New().String() - responseCh := pubsub2.GetResponseChannelMap().CreateChannel(requestID) - defer pubsub2.GetResponseChannelMap().Delete(requestID) - wErr = publishWorkRequest(api, requestID, workers.WORKER.WebSentiment, bodyBytes) + responseCh := workers.GetResponseChannelMap().CreateChannel(requestID) + wg := &sync.WaitGroup{} + defer workers.GetResponseChannelMap().Delete(requestID) + go handleWorkResponse(c, responseCh, wg) + + wErr = SendWorkRequest(api, requestID, data_types.WebSentiment, bodyBytes, wg) if wErr != nil { c.JSON(http.StatusBadRequest, gin.H{"error": wErr.Error()}) } - handleWorkResponse(c, responseCh) - // worker handler implementation + wg.Wait() } } @@ -372,14 +426,16 @@ func (api *API) SearchTweetsProfile() gin.HandlerFunc { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) } requestID := uuid.New().String() - responseCh := pubsub2.GetResponseChannelMap().CreateChannel(requestID) - defer pubsub2.GetResponseChannelMap().Delete(requestID) - err = publishWorkRequest(api, requestID, workers.WORKER.TwitterProfile, bodyBytes) + responseCh := workers.GetResponseChannelMap().CreateChannel(requestID) + wg := &sync.WaitGroup{} + defer workers.GetResponseChannelMap().Delete(requestID) + go handleWorkResponse(c, responseCh, wg) + + err = SendWorkRequest(api, requestID, data_types.TwitterProfile, bodyBytes, wg) if err != nil { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) } - handleWorkResponse(c, responseCh) - // worker handler implementation + wg.Wait() } } @@ -407,14 +463,16 @@ func (api *API) SearchDiscordProfile() gin.HandlerFunc { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) } requestID := uuid.New().String() - responseCh := pubsub2.GetResponseChannelMap().CreateChannel(requestID) - defer pubsub2.GetResponseChannelMap().Delete(requestID) - err = publishWorkRequest(api, requestID, workers.WORKER.DiscordProfile, bodyBytes) + responseCh := workers.GetResponseChannelMap().CreateChannel(requestID) + wg := &sync.WaitGroup{} + defer workers.GetResponseChannelMap().Delete(requestID) + go handleWorkResponse(c, responseCh, wg) + + err = SendWorkRequest(api, requestID, data_types.DiscordProfile, bodyBytes, wg) if err != nil { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) } - handleWorkResponse(c, responseCh) - // worker handler implementation + wg.Wait() } } @@ -451,16 +509,17 @@ func (api *API) SearchChannelMessages() gin.HandlerFunc { } requestID := uuid.New().String() - responseCh := pubsub2.GetResponseChannelMap().CreateChannel(requestID) - defer pubsub2.GetResponseChannelMap().Delete(requestID) + responseCh := workers.GetResponseChannelMap().CreateChannel(requestID) + wg := &sync.WaitGroup{} + defer workers.GetResponseChannelMap().Delete(requestID) + go handleWorkResponse(c, responseCh, wg) - err = publishWorkRequest(api, requestID, workers.WORKER.DiscordChannelMessages, bodyBytes) + err = SendWorkRequest(api, requestID, data_types.DiscordChannelMessages, bodyBytes, wg) if err != nil { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) return } - - handleWorkResponse(c, responseCh) + wg.Wait() } } @@ -484,13 +543,16 @@ func (api *API) SearchGuildChannels() gin.HandlerFunc { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) } requestID := uuid.New().String() - responseCh := pubsub2.GetResponseChannelMap().CreateChannel(requestID) - defer pubsub2.GetResponseChannelMap().Delete(requestID) - err = publishWorkRequest(api, requestID, workers.WORKER.DiscordGuildChannels, bodyBytes) + responseCh := workers.GetResponseChannelMap().CreateChannel(requestID) + wg := &sync.WaitGroup{} + defer workers.GetResponseChannelMap().Delete(requestID) + go handleWorkResponse(c, responseCh, wg) + + err = SendWorkRequest(api, requestID, data_types.DiscordGuildChannels, bodyBytes, wg) if err != nil { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) } - handleWorkResponse(c, responseCh) + wg.Wait() } } @@ -505,13 +567,16 @@ func (api *API) SearchUserGuilds() gin.HandlerFunc { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) } requestID := uuid.New().String() - responseCh := pubsub2.GetResponseChannelMap().CreateChannel(requestID) - defer pubsub2.GetResponseChannelMap().Delete(requestID) - err = publishWorkRequest(api, requestID, workers.WORKER.DiscordUserGuilds, bodyBytes) + responseCh := workers.GetResponseChannelMap().CreateChannel(requestID) + wg := &sync.WaitGroup{} + defer workers.GetResponseChannelMap().Delete(requestID) + go handleWorkResponse(c, responseCh, wg) + + err = SendWorkRequest(api, requestID, data_types.DiscordUserGuilds, bodyBytes, wg) if err != nil { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) } - handleWorkResponse(c, responseCh) + wg.Wait() } } @@ -561,7 +626,12 @@ func (api *API) SearchAllGuilds() gin.HandlerFunc { return } - defer resp.Body.Close() + defer func(Body io.ReadCloser) { + err := Body.Close() + if err != nil { + logrus.Error("[-] Error closing response body: ", err) + } + }(resp.Body) respBody, err := io.ReadAll(resp.Body) if err != nil { errCh <- fmt.Errorf("[-] Failed to read response body: %v", err) @@ -665,15 +735,16 @@ func (api *API) SearchTwitterFollowers() gin.HandlerFunc { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) } requestID := uuid.New().String() - responseCh := pubsub2.GetResponseChannelMap().CreateChannel(requestID) - defer pubsub2.GetResponseChannelMap().Delete(requestID) - err = publishWorkRequest(api, requestID, workers.WORKER.TwitterFollowers, bodyBytes) + responseCh := workers.GetResponseChannelMap().CreateChannel(requestID) + wg := &sync.WaitGroup{} + defer workers.GetResponseChannelMap().Delete(requestID) + go handleWorkResponse(c, responseCh, wg) + + err = SendWorkRequest(api, requestID, data_types.TwitterFollowers, bodyBytes, wg) if err != nil { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) } - handleWorkResponse(c, responseCh) - // worker handler implementation - + wg.Wait() } } @@ -705,14 +776,16 @@ func (api *API) SearchTweetsRecent() gin.HandlerFunc { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) } requestID := uuid.New().String() - responseCh := pubsub2.GetResponseChannelMap().CreateChannel(requestID) - defer pubsub2.GetResponseChannelMap().Delete(requestID) - err = publishWorkRequest(api, requestID, workers.WORKER.Twitter, bodyBytes) + responseCh := workers.GetResponseChannelMap().CreateChannel(requestID) + wg := &sync.WaitGroup{} + defer workers.GetResponseChannelMap().Delete(requestID) + go handleWorkResponse(c, responseCh, wg) + + err = SendWorkRequest(api, requestID, data_types.Twitter, bodyBytes, wg) if err != nil { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) } - handleWorkResponse(c, responseCh) - // worker handler implementation + wg.Wait() } } @@ -724,14 +797,16 @@ func (api *API) SearchTweetsTrends() gin.HandlerFunc { return func(c *gin.Context) { // worker handler implementation requestID := uuid.New().String() - responseCh := pubsub2.GetResponseChannelMap().CreateChannel(requestID) - defer pubsub2.GetResponseChannelMap().Delete(requestID) - err := publishWorkRequest(api, requestID, workers.WORKER.TwitterTrends, nil) + responseCh := workers.GetResponseChannelMap().CreateChannel(requestID) + wg := &sync.WaitGroup{} + defer workers.GetResponseChannelMap().Delete(requestID) + go handleWorkResponse(c, responseCh, wg) + + err := SendWorkRequest(api, requestID, data_types.TwitterTrends, nil, wg) if err != nil { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) } - handleWorkResponse(c, responseCh) - // worker handler implementation + wg.Wait() } } @@ -770,14 +845,17 @@ func (api *API) WebData() gin.HandlerFunc { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) } requestID := uuid.New().String() - responseCh := pubsub2.GetResponseChannelMap().CreateChannel(requestID) - err = publishWorkRequest(api, requestID, workers.WORKER.Web, bodyBytes) - defer pubsub2.GetResponseChannelMap().Delete(requestID) + responseCh := workers.GetResponseChannelMap().CreateChannel(requestID) + wg := &sync.WaitGroup{} + defer workers.GetResponseChannelMap().Delete(requestID) + go handleWorkResponse(c, responseCh, wg) + + err = SendWorkRequest(api, requestID, data_types.Web, bodyBytes, wg) + defer workers.GetResponseChannelMap().Delete(requestID) if err != nil { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) } - handleWorkResponse(c, responseCh) - // worker handler implementation + wg.Wait() } } @@ -809,13 +887,14 @@ func (api *API) CompleteAuth() gin.HandlerFunc { PhoneNumber string `json:"phone_number"` Code string `json:"code"` PhoneCodeHash string `json:"phone_code_hash"` + Password string `json:"password"` } if err := c.ShouldBindJSON(&reqBody); err != nil { c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid request body"}) return } - auth, err := telegram.CompleteAuthentication(context.Background(), reqBody.PhoneNumber, reqBody.Code, reqBody.PhoneCodeHash) + auth, err := telegram.CompleteAuthentication(context.Background(), reqBody.PhoneNumber, reqBody.Code, reqBody.PhoneCodeHash, reqBody.Password) if err != nil { // Check if 2FA is required if err.Error() == "2FA required" { @@ -853,13 +932,16 @@ func (api *API) GetChannelMessagesHandler() gin.HandlerFunc { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) } requestID := uuid.New().String() - responseCh := pubsub2.GetResponseChannelMap().CreateChannel(requestID) - defer pubsub2.GetResponseChannelMap().Delete(requestID) - err = publishWorkRequest(api, requestID, workers.WORKER.TelegramChannelMessages, bodyBytes) + responseCh := workers.GetResponseChannelMap().CreateChannel(requestID) + wg := &sync.WaitGroup{} + defer workers.GetResponseChannelMap().Delete(requestID) + go handleWorkResponse(c, responseCh, wg) + + err = SendWorkRequest(api, requestID, data_types.TelegramChannelMessages, bodyBytes, wg) if err != nil { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) } - handleWorkResponse(c, responseCh) + wg.Wait() } } @@ -906,14 +988,16 @@ func (api *API) LocalLlmChat() gin.HandlerFunc { } requestID := uuid.New().String() - responseCh := pubsub2.GetResponseChannelMap().CreateChannel(requestID) - defer pubsub2.GetResponseChannelMap().Delete(requestID) - err = publishWorkRequest(api, requestID, workers.WORKER.LLMChat, bodyBytes) + responseCh := workers.GetResponseChannelMap().CreateChannel(requestID) + wg := &sync.WaitGroup{} + defer workers.GetResponseChannelMap().Delete(requestID) + go handleWorkResponse(c, responseCh, wg) + + err = SendWorkRequest(api, requestID, data_types.LLMChat, bodyBytes, wg) if err != nil { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) } - handleWorkResponse(c, responseCh) - // worker handler implementation + wg.Wait() } } @@ -991,7 +1075,12 @@ func (api *API) CfLlmChat() gin.HandlerFunc { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) return } - defer resp.Body.Close() + defer func(Body io.ReadCloser) { + err := Body.Close() + if err != nil { + logrus.Error("[-] Error closing response body: ", err) + } + }(resp.Body) respBody, err := io.ReadAll(resp.Body) if err != nil { logrus.Error("[-] Error reading response body: ", err) diff --git a/pkg/config/app.go b/pkg/config/app.go index aae42f03..2e662f85 100644 --- a/pkg/config/app.go +++ b/pkg/config/app.go @@ -8,6 +8,8 @@ import ( "strings" "sync" + "github.com/masa-finance/masa-oracle/internal/versioning" + "github.com/gotd/contrib/bg" "github.com/joho/godotenv" "github.com/sirupsen/logrus" @@ -112,8 +114,8 @@ func GetInstance() *AppConfig { once.Do(func() { instance = &AppConfig{} - instance.setEnvVariableConfig() instance.setDefaultConfig() + instance.setEnvVariableConfig() instance.setFileConfig(viper.GetString("FILE_PATH")) err := instance.setCommandLineConfig() if err != nil { @@ -145,7 +147,7 @@ func (c *AppConfig) setDefaultConfig() { viper.SetDefault(MasaDir, filepath.Join(usr.HomeDir, ".masa")) // Set defaults - viper.SetDefault("Version", Version) + viper.SetDefault("Version", versioning.ProtocolVersion) viper.SetDefault(PortNbr, "4001") viper.SetDefault(UDP, true) viper.SetDefault(TCP, false) @@ -194,6 +196,7 @@ func (c *AppConfig) setCommandLineConfig() error { pflag.StringVar(&c.StakeAmount, "stake", viper.GetString(StakeAmount), "Amount of tokens to stake") pflag.BoolVar(&c.Debug, "debug", viper.GetBool(Debug), "Override some protections for debugging (temporary)") pflag.StringVar(&c.Environment, "env", viper.GetString(Environment), "Environment to connect to") + pflag.StringVar(&c.Version, "version", viper.GetString("VERSION"), "application version") pflag.BoolVar(&c.AllowedPeer, "allowedPeer", viper.GetBool(AllowedPeer), "Set to true to allow setting this node as the allowed peer") pflag.StringVar(&c.PrivateKey, "privateKey", viper.GetString(PrivateKey), "The private key") pflag.StringVar(&c.PrivateKeyFile, "privKeyFile", viper.GetString(PrivKeyFile), "The private key file") @@ -222,6 +225,7 @@ func (c *AppConfig) setCommandLineConfig() error { pflag.BoolVar(&c.WebScraper, "webScraper", viper.GetBool(WebScraper), "WebScraper") pflag.BoolVar(&c.LlmServer, "llmServer", viper.GetBool(LlmServer), "Can service LLM requests") pflag.BoolVar(&c.Faucet, "faucet", viper.GetBool(Faucet), "Faucet") + pflag.Parse() // Bind command line flags to Viper (optional, if you want to use Viper for additional configuration) diff --git a/pkg/config/constants.go b/pkg/config/constants.go index f1e49b5f..d3fd5a1e 100644 --- a/pkg/config/constants.go +++ b/pkg/config/constants.go @@ -100,6 +100,7 @@ const ( MasaPrefix = "/masa" OracleProtocol = "oracle_protocol" + WorkerProtocol = "worker_protocol" NodeDataSyncProtocol = "nodeDataSync" NodeGossipTopic = "gossip" PublicKeyTopic = "bootNodePublicKey" diff --git a/pkg/config/version.go b/pkg/config/version.go deleted file mode 100644 index b6de792e..00000000 --- a/pkg/config/version.go +++ /dev/null @@ -1,5 +0,0 @@ -package config - -var ( - Version string -) diff --git a/pkg/config/welcome.go b/pkg/config/welcome.go index 968edd0e..7f5000ba 100644 --- a/pkg/config/welcome.go +++ b/pkg/config/welcome.go @@ -4,7 +4,7 @@ import ( "fmt" ) -func DisplayWelcomeMessage(multiAddr, ipAddr, publicKeyHex string, isStaked bool, isValidator bool, isTwitterScraper bool, isTelegramScraper bool, isDiscordScraper bool, isWebScraper bool, version string) { +func DisplayWelcomeMessage(multiAddr, ipAddr, publicKeyHex string, isStaked bool, isValidator bool, isTwitterScraper bool, isTelegramScraper bool, isDiscordScraper bool, isWebScraper bool, version, protocolVersion string) { // ANSI escape code for yellow text yellow := "\033[33m" blue := "\033[34m" @@ -23,7 +23,8 @@ func DisplayWelcomeMessage(multiAddr, ipAddr, publicKeyHex string, isStaked bool fmt.Println(yellow + borderLine + reset) fmt.Println("") - fmt.Printf(blue+"%-20s %s\n"+reset, "Version:", yellow+version) + fmt.Printf(blue+"%-20s %s\n"+reset, "Application Version:", yellow+version) + fmt.Printf(blue+"%-20s %s\n"+reset, "Protocol Version:", yellow+protocolVersion) fmt.Printf(blue+"%-20s %s\n"+reset, "Multiaddress:", multiAddr) fmt.Printf(blue+"%-20s %s\n"+reset, "IP Address:", ipAddr) fmt.Printf(blue+"%-20s %s\n"+reset, "Public Key:", publicKeyHex) diff --git a/pkg/llmbridge/sentiment.go b/pkg/llmbridge/sentiment.go index e4384241..5f780567 100644 --- a/pkg/llmbridge/sentiment.go +++ b/pkg/llmbridge/sentiment.go @@ -10,10 +10,11 @@ import ( "strings" "github.com/gotd/td/tg" - "github.com/masa-finance/masa-oracle/pkg/config" twitterscraper "github.com/masa-finance/masa-twitter-scraper" "github.com/ollama/ollama/api" "github.com/sirupsen/logrus" + + "github.com/masa-finance/masa-oracle/pkg/config" ) // AnalyzeSentimentTweets analyzes the sentiment of the provided tweets by sending them to the Claude API. @@ -194,8 +195,8 @@ func AnalyzeSentimentWeb(data string, model string, prompt string) (string, stri genReq := api.ChatRequest{ Model: strings.TrimPrefix(model, "ollama/"), Messages: []api.Message{ - {Role: "user", Content: data}, {Role: "assistant", Content: prompt}, + {Role: "user", Content: data}, }, Stream: &stream, Options: map[string]interface{}{ diff --git a/pkg/masacrypto/hash.go b/pkg/masacrypto/hash.go new file mode 100644 index 00000000..76a07fac --- /dev/null +++ b/pkg/masacrypto/hash.go @@ -0,0 +1,33 @@ +package masacrypto + +import ( + "github.com/ipfs/go-cid" + mh "github.com/multiformats/go-multihash" + "github.com/sirupsen/logrus" +) + +// ComputeSha256Cid calculates the CID (Content Identifier) for a given string. +// +// Parameters: +// - str: The input string for which to compute the CID. +// +// Returns: +// - string: The computed CID as a string. +// - error: An error, if any occurred during the CID computation. +// +// The function uses the multihash package to create a SHA2-256 hash of the input string. +// It then creates a CID (version 1) from the multihash and returns the CID as a string. +// If an error occurs during the multihash computation or CID creation, it is returned. +func ComputeSha256Cid(str string) (string, error) { + logrus.Infof("Computing CID for string: %s", str) + // Create a multihash from the string + mhHash, err := mh.Sum([]byte(str), mh.SHA2_256, -1) + if err != nil { + logrus.Errorf("Error computing multihash for string: %s, error: %v", str, err) + return "", err + } + // Create a CID from the multihash + cidKey := cid.NewCidV1(cid.Raw, mhHash).String() + logrus.Infof("Computed CID: %s", cidKey) + return cidKey, nil +} diff --git a/pkg/network/discover.go b/pkg/network/discover.go index 95f6af1a..5c25c385 100644 --- a/pkg/network/discover.go +++ b/pkg/network/discover.go @@ -8,9 +8,10 @@ import ( "github.com/cenkalti/backoff/v4" "github.com/libp2p/go-libp2p/core/peer" - "github.com/masa-finance/masa-oracle/pkg/config" "github.com/multiformats/go-multiaddr" + "github.com/masa-finance/masa-oracle/pkg/config" + dht "github.com/libp2p/go-libp2p-kad-dht" "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/core/network" @@ -39,7 +40,7 @@ func Discover(ctx context.Context, host host.Host, dht *dht.IpfsDHT, protocol pr logrus.Infof("[+] Successfully advertised protocol %s", protocolString) } - ticker := time.NewTicker(time.Second * 10) + ticker := time.NewTicker(time.Minute * 1) defer ticker.Stop() var peerChan <-chan peer.AddrInfo diff --git a/pkg/network/kdht.go b/pkg/network/kdht.go index 9b6cd1a1..5bb8a78e 100644 --- a/pkg/network/kdht.go +++ b/pkg/network/kdht.go @@ -30,7 +30,7 @@ func (dbValidator) Validate(_ string, _ []byte) error { return nil } func (dbValidator) Select(_ string, _ [][]byte) (int, error) { return 0, nil } func WithDht(ctx context.Context, host host.Host, bootstrapNodes []multiaddr.Multiaddr, - protocolId, prefix protocol.ID, peerChan chan PeerEvent, isStaked bool, removePeerCallback func(peer.ID)) (*dht.IpfsDHT, error) { + protocolId, prefix protocol.ID, peerChan chan PeerEvent, isStaked bool) (*dht.IpfsDHT, error) { options := make([]dht.Option, 0) options = append(options, dht.BucketSize(100)) // Adjust bucket size options = append(options, dht.Concurrency(100)) // Increase concurrency @@ -46,8 +46,6 @@ func WithDht(ctx context.Context, host host.Host, bootstrapNodes []multiaddr.Mul go monitorRoutingTable(ctx, kademliaDHT, time.Minute) kademliaDHT.RoutingTable().PeerAdded = func(p peer.ID) { - logrus.Infof("[+] Peer added to DHT: %s", p.String()) - pe := PeerEvent{ AddrInfo: peer.AddrInfo{ID: p}, Action: PeerAdded, @@ -57,16 +55,12 @@ func WithDht(ctx context.Context, host host.Host, bootstrapNodes []multiaddr.Mul } kademliaDHT.RoutingTable().PeerRemoved = func(p peer.ID) { - logrus.Infof("[-] Peer removed from DHT: %s", p) pe := PeerEvent{ AddrInfo: peer.AddrInfo{ID: p}, Action: PeerRemoved, Source: "kdht", } peerChan <- pe - if removePeerCallback != nil { - removePeerCallback(p) - } } if err = kademliaDHT.Bootstrap(ctx); err != nil { @@ -127,7 +121,14 @@ func WithDht(ctx context.Context, host host.Host, bootstrapNodes []multiaddr.Mul logrus.Errorf("[-] Error closing stream: %s", err) } }(stream) // Close the stream when done - _, err = stream.Write(pubsub.GetSelfNodeDataJson(host, isStaked)) + + multiaddr, err := GetMultiAddressesForHost(host) + if err != nil { + logrus.Errorf("[-] Error getting multiaddresses for host: %s", err) + return + } + multaddrString := GetPriorityAddress(multiaddr) + _, err = stream.Write(pubsub.GetSelfNodeDataJson(host, isStaked, multaddrString.String())) if err != nil { logrus.Errorf("[-] Error writing to stream: %s", err) return @@ -153,10 +154,6 @@ func monitorRoutingTable(ctx context.Context, dht *dht.IpfsDHT, interval time.Du routingTable := dht.RoutingTable() // Log the size of the routing table logrus.Infof("[+] Routing table size: %d", routingTable.Size()) - // Log the peer IDs in the routing table - for _, p := range routingTable.ListPeers() { - logrus.Debugf("[+] Peer in routing table: %s", p.String()) - } case <-ctx.Done(): // If the context is cancelled, stop the goroutine return diff --git a/pkg/oracle_node.go b/pkg/oracle_node.go index ee93b70f..1089c56f 100644 --- a/pkg/oracle_node.go +++ b/pkg/oracle_node.go @@ -7,8 +7,6 @@ import ( "encoding/base64" "encoding/json" "fmt" - "io" - "log/slog" "net" "os" "reflect" @@ -16,15 +14,10 @@ import ( "sync" "time" - "github.com/asynkron/protoactor-go/actor" - "github.com/asynkron/protoactor-go/remote" - "github.com/chyeh/pubip" - "github.com/libp2p/go-libp2p" dht "github.com/libp2p/go-libp2p-kad-dht" "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/core/network" - "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/protocol" rcmgr "github.com/libp2p/go-libp2p/p2p/host/resource-manager" "github.com/libp2p/go-libp2p/p2p/muxer/yamux" @@ -37,7 +30,9 @@ import ( shell "github.com/ipfs/go-ipfs-api" pubsub "github.com/libp2p/go-libp2p-pubsub" - chain "github.com/masa-finance/masa-oracle/pkg/chain" + + "github.com/masa-finance/masa-oracle/internal/versioning" + "github.com/masa-finance/masa-oracle/pkg/chain" "github.com/masa-finance/masa-oracle/pkg/config" "github.com/masa-finance/masa-oracle/pkg/masacrypto" myNetwork "github.com/masa-finance/masa-oracle/pkg/network" @@ -66,8 +61,6 @@ type OracleNode struct { StartTime time.Time WorkerTracker *pubsub2.WorkerEventTracker BlockTracker *BlockEventTracker - ActorEngine *actor.RootContext - ActorRemote *remote.Remote Blockchain *chain.Chain } @@ -88,8 +81,14 @@ func getOutboundIP() string { conn, err := net.Dial("udp", "8.8.8.8:80") if err != nil { fmt.Println("[-] Error getting outbound IP") + return "" } - defer conn.Close() + defer func(conn net.Conn) { + err := conn.Close() + if err != nil { + + } + }(conn) localAddr := conn.LocalAddr().String() idx := strings.LastIndex(localAddr, ":") return localAddr[0:idx] @@ -150,27 +149,6 @@ func NewOracleNode(ctx context.Context, isStaked bool) (*OracleNode, error) { return nil, err } - system := actor.NewActorSystemWithConfig(actor.Configure( - actor.ConfigOption(func(config *actor.Config) { - config.LoggerFactory = func(system *actor.ActorSystem) *slog.Logger { - return slog.New(slog.NewTextHandler(io.Discard, nil)) - } - }), - )) - engine := system.Root - - var ip any - if cfg.Environment == "local" { - ip = getOutboundIP() - } else { - ip, _ = pubip.Get() - } - conf := remote.Configure("0.0.0.0", 4001, - remote.WithAdvertisedHost(fmt.Sprintf("%s:4001", ip))) - - r := remote.NewRemote(system, conf) - go r.Start() - return &OracleNode{ Host: hst, PrivKey: masacrypto.KeyManagerInstance().EcdsaPrivKey, @@ -178,7 +156,7 @@ func NewOracleNode(ctx context.Context, isStaked bool) (*OracleNode, error) { multiAddrs: myNetwork.GetMultiAddressesForHostQuiet(hst), Context: ctx, PeerChan: make(chan myNetwork.PeerEvent), - NodeTracker: pubsub2.NewNodeEventTracker(config.Version, cfg.Environment, hst.ID().String()), + NodeTracker: pubsub2.NewNodeEventTracker(versioning.ProtocolVersion, cfg.Environment, hst.ID().String()), PubSubManager: subscriptionManager, IsStaked: isStaked, IsValidator: cfg.Validator, @@ -187,8 +165,6 @@ func NewOracleNode(ctx context.Context, isStaked bool) (*OracleNode, error) { IsTelegramScraper: cfg.TelegramScraper, IsWebScraper: cfg.WebScraper, IsLlmServer: cfg.LlmServer, - ActorEngine: engine, - ActorRemote: r, Blockchain: &chain.Chain{}, }, nil } @@ -216,11 +192,7 @@ func (node *OracleNode) Start() (err error) { go node.ListenToNodeTracker() go node.handleDiscoveredPeers() - removePeerCallback := func(p peer.ID) { - node.NodeTracker.RemoveNodeData(p.String()) - } - - node.DHT, err = myNetwork.WithDht(node.Context, node.Host, bootNodeAddrs, node.Protocol, config.MasaPrefix, node.PeerChan, node.IsStaked, removePeerCallback) + node.DHT, err = myNetwork.WithDht(node.Context, node.Host, bootNodeAddrs, node.Protocol, config.MasaPrefix, node.PeerChan, node.IsStaked) if err != nil { return err } @@ -234,7 +206,8 @@ func (node *OracleNode) Start() (err error) { nodeData := node.NodeTracker.GetNodeData(node.Host.ID().String()) if nodeData == nil { publicKeyHex := masacrypto.KeyManagerInstance().EthAddress - nodeData = pubsub2.NewNodeData(node.GetMultiAddrs(), node.Host.ID(), publicKeyHex, pubsub2.ActivityJoined) + ma := myNetwork.GetMultiAddressesForHostQuiet(node.Host) + nodeData = pubsub2.NewNodeData(ma[0], node.Host.ID(), publicKeyHex, pubsub2.ActivityJoined) nodeData.IsStaked = node.IsStaked nodeData.SelfIdentified = true nodeData.IsDiscordScraper = node.IsDiscordScraper @@ -504,7 +477,12 @@ func SubscribeToBlocks(ctx context.Context, node *OracleNode) { return } - go node.Blockchain.Init() + go func() { + err := node.Blockchain.Init() + if err != nil { + logrus.Error(err) + } + }() updateTicker := time.NewTicker(time.Second * 60) defer updateTicker.Stop() diff --git a/pkg/oracle_node_listener.go b/pkg/oracle_node_listener.go index 06598aa3..228df490 100644 --- a/pkg/oracle_node_listener.go +++ b/pkg/oracle_node_listener.go @@ -146,7 +146,7 @@ func (node *OracleNode) SendNodeData(peerID peer.ID) { logrus.Debugf("[-] Failed to close stream: %v", err) } }(stream) // Ensure the stream is closed after sending the data - logrus.Infof("[+] Sending %d node data records to %s", totalRecords, peerID) + logrus.Debugf("[+] Sending %d node data records to %s", totalRecords, peerID) for pageNumber := 0; pageNumber < totalPages; pageNumber++ { node.SendNodeDataPage(nodeData, stream, pageNumber) } diff --git a/pkg/pubsub/node_data.go b/pkg/pubsub/node_data.go index dfcd6dc7..65ae139d 100644 --- a/pkg/pubsub/node_data.go +++ b/pkg/pubsub/node_data.go @@ -5,6 +5,7 @@ import ( "fmt" "time" + "github.com/masa-finance/masa-oracle/internal/versioning" "github.com/masa-finance/masa-oracle/pkg/config" "github.com/libp2p/go-libp2p/core/host" @@ -45,6 +46,7 @@ func (m *JSONMultiaddr) UnmarshalJSON(b []byte) error { type NodeData struct { Multiaddrs []JSONMultiaddr `json:"multiaddrs,omitempty"` + MultiaddrsString string `json:"multiaddrsString,omitempty"` PeerId peer.ID `json:"peerId"` FirstJoinedUnix int64 `json:"firstJoined,omitempty"` LastJoinedUnix int64 `json:"lastJoined,omitempty"` @@ -78,6 +80,7 @@ func NewNodeData(addr multiaddr.Multiaddr, peerId peer.ID, publicKey string, act return &NodeData{ PeerId: peerId, Multiaddrs: multiaddrs, + MultiaddrsString: addr.String(), LastUpdatedUnix: time.Now().Unix(), CurrentUptime: 0, AccumulatedUptime: 0, @@ -91,6 +94,10 @@ func NewNodeData(addr multiaddr.Multiaddr, peerId peer.ID, publicKey string, act // and peer ID in the format "/ip4/127.0.0.1/tcp/4001/p2p/QmcgpsyWgH8Y8ajJz1Cu72KnS5uo2Aa2LpzU7kinSupNKC". // This can be used by other nodes to connect to this node. func (n *NodeData) Address() string { + // Add a check for empty addresses + if len(n.Multiaddrs) == 0 { + return "" + } return fmt.Sprintf("%s/p2p/%s", n.Multiaddrs[0].String(), n.PeerId.String()) } @@ -117,16 +124,18 @@ func (n *NodeData) CanDoWork(workerType WorkerCategory) bool { logrus.Infof("[+] Skipping worker %s due to timeout", n.PeerId) return false } - + if !(n.IsStaked && n.IsActive) { + return false + } switch workerType { case CategoryTwitter: - return n.IsActive && n.IsTwitterScraper + return n.IsTwitterScraper case CategoryDiscord: - return n.IsActive && n.IsDiscordScraper + return n.IsDiscordScraper case CategoryTelegram: - return n.IsActive && n.IsTelegramScraper + return n.IsTelegramScraper case CategoryWeb: - return n.IsActive && n.IsWebScraper + return n.IsWebScraper default: return false } @@ -250,10 +259,11 @@ func (n *NodeData) UpdateAccumulatedUptime() { // It populates a NodeData struct with the node's ID, staking status, and Ethereum address. // The NodeData struct is then marshalled into a JSON byte array. // Returns nil if there is an error marshalling to JSON. -func GetSelfNodeDataJson(host host.Host, isStaked bool) []byte { +func GetSelfNodeDataJson(host host.Host, isStaked bool, multiaddrString string) []byte { // Create and populate NodeData nodeData := NodeData{ PeerId: host.ID(), + MultiaddrsString: multiaddrString, IsStaked: isStaked, EthAddress: masacrypto.KeyManagerInstance().EthAddress, IsTwitterScraper: config.GetInstance().TwitterScraper, @@ -262,7 +272,7 @@ func GetSelfNodeDataJson(host host.Host, isStaked bool) []byte { IsWebScraper: config.GetInstance().WebScraper, IsValidator: config.GetInstance().Validator, IsActive: true, - Version: config.Version, + Version: versioning.ProtocolVersion, } // Convert NodeData to JSON diff --git a/pkg/pubsub/node_event_tracker.go b/pkg/pubsub/node_event_tracker.go index f5402bda..c19f7f59 100644 --- a/pkg/pubsub/node_event_tracker.go +++ b/pkg/pubsub/node_event_tracker.go @@ -249,7 +249,7 @@ func (net *NodeEventTracker) GetUpdatedNodes(since time.Time) []NodeData { } // GetEthAddress returns the Ethereum address for the given remote peer. -// It gets the peer's public key from the network's peerstore, converts +// It gets the peer's public key from the network's peer store, converts // it to a hex string, and converts that to an Ethereum address. // Returns an empty string if there is no public key for the peer. func GetEthAddress(remotePeer peer.ID, n network.Network) string { @@ -273,6 +273,18 @@ func GetEthAddress(remotePeer peer.ID, n network.Network) string { return publicKeyHex } +// GetEligibleWorkerNodes returns a slice of NodeData for nodes that are eligible to perform a specific category of work. +func (net *NodeEventTracker) GetEligibleWorkerNodes(category WorkerCategory) []NodeData { + logrus.Debugf("Getting eligible worker nodes for category: %s", category) + result := make([]NodeData, 0) + for _, nodeData := range net.GetAllNodeData() { + if nodeData.CanDoWork(category) { + result = append(result, nodeData) + } + } + return result +} + // IsStaked returns whether the node with the given peerID is marked as staked in the node data tracker. // Returns false if no node data is found for the given peerID. func (net *NodeEventTracker) IsStaked(peerID string) bool { @@ -366,7 +378,7 @@ func (net *NodeEventTracker) AddOrUpdateNodeData(nodeData *NodeData, forceGossip // entry, and if expired, processes the connect and removes the entry. func (net *NodeEventTracker) ClearExpiredBufferEntries() { for { - time.Sleep(30 * time.Second) // E.g., every 5 seconds + time.Sleep(1 * time.Minute) now := time.Now() for peerID, entry := range net.ConnectBuffer { if now.Sub(entry.ConnectTime) > time.Minute*1 { @@ -388,11 +400,13 @@ func (net *NodeEventTracker) ClearExpiredBufferEntries() { // // Parameters: // - peerID: A string representing the ID of the peer to be removed. -func (net *NodeEventTracker) RemoveNodeData(peerID string) { - net.nodeData.Delete(peerID) - delete(net.ConnectBuffer, peerID) - logrus.Infof("[+] Removed peer %s from NodeTracker", peerID) -} +// +// TODO: we should never remove node data from the internal map. Otherwise we lose all tracking of activity. +//func (net *NodeEventTracker) RemoveNodeData(peerID string) { +// net.nodeData.Delete(peerID) +// delete(net.ConnectBuffer, peerID) +// logrus.Infof("[+] Removed peer %s from NodeTracker", peerID) +//} // ClearExpiredWorkerTimeouts periodically checks and clears expired worker timeouts. // It runs in an infinite loop, sleeping for 5 minutes between each iteration. @@ -445,7 +459,6 @@ func (net *NodeEventTracker) cleanupStalePeers(hostId string) { if now.Sub(time.Unix(nodeData.LastUpdatedUnix, 0)) > maxDisconnectionTime { if nodeData.PeerId.String() != hostId { logrus.Infof("Removing stale peer: %s", nodeData.PeerId) - net.RemoveNodeData(nodeData.PeerId.String()) delete(net.ConnectBuffer, nodeData.PeerId.String()) // Notify about peer removal diff --git a/pkg/scrapers/discord/discordprofile.go b/pkg/scrapers/discord/discordprofile.go index 0e392386..b66d54e9 100644 --- a/pkg/scrapers/discord/discordprofile.go +++ b/pkg/scrapers/discord/discordprofile.go @@ -1,4 +1,3 @@ -// discordprofile.go package discord import ( diff --git a/pkg/scrapers/telegram/telegram_client.go b/pkg/scrapers/telegram/telegram_client.go index 8bc42c8a..836d2ea3 100644 --- a/pkg/scrapers/telegram/telegram_client.go +++ b/pkg/scrapers/telegram/telegram_client.go @@ -8,6 +8,7 @@ import ( "os" "path/filepath" "strconv" + "strings" "sync" "github.com/gotd/contrib/bg" @@ -113,12 +114,13 @@ func StartAuthentication(ctx context.Context, phoneNumber string) (string, error } // CompleteAuthentication uses the provided code to authenticate with Telegram. -func CompleteAuthentication(ctx context.Context, phoneNumber, code, phoneCodeHash string) (*tg.AuthAuthorization, error) { +// CompleteAuthentication uses the provided code to authenticate with Telegram. +func CompleteAuthentication(ctx context.Context, phoneNumber, code, phoneCodeHash, password string) (*tg.AuthAuthorization, error) { // Initialize the Telegram client (if not already initialized) client, err := GetClient() if err != nil { logrus.Printf("Failed to initialize Telegram client: %v", err) - return nil, err // Edit: Added nil as the first return value + return nil, err } // Define a variable to hold the authentication result @@ -127,9 +129,18 @@ func CompleteAuthentication(ctx context.Context, phoneNumber, code, phoneCodeHas err = client.Run(ctx, func(ctx context.Context) error { // Use the provided code and phoneCodeHash to authenticate auth, err := client.Auth().SignIn(ctx, phoneNumber, code, phoneCodeHash) + if err != nil { - log.Printf("Error during SignIn: %v", err) - return err + if strings.Contains(err.Error(), "2FA required") { + auth, err = client.Auth().Password(ctx, password) + if err != nil { + log.Printf("Error during 2FA SignIn: %v", err) + return err + } + } else { + log.Printf("Error during SignIn: %v", err) + return err + } } // At this point, authentication was successful, and you have the user's Telegram auth data. diff --git a/pkg/scrapers/twitter/tweets.go b/pkg/scrapers/twitter/tweets.go index 8348d6de..c45d26bd 100644 --- a/pkg/scrapers/twitter/tweets.go +++ b/pkg/scrapers/twitter/tweets.go @@ -7,10 +7,11 @@ import ( _ "github.com/lib/pq" - "github.com/masa-finance/masa-oracle/pkg/config" - "github.com/masa-finance/masa-oracle/pkg/llmbridge" twitterscraper "github.com/masa-finance/masa-twitter-scraper" "github.com/sirupsen/logrus" + + "github.com/masa-finance/masa-oracle/pkg/config" + "github.com/masa-finance/masa-oracle/pkg/llmbridge" ) type TweetResult struct { @@ -151,7 +152,12 @@ func ScrapeTweetsByQuery(query string, count int) ([]*TweetResult, error) { } tweets = append(tweets, &tweet) } - return tweets, tweets[0].Error + + if len(tweets) == 0 { + return nil, fmt.Errorf("no tweets found for the given query") + } + + return tweets, nil } // ScrapeTweetsByTrends scrapes the current trending topics on Twitter. diff --git a/pkg/workers/config.go b/pkg/workers/config.go new file mode 100644 index 00000000..7da59bef --- /dev/null +++ b/pkg/workers/config.go @@ -0,0 +1,43 @@ +package workers + +import ( + "time" + + "github.com/sirupsen/logrus" +) + +type WorkerConfig struct { + WorkerTimeout time.Duration + WorkerResponseTimeout time.Duration + ConnectionTimeout time.Duration + MaxRetries int + MaxSpawnAttempts int + WorkerBufferSize int + MaxRemoteWorkers int +} + +var DefaultConfig = WorkerConfig{ + WorkerTimeout: 55 * time.Second, + WorkerResponseTimeout: 30 * time.Second, + ConnectionTimeout: 1 * time.Second, + MaxRetries: 1, + MaxSpawnAttempts: 1, + WorkerBufferSize: 100, + MaxRemoteWorkers: 1, +} + +var workerConfig *WorkerConfig + +func init() { + var err error + workerConfig, err = LoadConfig() + if err != nil { + logrus.Fatalf("Failed to load worker config: %v", err) + } +} + +func LoadConfig() (*WorkerConfig, error) { + // For now, we'll just return the default config + config := DefaultConfig + return &config, nil +} diff --git a/pkg/workers/handler.go b/pkg/workers/handler.go deleted file mode 100644 index c7bf94e0..00000000 --- a/pkg/workers/handler.go +++ /dev/null @@ -1,219 +0,0 @@ -package workers - -import ( - "context" - "encoding/json" - "fmt" - "net" - "strings" - - "github.com/asynkron/protoactor-go/actor" - pubsub2 "github.com/libp2p/go-libp2p-pubsub" - masa "github.com/masa-finance/masa-oracle/pkg" - "github.com/masa-finance/masa-oracle/pkg/config" - "github.com/masa-finance/masa-oracle/pkg/scrapers/discord" - "github.com/masa-finance/masa-oracle/pkg/scrapers/telegram" - "github.com/masa-finance/masa-oracle/pkg/scrapers/twitter" - "github.com/masa-finance/masa-oracle/pkg/scrapers/web" - "github.com/masa-finance/masa-oracle/pkg/workers/messages" - "github.com/multiformats/go-multiaddr" - "github.com/sirupsen/logrus" -) - -type LLMChatBody struct { - Model string `json:"model,omitempty"` - Messages []struct { - Role string `json:"role"` - Content string `json:"content"` - } `json:"messages,omitempty"` - Stream bool `json:"stream"` -} - -// getPeers is a function that takes an OracleNode as an argument and returns a slice of actor.PID pointers. -// These actor.PID pointers represent the peers of the given OracleNode in the network. -func getPeers(node *masa.OracleNode) []*actor.PID { - var actors []*actor.PID - peers := node.Host.Network().Peers() - for _, p := range peers { - conns := node.Host.Network().ConnsToPeer(p) - for _, conn := range conns { - addr := conn.RemoteMultiaddr() - ipAddr, _ := addr.ValueForProtocol(multiaddr.P_IP4) - if p.String() != node.Host.ID().String() { - spawned, err := node.ActorRemote.SpawnNamed(fmt.Sprintf("%s:4001", ipAddr), "worker", "peer", -1) - if err != nil { - if strings.Contains(err.Error(), "future: dead letter") { - logrus.Debugf("Ignoring dead letter error for peer %s: %v", p.String(), err) - continue - } - logrus.Debugf("Spawned error %v", err) - } else { - actors = append(actors, spawned.Pid) - } - } - } - } - return actors -} - -// HandleConnect is a method of the Worker struct that handles the connection of a worker. -// It takes in an actor context and a Connect message as parameters. -func (a *Worker) HandleConnect(ctx actor.Context, m *messages.Connect) { - logrus.Infof("[+] Worker %v connected", m.Sender) - clients.Add(m.Sender) -} - -// HandleLog is a method of the Worker struct that handles logging. -// It takes in an actor context and a string message as parameters. -func (a *Worker) HandleLog(ctx actor.Context, l string) { - logrus.Info(l) -} - -// HandleWork is a method of the Worker struct that handles the work assigned to a worker. -// It takes in an actor context and a Work message as parameters. -func (a *Worker) HandleWork(ctx actor.Context, m *messages.Work, node *masa.OracleNode) { - var resp interface{} - var err error - - var workData map[string]string - err = json.Unmarshal([]byte(m.Data), &workData) - if err != nil { - logrus.Errorf("[-] Error parsing work data: %v", err) - return - } - - var bodyData map[string]interface{} - if workData["body"] != "" { - if err := json.Unmarshal([]byte(workData["body"]), &bodyData); err != nil { - logrus.Errorf("[-] Error unmarshalling body: %v", err) - return - } - } - - switch workData["request"] { - case string(WORKER.DiscordProfile): - userID := bodyData["userID"].(string) - resp, err = discord.GetUserProfile(userID) - case string(WORKER.DiscordChannelMessages): - channelID := bodyData["channelID"].(string) - resp, err = discord.GetChannelMessages(channelID, bodyData["limit"].(string), bodyData["before"].(string)) - case string(WORKER.DiscordSentiment): - logrus.Infof("[+] Discord Channel Messages %s %s", m.Data, m.Sender) - channelID := bodyData["channelID"].(string) - _, resp, err = discord.ScrapeDiscordMessagesForSentiment(channelID, bodyData["model"].(string), bodyData["prompt"].(string)) - case string(WORKER.TelegramChannelMessages): - logrus.Infof("[+] Telegram Channel Messages %s %s", m.Data, m.Sender) - username := bodyData["username"].(string) - resp, err = telegram.FetchChannelMessages(context.Background(), username) // Removed the underscore placeholder - case string(WORKER.TelegramSentiment): - logrus.Infof("[+] Telegram Channel Messages %s %s", m.Data, m.Sender) - username := bodyData["username"].(string) - _, resp, err = telegram.ScrapeTelegramMessagesForSentiment(context.Background(), username, bodyData["model"].(string), bodyData["prompt"].(string)) - case string(WORKER.DiscordGuildChannels): - guildID := bodyData["guildID"].(string) - resp, err = discord.GetGuildChannels(guildID) - case string(WORKER.DiscordUserGuilds): - resp, err = discord.GetUserGuilds() - case string(WORKER.LLMChat): - uri := config.GetInstance().LLMChatUrl - if uri == "" { - logrus.Error("[-] Missing env variable LLM_CHAT_URL") - return - } - bodyBytes, _ := json.Marshal(bodyData) - headers := map[string]string{ - "Content-Type": "application/json", - } - resp, _ = Post(uri, bodyBytes, headers) - case string(WORKER.Twitter): - query := bodyData["query"].(string) - count := int(bodyData["count"].(float64)) - resp, err = twitter.ScrapeTweetsByQuery(query, count) - case string(WORKER.TwitterFollowers): - username := bodyData["username"].(string) - count := int(bodyData["count"].(float64)) - resp, err = twitter.ScrapeFollowersForProfile(username, count) - case string(WORKER.TwitterProfile): - username := bodyData["username"].(string) - resp, err = twitter.ScrapeTweetsProfile(username) - case string(WORKER.TwitterSentiment): - count := int(bodyData["count"].(float64)) - _, resp, err = twitter.ScrapeTweetsForSentiment(bodyData["query"].(string), count, bodyData["model"].(string)) - case string(WORKER.TwitterTrends): - resp, err = twitter.ScrapeTweetsByTrends() - case string(WORKER.Web): - depth := int(bodyData["depth"].(float64)) - resp, err = web.ScrapeWebData([]string{bodyData["url"].(string)}, depth) - case string(WORKER.WebSentiment): - depth := int(bodyData["depth"].(float64)) - _, resp, err = web.ScrapeWebDataForSentiment([]string{bodyData["url"].(string)}, depth, bodyData["model"].(string)) - case string(WORKER.Test): - count := int(bodyData["count"].(float64)) - resp, err = func(count int) (interface{}, error) { - return count, err - }(count) - default: - logrus.Warningf("[+] Received unknown message: %T", m) - return - } - - if err != nil { - host, _, err := net.SplitHostPort(m.Sender.Address) - addrs := node.Host.Addrs() - isLocalHost := false - for _, addr := range addrs { - addrStr := addr.String() - if strings.HasPrefix(addrStr, "/ip4/") { - ipStr := strings.Split(strings.Split(addrStr, "/")[2], "/")[0] - if host == ipStr { - isLocalHost = true - break - } - } - } - - if isLocalHost { - logrus.Errorf("[-] Local node: Error processing request: %s", err.Error()) - } else { - logrus.Errorf("[-] Remote node %s: Error processing request: %s", m.Sender, err.Error()) - } - - chanResponse := ChanResponse{ - Response: map[string]interface{}{"error": err.Error()}, - ChannelId: workData["request_id"], - } - val := &pubsub2.Message{ - ValidatorData: chanResponse, - ID: m.Id, - } - jsn, err := json.Marshal(val) - if err != nil { - logrus.Errorf("[-] Error marshalling response: %v", err) - return - } - ctx.Respond(&messages.Response{RequestId: workData["request_id"], Value: string(jsn)}) - } else { - chanResponse := ChanResponse{ - Response: map[string]interface{}{"data": resp}, - ChannelId: workData["request_id"], - } - val := &pubsub2.Message{ - ValidatorData: chanResponse, - ID: m.Id, - } - jsn, err := json.Marshal(val) - if err != nil { - logrus.Errorf("[-] Error marshalling response: %v", err) - return - } - cfg := config.GetInstance() - - if cfg.TwitterScraper || cfg.DiscordScraper || cfg.TelegramScraper || cfg.WebScraper { - ctx.Respond(&messages.Response{RequestId: workData["request_id"], Value: string(jsn)}) - } - for _, pid := range getPeers(node) { - ctx.Send(pid, &messages.Response{RequestId: workData["request_id"], Value: string(jsn)}) - } - } - ctx.Poison(ctx.Self()) -} diff --git a/pkg/workers/handlers/discord.go b/pkg/workers/handlers/discord.go new file mode 100644 index 00000000..4dfd3d14 --- /dev/null +++ b/pkg/workers/handlers/discord.go @@ -0,0 +1,89 @@ +package handlers + +import ( + "fmt" + + "github.com/sirupsen/logrus" + + "github.com/masa-finance/masa-oracle/pkg/scrapers/discord" + "github.com/masa-finance/masa-oracle/pkg/workers/types" +) + +type DiscordProfileHandler struct{} +type DiscordChannelHandler struct{} +type DiscordSentimentHandler struct{} +type DiscordGuildHandler struct{} +type DiscoreUserGuildsHandler struct{} + +// HandleWork implements the WorkHandler interface for DiscordProfileHandler. +func (h *DiscordProfileHandler) HandleWork(data []byte) data_types.WorkResponse { + logrus.Infof("[+] DiscordProfileHandler %s", data) + dataMap, err := JsonBytesToMap(data) + if err != nil { + return data_types.WorkResponse{Error: fmt.Sprintf("unable to parse discord json data: %v", err)} + } + userID := dataMap["userID"].(string) + resp, err := discord.GetUserProfile(userID) + if err != nil { + return data_types.WorkResponse{Data: resp, Error: fmt.Sprintf("unable to get discord user profile: %v", err)} + } + return data_types.WorkResponse{Data: resp} +} + +// HandleWork implements the WorkHandler interface for DiscordChannelHandler. +func (h *DiscordChannelHandler) HandleWork(data []byte) data_types.WorkResponse { + logrus.Infof("[+] DiscordChannelHandler %s", data) + dataMap, err := JsonBytesToMap(data) + if err != nil { + return data_types.WorkResponse{Error: fmt.Sprintf("unable to parse discord json data: %v", err)} + } + channelID := dataMap["channelID"].(string) + limit := dataMap["limit"].(string) + prompt := dataMap["prompt"].(string) + resp, err := discord.GetChannelMessages(channelID, limit, prompt) + if err != nil { + return data_types.WorkResponse{Error: fmt.Sprintf("unable to get discord channel messages: %v", err)} + } + return data_types.WorkResponse{Data: resp} +} + +// HandleWork implements the WorkHandler interface for DiscordSentimentHandler. +func (h *DiscordSentimentHandler) HandleWork(data []byte) data_types.WorkResponse { + logrus.Infof("[+] DiscordSentimentHandler %s", data) + dataMap, err := JsonBytesToMap(data) + if err != nil { + return data_types.WorkResponse{Error: fmt.Sprintf("unable to parse discord json data: %v", err)} + } + channelID := dataMap["channelID"].(string) + model := dataMap["model"].(string) + prompt := dataMap["prompt"].(string) + _, resp, err := discord.ScrapeDiscordMessagesForSentiment(channelID, model, prompt) + if err != nil { + return data_types.WorkResponse{Error: fmt.Sprintf("unable to get discord channel messages: %v", err)} + } + return data_types.WorkResponse{Data: resp} +} + +// HandleWork implements the WorkHandler interface for DiscordGuildHandler. +func (h *DiscordGuildHandler) HandleWork(data []byte) data_types.WorkResponse { + logrus.Infof("[+] DiscordGuildHandler %s", data) + dataMap, err := JsonBytesToMap(data) + if err != nil { + return data_types.WorkResponse{Error: fmt.Sprintf("unable to parse discord json data: %v", err)} + } + guildID := dataMap["guildID"].(string) + resp, err := discord.GetGuildChannels(guildID) + if err != nil { + return data_types.WorkResponse{Error: fmt.Sprintf("unable to get discord guild channels: %v", err)} + } + return data_types.WorkResponse{Data: resp} +} + +func (h *DiscoreUserGuildsHandler) HandleWork(data []byte) data_types.WorkResponse { + logrus.Infof("[+] DiscordUserGuildsHandler %s", data) + resp, err := discord.GetUserGuilds() + if err != nil { + return data_types.WorkResponse{Error: fmt.Sprintf("unable to get discord user guilds: %v", err)} + } + return data_types.WorkResponse{Data: resp} +} diff --git a/pkg/workers/handlers/helper.go b/pkg/workers/handlers/helper.go new file mode 100644 index 00000000..331dd716 --- /dev/null +++ b/pkg/workers/handlers/helper.go @@ -0,0 +1,14 @@ +package handlers + +import ( + "encoding/json" +) + +func JsonBytesToMap(jsonBytes []byte) (map[string]interface{}, error) { + var jsonMap map[string]interface{} + err := json.Unmarshal(jsonBytes, &jsonMap) + if err != nil { + return nil, err + } + return jsonMap, nil +} diff --git a/pkg/workers/client.go b/pkg/workers/handlers/http_client.go similarity index 99% rename from pkg/workers/client.go rename to pkg/workers/handlers/http_client.go index b4e9f16f..50f873b2 100644 --- a/pkg/workers/client.go +++ b/pkg/workers/handlers/http_client.go @@ -1,4 +1,4 @@ -package workers +package handlers import ( "bytes" diff --git a/pkg/workers/handlers/llm.go b/pkg/workers/handlers/llm.go new file mode 100644 index 00000000..658fcf77 --- /dev/null +++ b/pkg/workers/handlers/llm.go @@ -0,0 +1,48 @@ +package handlers + +import ( + "encoding/json" + "fmt" + + "github.com/sirupsen/logrus" + + "github.com/masa-finance/masa-oracle/pkg/config" + "github.com/masa-finance/masa-oracle/pkg/workers/types" +) + +// TODO: LLMChatBody isn't used anywhere in the codebase. Remove after testing +type LLMChatBody struct { + Model string `json:"model,omitempty"` + Messages []struct { + Role string `json:"role"` + Content string `json:"content"` + } `json:"messages,omitempty"` + Stream bool `json:"stream"` +} + +type LLMChatHandler struct{} + +// HandleWork implements the WorkHandler interface for LLMChatHandler. +// It contains the logic for processing LLMChat work. +func (h *LLMChatHandler) HandleWork(data []byte) data_types.WorkResponse { + logrus.Infof("[+] LLM Chat %s", data) + uri := config.GetInstance().LLMChatUrl + if uri == "" { + return data_types.WorkResponse{Error: "missing env variable LLM_CHAT_URL"} + } + + var dataMap map[string]interface{} + if err := json.Unmarshal(data, &dataMap); err != nil { + return data_types.WorkResponse{Error: fmt.Sprintf("unable to parse LLM chat data: %v", err)} + } + + jsnBytes, err := json.Marshal(dataMap) + if err != nil { + return data_types.WorkResponse{Error: fmt.Sprintf("unable to marshal LLM chat data: %v", err)} + } + resp, err := Post(uri, jsnBytes, nil) + if err != nil { + return data_types.WorkResponse{Error: fmt.Sprintf("unable to post LLM chat data: %v", err)} + } + return data_types.WorkResponse{Data: resp} +} diff --git a/pkg/workers/handlers/telegram.go b/pkg/workers/handlers/telegram.go new file mode 100644 index 00000000..234e3ee0 --- /dev/null +++ b/pkg/workers/handlers/telegram.go @@ -0,0 +1,46 @@ +package handlers + +import ( + "context" + "fmt" + + "github.com/sirupsen/logrus" + + "github.com/masa-finance/masa-oracle/pkg/scrapers/telegram" + "github.com/masa-finance/masa-oracle/pkg/workers/types" +) + +type TelegramSentimentHandler struct{} +type TelegramChannelHandler struct{} + +// HandleWork implements the WorkHandler interface for TelegramSentimentHandler. +func (h *TelegramSentimentHandler) HandleWork(data []byte) data_types.WorkResponse { + logrus.Infof("[+] TelegramSentimentHandler %s", data) + dataMap, err := JsonBytesToMap(data) + if err != nil { + return data_types.WorkResponse{Error: fmt.Sprintf("unable to parse telegram json data: %v", err)} + } + userName := dataMap["username"].(string) + model := dataMap["model"].(string) + prompt := dataMap["prompt"].(string) + _, resp, err := telegram.ScrapeTelegramMessagesForSentiment(context.Background(), userName, model, prompt) + if err != nil { + return data_types.WorkResponse{Error: fmt.Sprintf("unable to get telegram sentiment: %v", err)} + } + return data_types.WorkResponse{Data: resp} +} + +// HandleWork implements the WorkHandler interface for TelegramChannelHandler. +func (h *TelegramChannelHandler) HandleWork(data []byte) data_types.WorkResponse { + logrus.Infof("[+] TelegramChannelHandler %s", data) + dataMap, err := JsonBytesToMap(data) + if err != nil { + return data_types.WorkResponse{Error: fmt.Sprintf("unable to parse telegram json data: %v", err)} + } + userName := dataMap["username"].(string) + resp, err := telegram.FetchChannelMessages(context.Background(), userName) + if err != nil { + return data_types.WorkResponse{Error: fmt.Sprintf("unable to get telegram channel messages: %v", err)} + } + return data_types.WorkResponse{Data: resp} +} diff --git a/pkg/workers/handlers/twitter.go b/pkg/workers/handlers/twitter.go new file mode 100644 index 00000000..6db38d2d --- /dev/null +++ b/pkg/workers/handlers/twitter.go @@ -0,0 +1,85 @@ +package handlers + +import ( + "fmt" + + "github.com/sirupsen/logrus" + + "github.com/masa-finance/masa-oracle/pkg/scrapers/twitter" + "github.com/masa-finance/masa-oracle/pkg/workers/types" +) + +type TwitterQueryHandler struct{} +type TwitterFollowersHandler struct{} +type TwitterProfileHandler struct{} +type TwitterSentimentHandler struct{} +type TwitterTrendsHandler struct{} + +func (h *TwitterQueryHandler) HandleWork(data []byte) data_types.WorkResponse { + logrus.Infof("[+] TwitterQueryHandler %s", data) + dataMap, err := JsonBytesToMap(data) + if err != nil { + return data_types.WorkResponse{Error: fmt.Sprintf("unable to parse twitter query data: %v", err)} + } + count := int(dataMap["count"].(float64)) + query := dataMap["query"].(string) + resp, err := twitter.ScrapeTweetsByQuery(query, count) + if err != nil { + return data_types.WorkResponse{Error: fmt.Sprintf("unable to get twitter query: %v", err)} + } + return data_types.WorkResponse{Data: resp} +} + +func (h *TwitterFollowersHandler) HandleWork(data []byte) data_types.WorkResponse { + logrus.Infof("[+] TwitterFollowersHandler %s", data) + dataMap, err := JsonBytesToMap(data) + if err != nil { + return data_types.WorkResponse{Error: fmt.Sprintf("unable to parse twitter followers data: %v", err)} + } + username := dataMap["username"].(string) + count := int(dataMap["count"].(float64)) + resp, err := twitter.ScrapeFollowersForProfile(username, count) + if err != nil { + return data_types.WorkResponse{Error: fmt.Sprintf("unable to get twitter followers: %v", err)} + } + return data_types.WorkResponse{Data: resp} +} + +func (h *TwitterProfileHandler) HandleWork(data []byte) data_types.WorkResponse { + logrus.Infof("[+] TwitterProfileHandler %s", data) + dataMap, err := JsonBytesToMap(data) + if err != nil { + return data_types.WorkResponse{Error: fmt.Sprintf("unable to parse twitter profile data: %v", err)} + } + username := dataMap["username"].(string) + resp, err := twitter.ScrapeTweetsProfile(username) + if err != nil { + return data_types.WorkResponse{Error: fmt.Sprintf("unable to get twitter profile: %v", err)} + } + return data_types.WorkResponse{Data: resp} +} + +func (h *TwitterSentimentHandler) HandleWork(data []byte) data_types.WorkResponse { + logrus.Infof("[+] TwitterSentimentHandler %s", data) + dataMap, err := JsonBytesToMap(data) + if err != nil { + return data_types.WorkResponse{Error: fmt.Sprintf("unable to parse twitter sentiment data: %v", err)} + } + count := int(dataMap["count"].(float64)) + query := dataMap["query"].(string) + model := dataMap["model"].(string) + _, resp, err := twitter.ScrapeTweetsForSentiment(query, count, model) + if err != nil { + return data_types.WorkResponse{Error: fmt.Sprintf("unable to get twitter sentiment: %v", err)} + } + return data_types.WorkResponse{Data: resp} +} + +func (h *TwitterTrendsHandler) HandleWork(data []byte) data_types.WorkResponse { + logrus.Infof("[+] TwitterTrendsHandler %s", data) + resp, err := twitter.ScrapeTweetsByTrends() + if err != nil { + return data_types.WorkResponse{Error: fmt.Sprintf("unable to get twitter trends: %v", err)} + } + return data_types.WorkResponse{Data: resp} +} diff --git a/pkg/workers/handlers/web.go b/pkg/workers/handlers/web.go new file mode 100644 index 00000000..70c2ef3b --- /dev/null +++ b/pkg/workers/handlers/web.go @@ -0,0 +1,45 @@ +package handlers + +import ( + "fmt" + + "github.com/sirupsen/logrus" + + "github.com/masa-finance/masa-oracle/pkg/scrapers/web" + "github.com/masa-finance/masa-oracle/pkg/workers/types" +) + +// WebHandler - All the web handlers implement the WorkHandler interface. +type WebHandler struct{} +type WebSentimentHandler struct{} + +func (h *WebHandler) HandleWork(data []byte) data_types.WorkResponse { + logrus.Infof("[+] WebHandler %s", data) + dataMap, err := JsonBytesToMap(data) + if err != nil { + return data_types.WorkResponse{Error: fmt.Sprintf("unable to parse web data: %v", err)} + } + depth := int(dataMap["depth"].(float64)) + urls := []string{dataMap["url"].(string)} + resp, err := web.ScrapeWebData(urls, depth) + if err != nil { + return data_types.WorkResponse{Error: fmt.Sprintf("unable to get web data: %v", err)} + } + return data_types.WorkResponse{Data: resp} +} + +func (h *WebSentimentHandler) HandleWork(data []byte) data_types.WorkResponse { + logrus.Infof("[+] WebSentimentHandler %s", data) + dataMap, err := JsonBytesToMap(data) + if err != nil { + return data_types.WorkResponse{Error: fmt.Sprintf("unable to parse web sentiment data: %v", err)} + } + depth := int(dataMap["depth"].(float64)) + urls := []string{dataMap["url"].(string)} + model := dataMap["model"].(string) + _, resp, err := web.ScrapeWebDataForSentiment(urls, depth, model) + if err != nil { + return data_types.WorkResponse{Error: fmt.Sprintf("unable to get web sentiment: %v", err)} + } + return data_types.WorkResponse{Data: resp} +} diff --git a/pkg/workers/messages/build.sh b/pkg/workers/messages/build.sh deleted file mode 100755 index 279c821b..00000000 --- a/pkg/workers/messages/build.sh +++ /dev/null @@ -1,2 +0,0 @@ -protoc -I="/Users/john/Projects/masa/protoactor-go/actor" --go_out=. --go_opt=paths=source_relative --proto_path=. protos.proto - diff --git a/pkg/workers/messages/protos.pb.go b/pkg/workers/messages/protos.pb.go deleted file mode 100644 index b7fd1348..00000000 --- a/pkg/workers/messages/protos.pb.go +++ /dev/null @@ -1,374 +0,0 @@ -// Code generated by protoc-gen-go. DO NOT EDIT. -// versions: -// protoc-gen-go v1.33.0 -// protoc v5.26.1 -// source: protos.proto - -package messages - -import ( - actor "github.com/asynkron/protoactor-go/actor" - protoreflect "google.golang.org/protobuf/reflect/protoreflect" - protoimpl "google.golang.org/protobuf/runtime/protoimpl" - reflect "reflect" - sync "sync" -) - -const ( - // Verify that this generated code is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) - // Verify that runtime/protoimpl is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) -) - -type Connect struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Sender *actor.PID `protobuf:"bytes,1,opt,name=Sender,proto3" json:"Sender,omitempty"` -} - -func (x *Connect) Reset() { - *x = Connect{} - if protoimpl.UnsafeEnabled { - mi := &file_protos_proto_msgTypes[0] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *Connect) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*Connect) ProtoMessage() {} - -func (x *Connect) ProtoReflect() protoreflect.Message { - mi := &file_protos_proto_msgTypes[0] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use Connect.ProtoReflect.Descriptor instead. -func (*Connect) Descriptor() ([]byte, []int) { - return file_protos_proto_rawDescGZIP(), []int{0} -} - -func (x *Connect) GetSender() *actor.PID { - if x != nil { - return x.Sender - } - return nil -} - -type Connected struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Message string `protobuf:"bytes,1,opt,name=Message,proto3" json:"Message,omitempty"` -} - -func (x *Connected) Reset() { - *x = Connected{} - if protoimpl.UnsafeEnabled { - mi := &file_protos_proto_msgTypes[1] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *Connected) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*Connected) ProtoMessage() {} - -func (x *Connected) ProtoReflect() protoreflect.Message { - mi := &file_protos_proto_msgTypes[1] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use Connected.ProtoReflect.Descriptor instead. -func (*Connected) Descriptor() ([]byte, []int) { - return file_protos_proto_rawDescGZIP(), []int{1} -} - -func (x *Connected) GetMessage() string { - if x != nil { - return x.Message - } - return "" -} - -type Work struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Sender *actor.PID `protobuf:"bytes,1,opt,name=Sender,proto3" json:"Sender,omitempty"` - Data string `protobuf:"bytes,2,opt,name=Data,proto3" json:"Data,omitempty"` - Id string `protobuf:"bytes,3,opt,name=Id,proto3" json:"Id,omitempty"` - Type int64 `protobuf:"varint,4,opt,name=Type,proto3" json:"Type,omitempty"` -} - -func (x *Work) Reset() { - *x = Work{} - if protoimpl.UnsafeEnabled { - mi := &file_protos_proto_msgTypes[2] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *Work) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*Work) ProtoMessage() {} - -func (x *Work) ProtoReflect() protoreflect.Message { - mi := &file_protos_proto_msgTypes[2] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use Work.ProtoReflect.Descriptor instead. -func (*Work) Descriptor() ([]byte, []int) { - return file_protos_proto_rawDescGZIP(), []int{2} -} - -func (x *Work) GetSender() *actor.PID { - if x != nil { - return x.Sender - } - return nil -} - -func (x *Work) GetData() string { - if x != nil { - return x.Data - } - return "" -} - -func (x *Work) GetId() string { - if x != nil { - return x.Id - } - return "" -} - -func (x *Work) GetType() int64 { - if x != nil { - return x.Type - } - return 0 -} - -type Response struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Value string `protobuf:"bytes,1,opt,name=Value,proto3" json:"Value,omitempty"` - RequestId string `protobuf:"bytes,2,opt,name=RequestId,proto3" json:"RequestId,omitempty"` -} - -func (x *Response) Reset() { - *x = Response{} - if protoimpl.UnsafeEnabled { - mi := &file_protos_proto_msgTypes[3] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *Response) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*Response) ProtoMessage() {} - -func (x *Response) ProtoReflect() protoreflect.Message { - mi := &file_protos_proto_msgTypes[3] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use Response.ProtoReflect.Descriptor instead. -func (*Response) Descriptor() ([]byte, []int) { - return file_protos_proto_rawDescGZIP(), []int{3} -} - -func (x *Response) GetValue() string { - if x != nil { - return x.Value - } - return "" -} - -func (x *Response) GetRequestId() string { - if x != nil { - return x.RequestId - } - return "" -} - -var File_protos_proto protoreflect.FileDescriptor - -var file_protos_proto_rawDesc = []byte{ - 0x0a, 0x0c, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x08, - 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x1a, 0x0b, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x2d, 0x0a, 0x07, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, - 0x12, 0x22, 0x0a, 0x06, 0x53, 0x65, 0x6e, 0x64, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x0a, 0x2e, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x50, 0x49, 0x44, 0x52, 0x06, 0x53, 0x65, - 0x6e, 0x64, 0x65, 0x72, 0x22, 0x25, 0x0a, 0x09, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, - 0x64, 0x12, 0x18, 0x0a, 0x07, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x07, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x62, 0x0a, 0x04, 0x57, - 0x6f, 0x72, 0x6b, 0x12, 0x22, 0x0a, 0x06, 0x53, 0x65, 0x6e, 0x64, 0x65, 0x72, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x0a, 0x2e, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x50, 0x49, 0x44, 0x52, - 0x06, 0x53, 0x65, 0x6e, 0x64, 0x65, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x44, 0x61, 0x74, 0x61, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x44, 0x61, 0x74, 0x61, 0x12, 0x0e, 0x0a, 0x02, 0x49, - 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x49, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x54, - 0x79, 0x70, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x54, 0x79, 0x70, 0x65, 0x22, - 0x3e, 0x0a, 0x08, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x56, - 0x61, 0x6c, 0x75, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x56, 0x61, 0x6c, 0x75, - 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x42, - 0x36, 0x5a, 0x34, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6d, 0x61, - 0x73, 0x61, 0x2d, 0x66, 0x69, 0x6e, 0x61, 0x6e, 0x63, 0x65, 0x2f, 0x6d, 0x61, 0x73, 0x61, 0x2d, - 0x6f, 0x72, 0x61, 0x63, 0x6c, 0x65, 0x2f, 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x73, 0x2f, 0x6d, - 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, -} - -var ( - file_protos_proto_rawDescOnce sync.Once - file_protos_proto_rawDescData = file_protos_proto_rawDesc -) - -func file_protos_proto_rawDescGZIP() []byte { - file_protos_proto_rawDescOnce.Do(func() { - file_protos_proto_rawDescData = protoimpl.X.CompressGZIP(file_protos_proto_rawDescData) - }) - return file_protos_proto_rawDescData -} - -var file_protos_proto_msgTypes = make([]protoimpl.MessageInfo, 4) -var file_protos_proto_goTypes = []interface{}{ - (*Connect)(nil), // 0: messages.Connect - (*Connected)(nil), // 1: messages.Connected - (*Work)(nil), // 2: messages.Work - (*Response)(nil), // 3: messages.Response - (*actor.PID)(nil), // 4: actor.PID -} -var file_protos_proto_depIdxs = []int32{ - 4, // 0: messages.Connect.Sender:type_name -> actor.PID - 4, // 1: messages.Work.Sender:type_name -> actor.PID - 2, // [2:2] is the sub-list for method output_type - 2, // [2:2] is the sub-list for method input_type - 2, // [2:2] is the sub-list for extension type_name - 2, // [2:2] is the sub-list for extension extendee - 0, // [0:2] is the sub-list for field type_name -} - -func init() { file_protos_proto_init() } -func file_protos_proto_init() { - if File_protos_proto != nil { - return - } - if !protoimpl.UnsafeEnabled { - file_protos_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Connect); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_protos_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Connected); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_protos_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Work); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_protos_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Response); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - } - type x struct{} - out := protoimpl.TypeBuilder{ - File: protoimpl.DescBuilder{ - GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: file_protos_proto_rawDesc, - NumEnums: 0, - NumMessages: 4, - NumExtensions: 0, - NumServices: 0, - }, - GoTypes: file_protos_proto_goTypes, - DependencyIndexes: file_protos_proto_depIdxs, - MessageInfos: file_protos_proto_msgTypes, - }.Build() - File_protos_proto = out.File - file_protos_proto_rawDesc = nil - file_protos_proto_goTypes = nil - file_protos_proto_depIdxs = nil -} diff --git a/pkg/workers/messages/protos.proto b/pkg/workers/messages/protos.proto deleted file mode 100644 index 5fd0bc63..00000000 --- a/pkg/workers/messages/protos.proto +++ /dev/null @@ -1,24 +0,0 @@ -syntax = "proto3"; -package messages; -option go_package = "github.com/masa-finance/masa-oracle/workers/messages"; -import "actor.proto"; - -message Connect { - actor.PID Sender = 1; -} - -message Connected { - string Message = 1; -} - -message Work { - actor.PID Sender = 1; - string Data = 2; - string Id = 3; - int64 Type = 4; -} - -message Response { - string Value = 1; - string RequestId = 2; -} \ No newline at end of file diff --git a/pkg/pubsub/response_channel_map.go b/pkg/workers/response_channel_map.go similarity index 64% rename from pkg/pubsub/response_channel_map.go rename to pkg/workers/response_channel_map.go index 89340fd7..78a9f021 100644 --- a/pkg/pubsub/response_channel_map.go +++ b/pkg/workers/response_channel_map.go @@ -1,30 +1,34 @@ -package pubsub +package workers -import "sync" +import ( + "sync" + + "github.com/masa-finance/masa-oracle/pkg/workers/types" +) type ResponseChannelMap struct { mu sync.RWMutex - items map[string]chan []byte + items map[string]chan data_types.WorkResponse } var ( - instance *ResponseChannelMap - once sync.Once + rcmInstance *ResponseChannelMap + rcmOnce sync.Once ) -// GetResponseChannelMap returns the singleton instance of ResponseChannelMap. +// GetResponseChannelMap returns the singleton rcmInstance of ResponseChannelMap. func GetResponseChannelMap() *ResponseChannelMap { - once.Do(func() { - instance = &ResponseChannelMap{ - items: make(map[string]chan []byte), + rcmOnce.Do(func() { + rcmInstance = &ResponseChannelMap{ + items: make(map[string]chan data_types.WorkResponse), } }) - return instance + return rcmInstance } // Set associates the specified value with the specified key in the ResponseChannelMap. // It acquires a write lock to ensure thread-safety while setting the value. -func (drm *ResponseChannelMap) Set(key string, value chan []byte) { +func (drm *ResponseChannelMap) Set(key string, value chan data_types.WorkResponse) { drm.mu.Lock() defer drm.mu.Unlock() drm.items[key] = value @@ -34,7 +38,7 @@ func (drm *ResponseChannelMap) Set(key string, value chan []byte) { // It acquires a read lock to ensure thread-safety while reading the value. // If the key exists in the ResponseChannelMap, it returns the corresponding value and true. // If the key does not exist, it returns nil and false. -func (drm *ResponseChannelMap) Get(key string) (chan []byte, bool) { +func (drm *ResponseChannelMap) Get(key string) (chan data_types.WorkResponse, bool) { drm.mu.RLock() defer drm.mu.RUnlock() value, ok := drm.items[key] @@ -57,8 +61,8 @@ func (drm *ResponseChannelMap) Len() int { return len(drm.items) } -func (drm *ResponseChannelMap) CreateChannel(key string) chan []byte { - ch := make(chan []byte) +func (drm *ResponseChannelMap) CreateChannel(key string) chan data_types.WorkResponse { + ch := make(chan data_types.WorkResponse) drm.Set(key, ch) return ch } diff --git a/pkg/workers/types/request_response.go b/pkg/workers/types/request_response.go new file mode 100644 index 00000000..359c63cb --- /dev/null +++ b/pkg/workers/types/request_response.go @@ -0,0 +1,29 @@ +package data_types + +import ( + "github.com/libp2p/go-libp2p/core/peer" + + masa "github.com/masa-finance/masa-oracle/pkg" + "github.com/masa-finance/masa-oracle/pkg/pubsub" +) + +type Worker struct { + IsLocal bool + IPAddr string + AddrInfo *peer.AddrInfo + NodeData pubsub.NodeData + Node *masa.OracleNode +} + +type WorkRequest struct { + WorkType WorkerType `json:"workType,omitempty"` + RequestId string `json:"requestId,omitempty"` + Data []byte `json:"data,omitempty"` +} + +type WorkResponse struct { + WorkRequest *WorkRequest `json:"workRequest,omitempty"` + Data interface{} `json:"data,omitempty"` + Error string `json:"error,omitempty"` + WorkerPeerId string `json:"workerPeerId,omitempty"` +} diff --git a/pkg/workers/types/work_types.go b/pkg/workers/types/work_types.go new file mode 100644 index 00000000..e84fe439 --- /dev/null +++ b/pkg/workers/types/work_types.go @@ -0,0 +1,51 @@ +package data_types + +import ( + "github.com/sirupsen/logrus" + + "github.com/masa-finance/masa-oracle/pkg/pubsub" +) + +type WorkerType string + +const ( + Discord WorkerType = "discord" + DiscordProfile WorkerType = "discord-profile" + DiscordChannelMessages WorkerType = "discord-channel-messages" + DiscordSentiment WorkerType = "discord-sentiment" + TelegramSentiment WorkerType = "telegram-sentiment" + TelegramChannelMessages WorkerType = "telegram-channel-messages" + DiscordGuildChannels WorkerType = "discord-guild-channels" + DiscordUserGuilds WorkerType = "discord-user-guilds" + LLMChat WorkerType = "llm-chat" + Twitter WorkerType = "twitter" + TwitterFollowers WorkerType = "twitter-followers" + TwitterProfile WorkerType = "twitter-profile" + TwitterSentiment WorkerType = "twitter-sentiment" + TwitterTrends WorkerType = "twitter-trends" + Web WorkerType = "web" + WebSentiment WorkerType = "web-sentiment" + Test WorkerType = "test" +) + +// WorkerTypeToCategory maps WorkerType to WorkerCategory +func WorkerTypeToCategory(wt WorkerType) pubsub.WorkerCategory { + logrus.Infof("Mapping WorkerType %s to WorkerCategory", wt) + switch wt { + case Discord, DiscordProfile, DiscordChannelMessages, DiscordSentiment, DiscordGuildChannels, DiscordUserGuilds: + logrus.Info("WorkerType is related to Discord") + return pubsub.CategoryDiscord + case TelegramSentiment, TelegramChannelMessages: + logrus.Info("WorkerType is related to Telegram") + return pubsub.CategoryTelegram + case Twitter, TwitterFollowers, TwitterProfile, TwitterSentiment, TwitterTrends: + logrus.Info("WorkerType is related to Twitter") + return pubsub.CategoryTwitter + case Web, WebSentiment: + logrus.Info("WorkerType is related to Web") + return pubsub.CategoryWeb + default: + logrus.Warn("WorkerType is invalid or not recognized") + return -1 // Invalid category + } +} diff --git a/pkg/workers/worker_manager.go b/pkg/workers/worker_manager.go new file mode 100644 index 00000000..d4966cc6 --- /dev/null +++ b/pkg/workers/worker_manager.go @@ -0,0 +1,299 @@ +package workers + +import ( + "context" + "encoding/binary" + "encoding/json" + "errors" + "fmt" + "io" + "sync" + "time" + + "github.com/libp2p/go-libp2p/core/network" + "github.com/sirupsen/logrus" + + masa "github.com/masa-finance/masa-oracle/pkg" + "github.com/masa-finance/masa-oracle/pkg/config" + "github.com/masa-finance/masa-oracle/pkg/workers/handlers" + data_types "github.com/masa-finance/masa-oracle/pkg/workers/types" +) + +var ( + instance *WorkHandlerManager + once sync.Once +) + +func GetWorkHandlerManager() *WorkHandlerManager { + once.Do(func() { + instance = &WorkHandlerManager{ + handlers: make(map[data_types.WorkerType]*WorkHandlerInfo), + } + instance.setupHandlers() + }) + return instance +} + +// ErrHandlerNotFound is an error returned when a work handler cannot be found. +var ErrHandlerNotFound = errors.New("work handler not found") + +// WorkHandler defines the interface for handling different types of work. +type WorkHandler interface { + HandleWork(data []byte) data_types.WorkResponse +} + +// WorkHandlerInfo contains information about a work handler, including metrics. +type WorkHandlerInfo struct { + Handler WorkHandler + CallCount int64 + TotalRuntime time.Duration +} + +// WorkHandlerManager manages work handlers and tracks their execution metrics. +type WorkHandlerManager struct { + handlers map[data_types.WorkerType]*WorkHandlerInfo + mu sync.RWMutex +} + +func (whm *WorkHandlerManager) setupHandlers() { + cfg := config.GetInstance() + if cfg.TwitterScraper { + whm.addWorkHandler(data_types.Twitter, &handlers.TwitterQueryHandler{}) + whm.addWorkHandler(data_types.TwitterFollowers, &handlers.TwitterFollowersHandler{}) + whm.addWorkHandler(data_types.TwitterProfile, &handlers.TwitterProfileHandler{}) + whm.addWorkHandler(data_types.TwitterSentiment, &handlers.TwitterSentimentHandler{}) + whm.addWorkHandler(data_types.TwitterTrends, &handlers.TwitterTrendsHandler{}) + } + if cfg.WebScraper { + whm.addWorkHandler(data_types.Web, &handlers.WebHandler{}) + whm.addWorkHandler(data_types.WebSentiment, &handlers.WebSentimentHandler{}) + } + if cfg.LlmServer { + whm.addWorkHandler(data_types.LLMChat, &handlers.LLMChatHandler{}) + } + if cfg.DiscordScraper { + whm.addWorkHandler(data_types.Discord, &handlers.DiscordProfileHandler{}) + } +} + +// addWorkHandler registers a new work handler under a specific name. +func (whm *WorkHandlerManager) addWorkHandler(wType data_types.WorkerType, handler WorkHandler) { + whm.mu.Lock() + defer whm.mu.Unlock() + whm.handlers[wType] = &WorkHandlerInfo{Handler: handler} +} + +// getWorkHandler retrieves a registered work handler by name. +func (whm *WorkHandlerManager) getWorkHandler(wType data_types.WorkerType) (WorkHandler, bool) { + whm.mu.RLock() + defer whm.mu.RUnlock() + info, exists := whm.handlers[wType] + if !exists { + return nil, false + } + return info.Handler, true +} + +func (whm *WorkHandlerManager) DistributeWork(node *masa.OracleNode, workRequest data_types.WorkRequest) (response data_types.WorkResponse) { + category := data_types.WorkerTypeToCategory(workRequest.WorkType) + remoteWorkers, localWorker := GetEligibleWorkers(node, category, workerConfig) + + remoteWorkersAttempted := 0 + logrus.Info("Starting round-robin worker selection") + + // Try remote workers first, up to MaxRemoteWorkers + for _, worker := range remoteWorkers { + if remoteWorkersAttempted >= workerConfig.MaxRemoteWorkers { + logrus.Infof("Reached maximum remote workers (%d), stopping remote worker attempts", workerConfig.MaxRemoteWorkers) + break + } + remoteWorkersAttempted++ + logrus.Infof("Attempting remote worker %s (attempt %d/%d)", worker.NodeData.PeerId, remoteWorkersAttempted, workerConfig.MaxRemoteWorkers) + response = whm.sendWorkToWorker(node, worker, workRequest) + if response.Error != "" { + logrus.Errorf("error sending work to worker: %s: %s", response.WorkerPeerId, response.Error) + logrus.Infof("Remote worker %s failed, moving to next worker", worker.NodeData.PeerId) + continue + } + return response + } + // Fallback to local execution if local worker is eligible + if localWorker != nil { + return whm.ExecuteWork(workRequest) + } + if response.Error == "" { + response.Error = "no eligible workers found" + } else { + response.Error = fmt.Sprintf("no workers could process: remote attempt failed due to: %s", response.Error) + } + return response +} + +func (whm *WorkHandlerManager) sendWorkToWorker(node *masa.OracleNode, worker data_types.Worker, workRequest data_types.WorkRequest) (response data_types.WorkResponse) { + ctxWithTimeout, cancel := context.WithTimeout(context.Background(), workerConfig.WorkerResponseTimeout) + defer cancel() // Cancel the context when done to release resources + + if err := node.Host.Connect(ctxWithTimeout, *worker.AddrInfo); err != nil { + response.Error = fmt.Sprintf("failed to connect to remote peer %s: %v", worker.AddrInfo.ID.String(), err) + return + } else { + logrus.Debugf("[+] Connection established with node: %s", worker.AddrInfo.ID.String()) + stream, err := node.Host.NewStream(ctxWithTimeout, worker.AddrInfo.ID, config.ProtocolWithVersion(config.WorkerProtocol)) + if err != nil { + response.Error = fmt.Sprintf("error opening stream: %v", err) + return + } + // the stream should be closed by the receiver, but keeping this here just in case + defer func(stream network.Stream) { + err := stream.Close() + if err != nil { + logrus.Debugf("[-] Error closing stream: %s", err) + } + }(stream) // Close the stream when done + + // Write the request to the stream with length prefix + bytes, err := json.Marshal(workRequest) + if err != nil { + response.Error = fmt.Sprintf("error marshaling work request: %v", err) + return + } + lengthBuf := make([]byte, 4) + binary.BigEndian.PutUint32(lengthBuf, uint32(len(bytes))) + _, err = stream.Write(lengthBuf) + if err != nil { + response.Error = fmt.Sprintf("error writing length to stream: %v", err) + return + } + _, err = stream.Write(bytes) + if err != nil { + response.Error = fmt.Sprintf("error writing to stream: %v", err) + return + } + + // Read the response length + lengthBuf = make([]byte, 4) + _, err = io.ReadFull(stream, lengthBuf) + if err != nil { + response.Error = fmt.Sprintf("error reading response length: %v", err) + return + } + responseLength := binary.BigEndian.Uint32(lengthBuf) + + // Read the actual response + responseBuf := make([]byte, responseLength) + _, err = io.ReadFull(stream, responseBuf) + if err != nil { + response.Error = fmt.Sprintf("error reading response: %v", err) + return + } + err = json.Unmarshal(responseBuf, &response) + if err != nil { + response.Error = fmt.Sprintf("error unmarshaling response: %v", err) + return + } + } + return response +} + +// ExecuteWork finds and executes the work handler associated with the given name. +// It tracks the call count and execution duration for the handler. +func (whm *WorkHandlerManager) ExecuteWork(workRequest data_types.WorkRequest) (response data_types.WorkResponse) { + handler, exists := whm.getWorkHandler(workRequest.WorkType) + if !exists { + return data_types.WorkResponse{Error: ErrHandlerNotFound.Error()} + } + + // Create a context with a 30-second timeout + ctx, cancel := context.WithTimeout(context.Background(), workerConfig.WorkerResponseTimeout) + defer cancel() + + // Channel to receive the work response + responseChan := make(chan data_types.WorkResponse, 1) + + // Execute the work in a separate goroutine + go func() { + startTime := time.Now() + workResponse := handler.HandleWork(workRequest.Data) + if workResponse.Error == "" { + duration := time.Since(startTime) + whm.mu.Lock() + handlerInfo := whm.handlers[workRequest.WorkType] + handlerInfo.CallCount++ + handlerInfo.TotalRuntime += duration + whm.mu.Unlock() + } + responseChan <- workResponse + }() + + select { + case <-ctx.Done(): + // Context timed out + return data_types.WorkResponse{Error: "work execution timed out"} + case response = <-responseChan: + // Work completed within the timeout + return response + } +} + +func (whm *WorkHandlerManager) HandleWorkerStream(stream network.Stream) { + defer func(stream network.Stream) { + err := stream.Close() + if err != nil { + logrus.Errorf("[-] Error closing stream in handler: %s", err) + } + }(stream) + + // Read the length of the message + lengthBuf := make([]byte, 4) + _, err := io.ReadFull(stream, lengthBuf) + if err != nil { + logrus.Errorf("error reading message length: %v", err) + return + } + messageLength := binary.BigEndian.Uint32(lengthBuf) + + // Read the actual message + messageBuf := make([]byte, messageLength) + _, err = io.ReadFull(stream, messageBuf) + if err != nil { + logrus.Errorf("error reading message: %v", err) + return + } + + var workRequest data_types.WorkRequest + err = json.Unmarshal(messageBuf, &workRequest) + if err != nil { + logrus.Errorf("error unmarshaling work request: %v", err) + return + } + peerId := stream.Conn().LocalPeer().String() + workResponse := whm.ExecuteWork(workRequest) + if workResponse.Error != "" { + logrus.Errorf("error from remote worker %s: executing work: %s", peerId, workResponse.Error) + } + workResponse.WorkerPeerId = peerId + + // Write the response to the stream + responseBytes, err := json.Marshal(workResponse) + if err != nil { + logrus.Errorf("error marshaling work response: %v", err) + return + } + + // Prefix the response with its length + responseLength := uint32(len(responseBytes)) + lengthBuf = make([]byte, 4) + binary.BigEndian.PutUint32(lengthBuf, responseLength) + + _, err = stream.Write(lengthBuf) + if err != nil { + logrus.Errorf("error writing response length to stream: %v", err) + return + } + + _, err = stream.Write(responseBytes) + if err != nil { + logrus.Errorf("error writing response to stream: %v", err) + return + } +} diff --git a/pkg/workers/worker_selection.go b/pkg/workers/worker_selection.go new file mode 100644 index 00000000..9e1d6dcb --- /dev/null +++ b/pkg/workers/worker_selection.go @@ -0,0 +1,67 @@ +package workers + +import ( + "context" + "math/rand/v2" + "time" + + "github.com/libp2p/go-libp2p/core/peer" + "github.com/multiformats/go-multiaddr" + "github.com/sirupsen/logrus" + + masa "github.com/masa-finance/masa-oracle/pkg" + "github.com/masa-finance/masa-oracle/pkg/pubsub" + "github.com/masa-finance/masa-oracle/pkg/workers/types" +) + +// GetEligibleWorkers Uses the new NodeTracker method to get the eligible workers for a given message type +// I'm leaving this returning an array so that we can easily increase the number of workers in the future +func GetEligibleWorkers(node *masa.OracleNode, category pubsub.WorkerCategory, config *WorkerConfig) ([]data_types.Worker, *data_types.Worker) { + + var workers []data_types.Worker + nodes := node.NodeTracker.GetEligibleWorkerNodes(category) + var localWorker *data_types.Worker + + // Shuffle the node list first to avoid always selecting the same node + rand.Shuffle(len(nodes), func(i, j int) { + nodes[i], nodes[j] = nodes[j], nodes[i] + }) + + logrus.Info("checking connections to eligible workers") + start := time.Now() + for _, eligible := range nodes { + if eligible.PeerId.String() == node.Host.ID().String() { + localWorker = &data_types.Worker{IsLocal: true, NodeData: eligible} + continue + } + addr, err := multiaddr.NewMultiaddr(eligible.MultiaddrsString) + if err != nil { + logrus.Errorf("error creating multiaddress: %s", err.Error()) + continue + } + peerInfo, err := peer.AddrInfoFromP2pAddr(addr) + if err != nil { + logrus.Errorf("Failed to get peer info: %s", err) + continue + } + ctxWithTimeout, cancel := context.WithTimeout(context.Background(), config.ConnectionTimeout) + defer cancel() // Cancel the context when done to release resources + if err := node.Host.Connect(ctxWithTimeout, *peerInfo); err != nil { + logrus.Debugf("Failed to connect to peer: %v", err) + continue + } + workers = append(workers, data_types.Worker{IsLocal: false, NodeData: eligible, AddrInfo: peerInfo}) + // print duration of worker selection in seconds with floating point precision + dur := time.Since(start).Milliseconds() + logrus.Infof("Worker selection took %v milliseconds", dur) + break + } + // make sure we get the local node in the list + if localWorker == nil { + nd := node.NodeTracker.GetNodeData(node.Host.ID().String()) + if nd.CanDoWork(category) { + localWorker = &data_types.Worker{IsLocal: true, NodeData: *nd} + } + } + return workers, localWorker +} diff --git a/pkg/workers/workers.go b/pkg/workers/workers.go deleted file mode 100644 index f7f862e4..00000000 --- a/pkg/workers/workers.go +++ /dev/null @@ -1,491 +0,0 @@ -package workers - -import ( - "context" - "encoding/json" - "fmt" - "sync" - "time" - - "github.com/libp2p/go-libp2p/core/peer" - - masa "github.com/masa-finance/masa-oracle/pkg" - "github.com/masa-finance/masa-oracle/pkg/config" - "github.com/masa-finance/masa-oracle/pkg/db" - "github.com/masa-finance/masa-oracle/pkg/pubsub" - "github.com/masa-finance/masa-oracle/pkg/workers/messages" - - "github.com/multiformats/go-multiaddr" - - "github.com/asynkron/protoactor-go/actor" - - "github.com/ipfs/go-cid" - - pubsub2 "github.com/libp2p/go-libp2p-pubsub" - mh "github.com/multiformats/go-multihash" - "github.com/sirupsen/logrus" -) - -type WorkerType string - -const ( - Discord WorkerType = "discord" - DiscordProfile WorkerType = "discord-profile" - DiscordChannelMessages WorkerType = "discord-channel-messages" - DiscordSentiment WorkerType = "discord-sentiment" - TelegramSentiment WorkerType = "telegram-sentiment" - TelegramChannelMessages WorkerType = "telegram-channel-messages" - DiscordGuildChannels WorkerType = "discord-guild-channels" - DiscordUserGuilds WorkerType = "discord-user-guilds" - LLMChat WorkerType = "llm-chat" - Twitter WorkerType = "twitter" - TwitterFollowers WorkerType = "twitter-followers" - TwitterProfile WorkerType = "twitter-profile" - TwitterSentiment WorkerType = "twitter-sentiment" - TwitterTrends WorkerType = "twitter-trends" - Web WorkerType = "web" - WebSentiment WorkerType = "web-sentiment" - Test WorkerType = "test" -) - -var WORKER = struct { - Discord, DiscordProfile, DiscordChannelMessages, DiscordSentiment, TelegramSentiment, TelegramChannelMessages, DiscordGuildChannels, DiscordUserGuilds, LLMChat, Twitter, TwitterFollowers, TwitterProfile, TwitterSentiment, TwitterTrends, Web, WebSentiment, Test WorkerType -}{ - Discord: Discord, - DiscordProfile: DiscordProfile, - DiscordChannelMessages: DiscordChannelMessages, - DiscordSentiment: DiscordSentiment, - TelegramSentiment: TelegramSentiment, - TelegramChannelMessages: TelegramChannelMessages, - DiscordGuildChannels: DiscordGuildChannels, - DiscordUserGuilds: DiscordUserGuilds, - LLMChat: LLMChat, - Twitter: Twitter, - TwitterFollowers: TwitterFollowers, - TwitterProfile: TwitterProfile, - TwitterSentiment: TwitterSentiment, - TwitterTrends: TwitterTrends, - Web: Web, - WebSentiment: WebSentiment, - Test: Test, -} - -var ( - clients = actor.NewPIDSet() - workerStatusCh = make(chan *pubsub2.Message) - workerDoneCh = make(chan *pubsub2.Message) -) - -// WorkerTypeToCategory maps WorkerType to WorkerCategory -func WorkerTypeToCategory(wt WorkerType) pubsub.WorkerCategory { - switch wt { - case Discord, DiscordProfile, DiscordChannelMessages, DiscordSentiment, DiscordGuildChannels, DiscordUserGuilds: - return pubsub.CategoryDiscord - case TelegramSentiment, TelegramChannelMessages: - return pubsub.CategoryTelegram - case Twitter, TwitterFollowers, TwitterProfile, TwitterSentiment, TwitterTrends: - return pubsub.CategoryTwitter - case Web, WebSentiment: - return pubsub.CategoryWeb - default: - return -1 // Invalid category - } -} - -type ChanResponse struct { - Response map[string]interface{} - ChannelId string -} - -type Worker struct { - Node *masa.OracleNode -} - -// NewWorker creates a new instance of the Worker actor. -// It implements the actor.Receiver interface, allowing it to receive and handle messages. -// -// Returns: -// - An instance of the Worker struct that implements the actor.Receiver interface. -func NewWorker(node *masa.OracleNode) actor.Producer { - return func() actor.Actor { - return &Worker{Node: node} - } -} - -// Receive is the message handling method for the Worker actor. -// It receives messages through the actor context and processes them based on their type. -func (a *Worker) Receive(ctx actor.Context) { - switch m := ctx.Message().(type) { - case *messages.Connect: - a.HandleConnect(ctx, m) - case *actor.Started: - if a.Node.IsWorker() { - a.HandleLog(ctx, "[+] Actor started") - } - case *actor.Stopping: - if a.Node.IsWorker() { - a.HandleLog(ctx, "[+] Actor stopping") - } - case *actor.Stopped: - if a.Node.IsWorker() { - a.HandleLog(ctx, "[+] Actor stopped") - } - case *messages.Work: - if a.Node.IsWorker() { - logrus.Infof("[+] Received Work") - a.HandleWork(ctx, m, a.Node) - } - case *messages.Response: - logrus.Infof("[+] Received Response") - msg := &pubsub2.Message{} - err := json.Unmarshal([]byte(m.Value), msg) - if err != nil { - msg, err = getResponseMessage(m) - if err != nil { - logrus.Errorf("[-] Error getting response message: %v", err) - return - } - } - workerDoneCh <- msg - ctx.Poison(ctx.Self()) - default: - logrus.Warningf("[+] Received unknown message in workers: %T, message: %+v", m, m) - } -} - -// computeCid calculates the CID (Content Identifier) for a given string. -// -// Parameters: -// - str: The input string for which to compute the CID. -// -// Returns: -// - string: The computed CID as a string. -// - error: An error, if any occurred during the CID computation. -// -// The function uses the multihash package to create a SHA2-256 hash of the input string. -// It then creates a CID (version 1) from the multihash and returns the CID as a string. -// If an error occurs during the multihash computation or CID creation, it is returned. -func computeCid(str string) (string, error) { - // Create a multihash from the string - mhHash, err := mh.Sum([]byte(str), mh.SHA2_256, -1) - if err != nil { - return "", err - } - // Create a CID from the multihash - cidKey := cid.NewCidV1(cid.Raw, mhHash).String() - return cidKey, nil -} - -// getResponseMessage converts a messages.Response object into a pubsub2.Message object. -// It unmarshals the JSON-encoded response value into a map and then constructs a new pubsub2.Message -// using the extracted data. -// -// Parameters: -// - response: A pointer to a messages.Response object containing the JSON-encoded response data. -// -// Returns: -// - A pointer to a pubsub2.Message object constructed from the response data. -// - An error if there is an issue with unmarshalling the response data. -func getResponseMessage(response *messages.Response) (*pubsub2.Message, error) { - responseData := map[string]interface{}{} - - err := json.Unmarshal([]byte(response.Value), &responseData) - if err != nil { - return nil, err - } - msg := &pubsub2.Message{ - ID: responseData["ID"].(string), - ReceivedFrom: peer.ID(responseData["ReceivedFrom"].(string)), - ValidatorData: responseData["ValidatorData"], - Local: responseData["Local"].(bool), - } - return msg, nil -} - -// SendWork is a function that sends work to a node. It takes two parameters: -// node: A pointer to a masa.OracleNode object. This is the node to which the work will be sent. -// m: A pointer to a pubsub2.Message object. This is the message that contains the work to be sent. -func SendWork(node *masa.OracleNode, m *pubsub2.Message) { - var wg sync.WaitGroup - props := actor.PropsFromProducer(NewWorker(node)) - pid := node.ActorEngine.Spawn(props) - message := &messages.Work{Data: string(m.Data), Sender: pid, Id: m.ReceivedFrom.String(), Type: int64(pubsub.CategoryTwitter)} - n := 0 - - responseCollector := make(chan *pubsub2.Message, 100) // Buffered channel to collect responses - timeout := time.After(8 * time.Second) - - // Local worker - if node.IsStaked && node.IsWorker() { - wg.Add(1) - go func() { - defer wg.Done() - future := node.ActorEngine.RequestFuture(pid, message, 60*time.Second) // Increase timeout from 30 to 60 seconds - result, err := future.Result() - if err != nil { - logrus.Errorf("[-] Error receiving response from local worker: %v", err) - responseCollector <- &pubsub2.Message{ - ValidatorData: map[string]interface{}{"error": err.Error()}, - } - return - } - response := result.(*messages.Response) - msg := &pubsub2.Message{} - rErr := json.Unmarshal([]byte(response.Value), msg) - if rErr != nil { - gMsg, gErr := getResponseMessage(result.(*messages.Response)) - if gErr != nil { - logrus.Errorf("[-] Error getting response message: %v", gErr) - responseCollector <- &pubsub2.Message{ - ValidatorData: map[string]interface{}{"error": gErr.Error()}, - } - return - } - msg = gMsg - } - responseCollector <- msg - n++ - }() - } - - // Remote workers - peers := node.NodeTracker.GetAllNodeData() - for _, p := range peers { - for _, addr := range p.Multiaddrs { - ipAddr, _ := addr.ValueForProtocol(multiaddr.P_IP4) - if (p.PeerId.String() != node.Host.ID().String()) && - p.IsStaked && - node.NodeTracker.GetNodeData(p.PeerId.String()).CanDoWork(pubsub.WorkerCategory(message.Type)) { - logrus.Infof("[+] Worker Address: %s", ipAddr) - wg.Add(1) - go func(p pubsub.NodeData) { - defer wg.Done() - spawned, err := node.ActorRemote.SpawnNamed(fmt.Sprintf("%s:4001", ipAddr), "worker", "peer", -1) - if err != nil { - logrus.Debugf("[-] Error spawning remote worker: %v", err) - responseCollector <- &pubsub2.Message{ - ValidatorData: map[string]interface{}{"error": err.Error()}, - } - return - } - spawnedPID := spawned.Pid - logrus.Infof("[+] Worker Address: %s", spawnedPID) - if spawnedPID == nil { - logrus.Errorf("[-] Spawned PID is nil for IP: %s", ipAddr) - responseCollector <- &pubsub2.Message{ - ValidatorData: map[string]interface{}{"error": "Spawned PID is nil"}, - } - return - } - client := node.ActorEngine.Spawn(props) - node.ActorEngine.Send(spawnedPID, &messages.Connect{Sender: client}) - future := node.ActorEngine.RequestFuture(spawnedPID, message, 30*time.Second) - result, fErr := future.Result() - if fErr != nil { - logrus.Debugf("[-] Error receiving response from remote worker: %v", fErr) - responseCollector <- &pubsub2.Message{ - ValidatorData: map[string]interface{}{"error": fErr.Error()}, - } - return - } - response := result.(*messages.Response) - msg := &pubsub2.Message{} - rErr := json.Unmarshal([]byte(response.Value), &msg) - if rErr != nil { - gMsg, gErr := getResponseMessage(response) - if gErr != nil { - logrus.Errorf("[-] Error getting response message: %v", gErr) - responseCollector <- &pubsub2.Message{ - ValidatorData: map[string]interface{}{"error": gErr.Error()}, - } - return - } - if gMsg != nil { - msg = gMsg - } - } - responseCollector <- msg - n++ - // cap at 3 for performance - if n == len(peers) || n == 3 { - logrus.Info("[+] All workers have responded") - responseCollector <- msg - } - }(p) - } - } - } - - // Queue responses and send to workerDoneCh - go func() { - var responses []*pubsub2.Message - for { - select { - case response := <-responseCollector: - responses = append(responses, response) - case <-timeout: - for _, resp := range responses { - workerDoneCh <- resp - } - return - } - } - }() - - wg.Wait() -} - -// MonitorWorkers monitors worker data by subscribing to the completed work topic, -// computing a CID for each received data, and writing the data to the database. -// -// Parameters: -// - ctx: The context for the monitoring operation. -// - node: A pointer to the OracleNode instance. -// -// The function uses a ticker to periodically log a debug message every 60 seconds. -// It subscribes to the completed work topic using the PubSubManager and handles the received data. -// For each received data, it computes a CID using the computeCid function, logs the CID, -// marshals the data to JSON, and writes it to the database using the WriteData function. -// The monitoring continues until the context is done. -func MonitorWorkers(ctx context.Context, node *masa.OracleNode) { - node.WorkerTracker = &pubsub.WorkerEventTracker{WorkerStatusCh: workerStatusCh} - err := node.PubSubManager.AddSubscription(config.TopicWithVersion(config.WorkerTopic), node.WorkerTracker, true) - if err != nil { - logrus.Errorf("[-] Subscribe error %v", err) - } - - // Register self as a remote node for the network - node.ActorRemote.Register("peer", actor.PropsFromProducer(NewWorker(node))) - - if node.WorkerTracker == nil { - logrus.Error("[-] MonitorWorkers: WorkerTracker is nil") - return - } - - if node.WorkerTracker.WorkerStatusCh == nil { - logrus.Error("[-] MonitorWorkers: WorkerStatusCh is nil") - return - } - - ticker := time.NewTicker(time.Second * 15) - defer ticker.Stop() - rcm := pubsub.GetResponseChannelMap() - var startTime time.Time - - for { - select { - case work := <-node.WorkerTracker.WorkerStatusCh: - logrus.Info("[+] Sending work to network") - var workData map[string]string - - err := json.Unmarshal(work.Data, &workData) - if err != nil { - logrus.Error("[-] Error unmarshalling work: ", err) - continue - } - startTime = time.Now() - go SendWork(node, work) - case data := <-workerDoneCh: - validatorDataMap, ok := data.ValidatorData.(map[string]interface{}) - if !ok { - logrus.Errorf("[-] Error asserting type: %v", ok) - continue - } - - if ch, ok := rcm.Get(validatorDataMap["ChannelId"].(string)); ok { - validatorData, err := json.Marshal(validatorDataMap["Response"]) - if err != nil { - logrus.Errorf("[-] Error marshalling data.ValidatorData: %v", err) - continue - } - ch <- validatorData - defer close(ch) - } else { - logrus.Debugf("Error processing data.ValidatorData: %v", data.ValidatorData) - continue - } - - processValidatorData(data, validatorDataMap, &startTime, node) - - case <-ticker.C: - logrus.Info("[+] worker tick") - - case <-ctx.Done(): - return - } - } -} - -/** - * Processes the validator data received from the network. - * - * @param {pubsub2.Message} data - The message data received from the network. - * @param {map[string]interface{}} validatorDataMap - The map containing validator data. - * @param {time.Time} startTime - The start time of the work. - * @param {masa.OracleNode} node - The OracleNode instance. - */ -func processValidatorData(data *pubsub2.Message, validatorDataMap map[string]interface{}, startTime *time.Time, node *masa.OracleNode) { - //logrus.Infof("[+] Work validatorDataMap %s", validatorDataMap) - if response, ok := validatorDataMap["Response"].(map[string]interface{}); ok { - if _, ok := response["error"].(string); ok { - logrus.Infof("[+] Work failed %s", response["error"]) - - // Set WorkerTimeout for the node - nodeData := node.NodeTracker.GetNodeData(data.ReceivedFrom.String()) - if nodeData != nil { - nodeData.WorkerTimeout = time.Now() - node.NodeTracker.AddOrUpdateNodeData(nodeData, true) - } - - } else if work, ok := response["data"].(string); ok { - processWork(data, work, startTime, node) - - } else if w, ok := response["data"].(map[string]interface{}); ok { - work, err := json.Marshal(w) - - if err != nil { - logrus.Errorf("[-] Error marshalling data.ValidatorData: %v", err) - return - } - - processWork(data, string(work), startTime, node) - } else { - work, err := json.Marshal(response["data"]) - if err != nil { - logrus.Errorf("[-] Error marshalling data.ValidatorData: %v", err) - return - } - - processWork(data, string(work), startTime, node) - - } - } -} - -/** - * Processes the work received from the network. - * - * @param {pubsub2.Message} data - The message data received from the network. - * @param {string} work - The work data as a string. - * @param {time.Time} startTime - The start time of the work. - * @param {masa.OracleNode} node - The OracleNode instance. - */ -func processWork(data *pubsub2.Message, work string, startTime *time.Time, node *masa.OracleNode) { - key, _ := computeCid(work) - logrus.Infof("[+] Work done %s", key) - - endTime := time.Now() - duration := endTime.Sub(*startTime) - - workEvent := db.WorkEvent{ - CID: key, - PeerId: data.ID, - Payload: []byte(work), - Duration: duration.Seconds(), - Timestamp: time.Now().Unix(), - } - logrus.Infof("[+] Publishing work event : %v for Peer %s", workEvent.CID, workEvent.PeerId) - logrus.Debugf("[+] Publishing work event : %v", workEvent) - - _ = node.PubSubManager.Publish(config.TopicWithVersion(config.BlockTopic), workEvent.Payload) -}