From 9a6ebb5328a63cd42a35f3e157a0ae137b4f5d86 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Thu, 15 Aug 2024 18:56:38 +0200 Subject: [PATCH 01/10] feat(versioning): split application and protocol versioning (#505) * feat(versioning): split application and protocol versioning Signed-off-by: mudler * be consistent with the Makefile Signed-off-by: mudler * move versioning into internal/ There is no need to expose versioning and let it be imported outside of the oracle code Signed-off-by: mudler --------- Signed-off-by: mudler --- Makefile | 4 ++-- cmd/masa-node-cli/components.go | 5 +++-- cmd/masa-node/main.go | 5 +++-- internal/versioning/version.go | 9 +++++++++ pkg/config/app.go | 4 +++- pkg/config/version.go | 5 ----- pkg/config/welcome.go | 5 +++-- pkg/oracle_node.go | 3 ++- pkg/pubsub/node_data.go | 3 ++- 9 files changed, 27 insertions(+), 16 deletions(-) create mode 100644 internal/versioning/version.go delete mode 100644 pkg/config/version.go diff --git a/Makefile b/Makefile index 29b9ba4c..117222e3 100644 --- a/Makefile +++ b/Makefile @@ -4,8 +4,8 @@ print-version: @echo "Version: ${VERSION}" build: - @go build -v -ldflags "-X github.com/masa-finance/masa-oracle/pkg/config.Version=${VERSION}" -o ./bin/masa-node ./cmd/masa-node - @go build -v -ldflags "-X github.com/masa-finance/masa-oracle/pkg/config.Version=${VERSION}" -o ./bin/masa-node-cli ./cmd/masa-node-cli + @go build -v -ldflags "-X github.com/masa-finance/masa-oracle/internal/versioning.ApplicationVersion=${VERSION}" -o ./bin/masa-node ./cmd/masa-node + @go build -v -ldflags "-X github.com/masa-finance/masa-oracle/internal/versioning.ApplicationVersion=${VERSION}" -o ./bin/masa-node-cli ./cmd/masa-node-cli install: @sh ./node_install.sh diff --git a/cmd/masa-node-cli/components.go b/cmd/masa-node-cli/components.go index dca4b4f3..e43a3766 100644 --- a/cmd/masa-node-cli/components.go +++ b/cmd/masa-node-cli/components.go @@ -5,7 +5,8 @@ import ( "strings" "github.com/gdamore/tcell/v2" - "github.com/masa-finance/masa-oracle/pkg/config" + "github.com/masa-finance/masa-oracle/internal/versioning" + "github.com/rivo/tview" ) @@ -125,7 +126,7 @@ const ( navigation = `[yellow]use keys or mouse to navigate` ) -var version string = fmt.Sprintf("[green]%s", config.Version) +var version string = fmt.Sprintf(`[green]Application version: %s\n[green]Protocol Version: %s`, versioning.ApplicationVersion, versioning.ProtocolVersion) // Splash shows the app info func Splash() (content tview.Primitive) { diff --git a/cmd/masa-node/main.go b/cmd/masa-node/main.go index ea416574..e63acc79 100644 --- a/cmd/masa-node/main.go +++ b/cmd/masa-node/main.go @@ -6,6 +6,7 @@ import ( "os/signal" "syscall" + "github.com/masa-finance/masa-oracle/internal/versioning" "github.com/masa-finance/masa-oracle/pkg/workers" "github.com/sirupsen/logrus" @@ -21,7 +22,7 @@ import ( func main() { if len(os.Args) > 1 && os.Args[1] == "--version" { - logrus.Infof("Masa Oracle Node Version: %s\n", config.Version) + logrus.Infof("Masa Oracle Node Version: %s\nMasa Oracle Protocol verison: %s", versioning.ApplicationVersion, versioning.ProtocolVersion) os.Exit(0) } @@ -135,7 +136,7 @@ func main() { multiAddr := node.GetMultiAddrs().String() // Get the multiaddress ipAddr := node.Host.Addrs()[0].String() // Get the IP address // Display the welcome message with the multiaddress and IP address - config.DisplayWelcomeMessage(multiAddr, ipAddr, keyManager.EthAddress, isStaked, isValidator, cfg.TwitterScraper, cfg.TelegramScraper, cfg.DiscordScraper, cfg.WebScraper, config.Version) + config.DisplayWelcomeMessage(multiAddr, ipAddr, keyManager.EthAddress, isStaked, isValidator, cfg.TwitterScraper, cfg.TelegramScraper, cfg.DiscordScraper, cfg.WebScraper, versioning.ApplicationVersion, versioning.ProtocolVersion) <-ctx.Done() } diff --git a/internal/versioning/version.go b/internal/versioning/version.go new file mode 100644 index 00000000..c1f58485 --- /dev/null +++ b/internal/versioning/version.go @@ -0,0 +1,9 @@ +package versioning + +var ( + ApplicationVersion string + + // XXX: Bump this value only when there are protocol changes that makes the oracle + // incompatible between version! + ProtocolVersion = `v0.5.1.2` +) diff --git a/pkg/config/app.go b/pkg/config/app.go index aae42f03..6a0ecad8 100644 --- a/pkg/config/app.go +++ b/pkg/config/app.go @@ -8,6 +8,8 @@ import ( "strings" "sync" + "github.com/masa-finance/masa-oracle/internal/versioning" + "github.com/gotd/contrib/bg" "github.com/joho/godotenv" "github.com/sirupsen/logrus" @@ -145,7 +147,7 @@ func (c *AppConfig) setDefaultConfig() { viper.SetDefault(MasaDir, filepath.Join(usr.HomeDir, ".masa")) // Set defaults - viper.SetDefault("Version", Version) + viper.SetDefault("Version", versioning.ApplicationVersion) viper.SetDefault(PortNbr, "4001") viper.SetDefault(UDP, true) viper.SetDefault(TCP, false) diff --git a/pkg/config/version.go b/pkg/config/version.go deleted file mode 100644 index b6de792e..00000000 --- a/pkg/config/version.go +++ /dev/null @@ -1,5 +0,0 @@ -package config - -var ( - Version string -) diff --git a/pkg/config/welcome.go b/pkg/config/welcome.go index 968edd0e..7f5000ba 100644 --- a/pkg/config/welcome.go +++ b/pkg/config/welcome.go @@ -4,7 +4,7 @@ import ( "fmt" ) -func DisplayWelcomeMessage(multiAddr, ipAddr, publicKeyHex string, isStaked bool, isValidator bool, isTwitterScraper bool, isTelegramScraper bool, isDiscordScraper bool, isWebScraper bool, version string) { +func DisplayWelcomeMessage(multiAddr, ipAddr, publicKeyHex string, isStaked bool, isValidator bool, isTwitterScraper bool, isTelegramScraper bool, isDiscordScraper bool, isWebScraper bool, version, protocolVersion string) { // ANSI escape code for yellow text yellow := "\033[33m" blue := "\033[34m" @@ -23,7 +23,8 @@ func DisplayWelcomeMessage(multiAddr, ipAddr, publicKeyHex string, isStaked bool fmt.Println(yellow + borderLine + reset) fmt.Println("") - fmt.Printf(blue+"%-20s %s\n"+reset, "Version:", yellow+version) + fmt.Printf(blue+"%-20s %s\n"+reset, "Application Version:", yellow+version) + fmt.Printf(blue+"%-20s %s\n"+reset, "Protocol Version:", yellow+protocolVersion) fmt.Printf(blue+"%-20s %s\n"+reset, "Multiaddress:", multiAddr) fmt.Printf(blue+"%-20s %s\n"+reset, "IP Address:", ipAddr) fmt.Printf(blue+"%-20s %s\n"+reset, "Public Key:", publicKeyHex) diff --git a/pkg/oracle_node.go b/pkg/oracle_node.go index ee93b70f..0ec8a3fe 100644 --- a/pkg/oracle_node.go +++ b/pkg/oracle_node.go @@ -37,6 +37,7 @@ import ( shell "github.com/ipfs/go-ipfs-api" pubsub "github.com/libp2p/go-libp2p-pubsub" + "github.com/masa-finance/masa-oracle/internal/versioning" chain "github.com/masa-finance/masa-oracle/pkg/chain" "github.com/masa-finance/masa-oracle/pkg/config" "github.com/masa-finance/masa-oracle/pkg/masacrypto" @@ -178,7 +179,7 @@ func NewOracleNode(ctx context.Context, isStaked bool) (*OracleNode, error) { multiAddrs: myNetwork.GetMultiAddressesForHostQuiet(hst), Context: ctx, PeerChan: make(chan myNetwork.PeerEvent), - NodeTracker: pubsub2.NewNodeEventTracker(config.Version, cfg.Environment, hst.ID().String()), + NodeTracker: pubsub2.NewNodeEventTracker(versioning.ProtocolVersion, cfg.Environment, hst.ID().String()), PubSubManager: subscriptionManager, IsStaked: isStaked, IsValidator: cfg.Validator, diff --git a/pkg/pubsub/node_data.go b/pkg/pubsub/node_data.go index dfcd6dc7..6d2bea25 100644 --- a/pkg/pubsub/node_data.go +++ b/pkg/pubsub/node_data.go @@ -5,6 +5,7 @@ import ( "fmt" "time" + "github.com/masa-finance/masa-oracle/internal/versioning" "github.com/masa-finance/masa-oracle/pkg/config" "github.com/libp2p/go-libp2p/core/host" @@ -262,7 +263,7 @@ func GetSelfNodeDataJson(host host.Host, isStaked bool) []byte { IsWebScraper: config.GetInstance().WebScraper, IsValidator: config.GetInstance().Validator, IsActive: true, - Version: config.Version, + Version: versioning.ProtocolVersion, } // Convert NodeData to JSON From ea205b514b8ead5a26c7c3ae6c5c32c99bb99d92 Mon Sep 17 00:00:00 2001 From: JD <156010594+5u6r054@users.noreply.github.com> Date: Thu, 15 Aug 2024 11:05:20 -0700 Subject: [PATCH 02/10] fix where app was using ApplicationVersion instead of ProtocolVersion --- pkg/config/app.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/config/app.go b/pkg/config/app.go index 6a0ecad8..4b6f20ca 100644 --- a/pkg/config/app.go +++ b/pkg/config/app.go @@ -147,7 +147,7 @@ func (c *AppConfig) setDefaultConfig() { viper.SetDefault(MasaDir, filepath.Join(usr.HomeDir, ".masa")) // Set defaults - viper.SetDefault("Version", versioning.ApplicationVersion) + viper.SetDefault("Version", versioning.ProtocolVersion) viper.SetDefault(PortNbr, "4001") viper.SetDefault(UDP, true) viper.SetDefault(TCP, false) From be7ce06d997e07b4fc003403dc485301d9417360 Mon Sep 17 00:00:00 2001 From: Brendan Playford <34052452+teslashibe@users.noreply.github.com> Date: Fri, 16 Aug 2024 11:59:49 -0700 Subject: [PATCH 03/10] Feat/libp2p worker (#504) * feat: refactor sendWork function in workers.go to make modular - Separate out localWorker to send local work requests - Separate out remoteWorker to enable selection and handling of remote work requests - Lay foundation for remote worker selection - even though queue is limited all remoteWorkers still get the same request which is inefficient - Log number of responses in the queue - Make logs verbose for easy debugging of localWork and remoteWork * feat: worker-sub selection refactor * feat: Implement round-robin worker selection for distributed task processing - Add round-robin iterator for fair worker distribution - Prioritize local worker when eligible - Cycle through remote workers in subsequent calls - Improve error handling and logging for worker responses - Enhance code readability and maintainability This update ensures a balanced workload across all available workers over time, while still prioritizing local processing when possible. * Improve error handling and resilience in send_work.go - Add maxSpawnAttempts constant to limit remote worker spawn attempts - Implement retry mechanism for spawning remote workers with exponential backoff - Introduce spawnRemoteWorker function for better organization and error handling - Enhance logging for better visibility into worker spawning and processing - Improve handling of dead letters and timeouts in remote worker operations - Refactor handleRemoteWorker to be more robust against transient failures - Update tryWorker function to handle both local and remote worker scenarios - Implement round-robin worker selection with retries in SendWork function These changes aim to increase the reliability of the worker system, particularly when dealing with remote workers, and provide better insights into error scenarios for easier debugging and monitoring. * feat: add config.go to worker package to simplify settings * refactor(workers): move worker selection logic to separate file - Created new file worker_selection.go to house worker selection logic - Moved GetEligibleWorkers, isEligibleRemoteWorker, and RoundRobinIterator to worker_selection.go - Updated send_work.go to use new exported functions from worker_selection.go - Renamed newRoundRobinIterator to NewRoundRobinIterator for proper exporting - Updated imports and function calls in send_work.go to reflect new structure - Improved code organization and modularity * fix: duplication of getEligibleWorkers func * feat: Enhance worker selection process with configurable remote worker limit - Add MaxRemoteWorkers to WorkerConfig in config.go - Update tryWorkersRoundRobin function in send_work.go to respect MaxRemoteWorkers limit - Implement fallback mechanism to local worker when remote workers fail or are unavailable - Add detailed logging throughout the worker selection process for better debugging - Ensure a last resort local worker is always available if no other workers are found * fix: fine tune timeouts with testing * Refactor worker selection and add eligibility check Refactored worker selection to use NodeTracker's new GetEligibleWorkerNodes method and introduced an eligibility check for staked workers. Added a new utility function for converting strings to WorkerTypes and a method in NodeEventTracker to retrieve eligible nodes based on work categories. * fixed case where err was reassigned and hiding the original * fix: local work is not timing out for long queries >20s * added more error handling and bubbled up to the tryWorker level * chore: swagger updates (#494) * ci: update workflow for deploy based on tag. (#485) * update workflow for deploy based on tag. * give workflow better name, fix env. refs. * ci: remove unneeded / untested workflows (#487) * update workflow for deploy based on tag. * give workflow better name, fix env. refs. * this needs work, not functional now. * handy cleanup script to clear out old workflows * give the workflow for running Go tests a more descriptive name * remove duplicate workflow. * update cleanup script to handle multiple workflows with same name * fix auth pattern to GCP * fix: repair deploy portion of the build and deploy to gcp bootnodes workflow (#488) * update workflow for deploy based on tag. * give workflow better name, fix env. refs. * this needs work, not functional now. * handy cleanup script to clear out old workflows * give the workflow for running Go tests a more descriptive name * remove duplicate workflow. * update cleanup script to handle multiple workflows with same name * fix auth pattern to GCP * update workflow to deploy to test env. in GCP * chore: 2fa initial fixes * stash * chore: updates to docs.go for swagger --------- Co-authored-by: J2D3 <156010594+5u6r054@users.noreply.github.com> * Enable randomized node selection and improve config settings Shuffle eligible nodes before selecting workers. Added context-based connection attempts to ensure we can connect to selected worker before returning. Adjusted AppConfig initialization order and added version flag for CLI configuration. * x: add error handling for peer info creation and fine tune timeouts - Check and log errors from NewMultiaddr and AddrInfoFromP2pAddr - Resolve "value of err is never used" linter warning - Change timeouts on workers to 20s and 15s respectively * chore: add configurable connection timeout to config.go - Add ConnectionTimeout to WorkerConfig in config.go - Update GetEligibleWorkers to use the configurable timeout * chore: revert makefile version tagger * Adjust time intervals and remove unnecessary peer removal Updated sleep durations to one minute for efficiency and commented out peer removal logic to retain node activity tracking. Also removed the callback for peer removal in DHT initialization. * resolving comments in PR * fix: cleanup logs and tune timeouts * Add worker management for distributed task handling Implemented `WorkHandlerManager` to manage different work handlers, and integrated it for handling remote and local worker tasks. Introduced new functions in the workers package to execute and manage work streams. Updated API layer to utilize the new worker management system for request distribution and response handling. * fix multiaddress and local worker assignment * fix incorrect node reference * Add length-prefix to stream communication This update adds a length-prefix to both request and response messages in the stream communication to ensure proper message boundary handling. It includes encoding the length of the messages using big-endian encoding before writing to the stream and reading the length before processing the actual messages. This change improves the robustness and reliability of streaming operations. * update logging and pass back original error it happened on remote work. * Remove actor framework and work distribution logic Deleted all files and scripts related to the actor-based framework and work distribution. This includes the removal of `workers.go`, `handlers.go`, and corresponding protobuf messages and build scripts. Restructured the package and broke up the handlers into smaller more manageable files * Refactor error handling and JSON tags in worker types Updated the error field in WorkResponse to a string as it should have been. Also, added JSON tags to struct fields for better JSON serialization. updated assignment of Error in handler functions to use formatted strings to conform to the data type change. * Fixed introduced bug with error in remote worker also fixed logging to correctly display peerId when errors occur. * fixed incorrectly scoped assignment to response it needed to be the method declared not a new variable * fixing error messages * set request to nil so that it is not in the resulting JSON * fix: empty tweet case causing panic * fix: linting error and tidy * chore: Version 0.6.0 --------- Co-authored-by: Bob Stevens <35038919+restevens402@users.noreply.github.com> Co-authored-by: Nolan Jacobson <50815660+nolanjacobson@users.noreply.github.com> Co-authored-by: J2D3 <156010594+5u6r054@users.noreply.github.com> --- cmd/masa-node/main.go | 2 +- docs/docs.go | 1968 +++++------------ go.mod | 15 +- go.sum | 31 - internal/versioning/version.go | 2 +- pkg/api/config.go | 22 + pkg/api/handlers_data.go | 264 ++- pkg/config/app.go | 4 +- pkg/config/constants.go | 1 + pkg/llmbridge/sentiment.go | 5 +- pkg/masacrypto/hash.go | 33 + pkg/network/discover.go | 5 +- pkg/network/kdht.go | 21 +- pkg/oracle_node.go | 59 +- pkg/oracle_node_listener.go | 2 +- pkg/pubsub/node_data.go | 21 +- pkg/pubsub/node_event_tracker.go | 29 +- pkg/scrapers/discord/discordprofile.go | 1 - pkg/scrapers/twitter/tweets.go | 12 +- pkg/workers/config.go | 43 + pkg/workers/handler.go | 219 -- pkg/workers/handlers/discord.go | 89 + pkg/workers/handlers/helper.go | 14 + .../{client.go => handlers/http_client.go} | 2 +- pkg/workers/handlers/llm.go | 48 + pkg/workers/handlers/telegram.go | 46 + pkg/workers/handlers/twitter.go | 85 + pkg/workers/handlers/web.go | 45 + pkg/workers/messages/build.sh | 2 - pkg/workers/messages/protos.pb.go | 374 ---- pkg/workers/messages/protos.proto | 24 - .../response_channel_map.go | 32 +- pkg/workers/types/request_response.go | 29 + pkg/workers/types/work_types.go | 51 + pkg/workers/worker_manager.go | 299 +++ pkg/workers/worker_selection.go | 67 + pkg/workers/workers.go | 491 ---- 37 files changed, 1700 insertions(+), 2757 deletions(-) create mode 100644 pkg/api/config.go create mode 100644 pkg/masacrypto/hash.go create mode 100644 pkg/workers/config.go delete mode 100644 pkg/workers/handler.go create mode 100644 pkg/workers/handlers/discord.go create mode 100644 pkg/workers/handlers/helper.go rename pkg/workers/{client.go => handlers/http_client.go} (99%) create mode 100644 pkg/workers/handlers/llm.go create mode 100644 pkg/workers/handlers/telegram.go create mode 100644 pkg/workers/handlers/twitter.go create mode 100644 pkg/workers/handlers/web.go delete mode 100755 pkg/workers/messages/build.sh delete mode 100644 pkg/workers/messages/protos.pb.go delete mode 100644 pkg/workers/messages/protos.proto rename pkg/{pubsub => workers}/response_channel_map.go (64%) create mode 100644 pkg/workers/types/request_response.go create mode 100644 pkg/workers/types/work_types.go create mode 100644 pkg/workers/worker_manager.go create mode 100644 pkg/workers/worker_selection.go delete mode 100644 pkg/workers/workers.go diff --git a/cmd/masa-node/main.go b/cmd/masa-node/main.go index e63acc79..ff934f8f 100644 --- a/cmd/masa-node/main.go +++ b/cmd/masa-node/main.go @@ -98,7 +98,7 @@ func main() { // and other peers can do work we only need to check this here // if this peer can or cannot scrape or write that is checked in other places if node.IsStaked { - go workers.MonitorWorkers(ctx, node) + node.Host.SetStreamHandler(config.ProtocolWithVersion(config.WorkerProtocol), workers.GetWorkHandlerManager().HandleWorkerStream) go masa.SubscribeToBlocks(ctx, node) go node.NodeTracker.ClearExpiredWorkerTimeouts() } diff --git a/docs/docs.go b/docs/docs.go index 0fa4b234..28b9199e 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -4,1452 +4,580 @@ package docs import "github.com/swaggo/swag" const docTemplate = `{ - "schemes": {{ marshal .Schemes }}, - "swagger": "2.0", - "info": { - "description": "{{escape .Description}}", - "title": "{{.Title}}", - "contact": { - "name": "Masa API Support", - "url": "https://masa.ai", - "email": "support@masa.ai" - }, - "license": { - "name": "MIT", - "url": "https://opensource.org/license/mit" - }, - "version": "{{.Version}}" - }, - "host": "{{.Host}}", - "basePath": "{{.BasePath}}", - "securityDefinitions": { - "Bearer": { - "type": "apiKey", - "name": "Authorization", - "in": "header" - } - }, - "security": [ - { - "Bearer": [] - } - ], - "paths": { - "/peers": { - "get": { - "description": "Retrieves a list of peers connected to the node", - "consumes": [ - "application/json" - ], - "produces": [ - "application/json" - ], - "tags": [ - "Peers" - ], - "summary": "Get list of peers", - "responses": { - "200": { - "description": "List of peer IDs", - "schema": { - "type": "array", - "items": { - "type": "string" - } - } - } - }, - "security": [ - { - "Bearer": [] - } - ] - } - }, - "/peer/addresses": { - "get": { - "description": "Retrieves a list of peer addresses connected to the node", - "consumes": [ - "application/json" - ], - "produces": [ - "application/json" - ], - "tags": [ - "Peers" - ], - "summary": "Get peer addresses", - "responses": { - "200": { - "description": "List of peer addresses", - "schema": { - "type": "array", - "items": { - "type": "string" - } - } - } - }, - "security": [ - { - "Bearer": [] - } - ] - } - }, - "/data/twitter/profile/{username}": { - "get": { - "description": "Retrieves tweets from a specific Twitter profile", - "consumes": [ - "application/json" - ], - "produces": [ - "application/json" - ], - "tags": [ - "Twitter" - ], - "summary": "Search Twitter Profile", - "parameters": [ - { - "type": "string", - "description": "Twitter Username", - "name": "username", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "List of tweets from the profile", - "schema": { - "type": "array", - "items": { - "$ref": "#/definitions/Tweet" - } - } - }, - "400": { - "description": "Invalid username or error fetching tweets", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "security": [ - { - "Bearer": [] - } - ] - } - }, - "/data/twitter/followers/{username}": { - "get": { - "description": "Retrieves followers from a specific Twitter profile.", - "consumes": [ - "application/json" - ], - "produces": [ - "application/json" - ], - "tags": [ - "Twitter" - ], - "summary": "Search Followers by Twitter Username", - "parameters": [ - { - "type": "string", - "description": "Twitter Username", - "name": "username", - "in": "path", - "required": true - }, - { - "type": "integer", - "description": "Maximum number of users to return", - "name": "count", - "in": "query", - "required": false, - "default": 20 - } - ], - "responses": { - "200": { - "description": "Array of profiles a user has as followers", - "schema": { - "type": "array", - "items": { - "$ref": "#/definitions/Profile" - } - } - }, - "400": { - "description": "Invalid username or error fetching followers", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - } - } - }, - "/data/twitter/tweets/recent": { - "post": { - "description": "Retrieves recent tweets based on query parameters", - "consumes": [ - "application/json" - ], - "produces": [ - "application/json" - ], - "tags": [ - "Twitter" - ], - "summary": "Search recent tweets", - "parameters": [ - { - "in": "body", - "name": "body", - "description": "Search parameters", - "required": true, - "schema": { - "type": "object", - "properties": { - "query": { - "type": "string", - "description": "Search Query" - }, - "count": { - "type": "integer", - "description": "Number of tweets to return" - } - } - } - } - ], - "responses": { - "200": { - "description": "List of recent tweets", - "schema": { - "type": "array", - "items": { - "$ref": "#/definitions/Tweet" - } - } - }, - "400": { - "description": "Invalid query or error fetching tweets", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "security": [ - { - "Bearer": [] - } - ] - } + "swagger": "2.0", + "info": { + "description": "{{escape .Description}}", + "title": "{{.Title}}", + "contact": { + "name": "Masa API Support", + "url": "https://masa.ai", + "email": "support@masa.ai" + }, + "license": { + "name": "MIT", + "url": "https://opensource.org/license/mit" + }, + "version": "{{.Version}}" + }, + "host": "{{.Host}}", + "basePath": "{{.BasePath}}", + "securityDefinitions": { + "Bearer": { + "type": "apiKey", + "name": "Authorization", + "in": "header" + } + }, + "security": [ + { + "Bearer": [] + } + ], + "paths": { + "/data/twitter/profile/{username}": { + "get": { + "description": "Retrieves tweets from a specific Twitter profile", + "consumes": ["application/json"], + "produces": ["application/json"], + "tags": ["Twitter"], + "summary": "Search Twitter Profile", + "parameters": [ + { + "type": "string", + "description": "Twitter Username", + "name": "username", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "List of tweets from the profile", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/Tweet" + } + } }, - "/data/twitter/tweets/trends": { - "get": { - "description": "Retrieves the latest Twitter trending topics", - "consumes": [ - "application/json" - ], - "produces": [ - "application/json" - ], - "tags": [ - "Twitter" - ], - "summary": "Twitter Trends", - "responses": { - "200": { - "description": "List of trending topics", - "schema": { - "type": "array", - "items": { - "$ref": "#/definitions/Trend" - } - } - }, - "400": { - "description": "Error fetching Twitter trends", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "security": [ - { - "Bearer": [] - } - ] - } + "400": { + "description": "Invalid username or error fetching tweets", + "schema": { + "$ref": "#/definitions/ErrorResponse" + } + } + }, + "security": [ + { + "Bearer": [] + } + ] + } + }, + "/data/twitter/followers/{username}": { + "get": { + "description": "Retrieves followers from a specific Twitter profile.", + "consumes": ["application/json"], + "produces": ["application/json"], + "tags": ["Twitter"], + "summary": "Search Twitter Followers", + "parameters": [ + { + "name": "username", + "in": "path", + "description": "Twitter Username", + "required": true, + "type": "string" }, - "/data/discord/profile/{userID}": { - "get": { - "description": "Retrieves a Discord user profile by user ID.", - "consumes": [ - "application/json" - ], - "produces": [ - "application/json" - ], - "tags": [ - "Discord" - ], - "summary": "Search Discord Profile", - "parameters": [ - { - "name": "userID", - "in": "path", - "description": "Discord User ID", - "required": true, - "type": "string" - } - ], - "responses": { - "200": { - "description": "Successfully retrieved Discord user profile", - "schema": { - "$ref": "#/definitions/UserProfile" - } - }, - "400": { - "description": "Invalid user ID or error fetching profile", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "security": [ - { - "Bearer": [] - } - ] - } + { + "name": "limit", + "in": "query", + "description": "The maximum number of followers to return", + "required": false, + "type": "integer", + "format": "int32", + "default": 20 + } + ], + "responses": { + "200": { + "description": "Array of profiles a user has as followers", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/Profile" + } + } }, - "/data/discord/channels/{channelID}/messages": { - "get": { - "description": "Retrieves messages from a specified Discord channel.", - "tags": ["Discord"], - "summary": "Get messages from a Discord channel", - "parameters": [ - { - "name": "channelID", - "in": "path", - "description": "Discord Channel ID", - "required": true, - "type": "string" - }, - { - "name": "limit", - "in": "query", - "description": "The maximum number of messages to return", - "required": false, - "type": "integer", - "format": "int32" - }, - { - "name": "before", - "in": "query", - "description": "A message ID to return messages posted before this message", - "required": false, - "type": "string" - } - ], - "responses": { - "200": { - "description": "Successfully retrieved messages from the Discord channel", - "schema": { - "type": "array", - "items": { - "$ref": "#/definitions/ChannelMessage" - } - } - }, - "400": { - "description": "Invalid channel ID or error fetching messages", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "security": [ - { - "Bearer": [] - } - ] - } - }, - "/data/discord/guilds/{guildID}/channels": { - "get": { - "description": "Retrieves channels from a specified Discord guild.", - "tags": ["Discord"], - "summary": "Get channels from a Discord guild", - "parameters": [ - { - "name": "guildID", - "in": "path", - "description": "Discord Guild ID", - "required": true, - "type": "string" - } - ], - "responses": { - "200": { - "description": "Successfully retrieved channels from the Discord guild", - "schema": { - "type": "array", - "items": { - "$ref": "#/definitions/GuildChannel" - } - } - }, - "400": { - "description": "Invalid guild ID or error fetching channels", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "security": [ - { - "Bearer": [] - } - ] - } - }, - "/data/discord/user/guilds": { - "get": { - "description": "Retrieves guilds from a specified Discord user.", - "tags": ["Discord"], - "summary": "Get guilds from a Discord user", - "parameters": [ - ], - "responses": { - "200": { - "description": "Successfully retrieved guilds from the Discord user", - "schema": { - "type": "array", - "items": { - "$ref": "#/definitions/UserGuild" - } - } - }, - "400": { - "description": "Invalid user ID or error fetching guilds", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } + "400": { + "description": "Invalid username or error fetching followers", + "schema": { + "$ref": "#/definitions/ErrorResponse" + } + } + } + } + }, + "/data/twitter/tweets/recent": { + "post": { + "description": "Retrieves recent tweets based on query parameters", + "consumes": ["application/json"], + "produces": ["application/json"], + "tags": ["Twitter"], + "summary": "Search recent tweets", + "parameters": [ + { + "in": "body", + "name": "body", + "description": "Search parameters", + "required": true, + "schema": { + "type": "object", + "properties": { + "query": { + "type": "string", + "description": "Search Query" }, - "security": [ - { - "Bearer": [] - } - ] - } - }, - "/data/discord/guilds/all": { - "get": { - "description": "Retrieves all guilds that all the Discord workers are apart of.", - "consumes": [ - "application/json" - ], - "produces": [ - "application/json" - ], - "tags": [ - "Discord" - ], - "summary": "Get all guilds", - "responses": { - "200": { - "description": "Successfully retrieved all guilds for the Discord user", - "schema": { - "type": "array", - "items": { - "$ref": "#/definitions/Guild" - } - } - }, - "400": { - "description": "Error fetching guilds or invalid access token", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "security": [ - { - "Bearer": [] - } - ] - } - }, - "/data/telegram/channel/messages": { - "post": { - "description": "Retrieves messages from a specified Telegram channel.", - "tags": ["Telegram"], - "summary": "Get Telegram Channel Messages", - "parameters": [ - { - "in": "body", - "name": "body", - "description": "Request body", - "required": true, - "schema": { - "type": "object", - "properties": { - "username": { - "type": "string", - "description": "Telegram Username" - } - }, - "required": ["username"] - } - } - ], - "responses": { - "200": { - "description": "Successfully retrieved messages", - "schema": { - "type": "array", - "items": { - "$ref": "#/definitions/Message" - } - } - }, - "400": { - "description": "Invalid username or error fetching messages", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "security": [ - { - "Bearer": [] - } - ] - } - }, - "/data/web": { - "post": { - "description": "Retrieves data from the web", - "consumes": [ - "application/json" - ], - "produces": [ - "application/json" - ], - "tags": [ - "Web" - ], - "summary": "Web Data", - "parameters": [ - { - "in": "body", - "name": "body", - "description": "Search parameters", - "required": true, - "schema": { - "type": "object", - "properties": { - "url": { - "type": "string", - "description": "Url" - }, - "depth": { - "type": "integer", - "description": "Number of pages to scrape" - } - } - } - } - ], - "responses": { - "200": { - "description": "Successfully retrieved web data", - "schema": { - "$ref": "#/definitions/WebDataResponse" - } - }, - "400": { - "description": "Invalid query or error fetching web data", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "security": [ - { - "Bearer": [] - } - ] - } - }, - "/dht": { - "get": { - "description": "Retrieves data from the DHT (Distributed Hash Table)", - "consumes": [ - "application/json" - ], - "produces": [ - "application/json" - ], - "tags": [ - "DHT" - ], - "summary": "Get DHT Data", - "parameters": [ - { - "in": "query", - "name": "key", - "description": "Key to retrieve data for", - "required": true, - "type": "string" - } - ], - "responses": { - "200": { - "description": "Successfully retrieved data from DHT", - "schema": { - "$ref": "#/definitions/DHTResponse" - } - }, - "400": { - "description": "Error retrieving data from DHT", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "security": [ - { - "Bearer": [] - } - ] - }, - "post": { - "description": "Adds data to the DHT (Distributed Hash Table)", - "consumes": [ - "application/json" - ], - "produces": [ - "application/json" - ], - "tags": [ - "DHT" - ], - "summary": "Post to DHT", - "parameters": [ - { - "description": "Data to store in DHT", - "name": "data", - "in": "body", - "required": true, - "schema": { - "type": "object", - "properties": { - "key": { - "type": "string" - }, - "value": { - "type": "string" - } - } - } - } - ], - "responses": { - "200": { - "description": "Successfully added data to DHT", - "schema": { - "$ref": "#/definitions/SuccessResponse" - } - }, - "400": { - "description": "Error adding data to DHT", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "security": [ - { - "Bearer": [] - } - ] - } - }, - "/llm/models": { - "get": { - "description": "Retrieves the available LLM models", - "consumes": [ - "application/json" - ], - "produces": [ - "application/json" - ], - "tags": [ - "LLM" - ], - "summary": "Get LLM Models", - "responses": { - "200": { - "description": "Successfully retrieved LLM models", - "schema": { - "$ref": "#/definitions/LLMModelsResponse" - } - }, - "400": { - "description": "Error retrieving LLM models", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "security": [ - { - "Bearer": [] - } - ] - } - }, - "/chat": { - "post": { - "summary": "Chat with AI", - "description": "Initiates a chat session with an AI model.", - "consumes": ["application/json"], - "produces": ["application/json"], - "parameters": [ - { - "in": "body", - "name": "body", - "description": "Chat request payload", - "required": true, - "schema": { - "type": "object", - "properties": { - "model": { - "type": "string", - "example": "llama3" - }, - "messages": { - "type": "array", - "items": { - "type": "object", - "properties": { - "role": { - "type": "string", - "example": "user" - }, - "content": { - "type": "string", - "example": "why is the sky blue?" - } - } - } - }, - "stream": { - "type": "boolean", - "example": false - } - }, - "required": ["model", "messages", "stream"] - } - } - ], - "responses": { - "200": { - "description": "Successfully received response from AI", - "schema": { - "$ref": "#/definitions/ChatResponse" - } - }, - "400": { - "description": "Error communicating with AI", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - } - } - }, - "/node/data": { - "get": { - "description": "Retrieves data from the node", - "consumes": [ - "application/json" - ], - "produces": [ - "application/json" - ], - "tags": [ - "Node" - ], - "summary": "Node Data", - "responses": { - "200": { - "description": "Successfully retrieved node data", - "schema": { - "$ref": "#/definitions/NodeDataResponse" - } - }, - "400": { - "description": "Error retrieving node data", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "security": [ - { - "Bearer": [] - } - ] - } - }, - "/node/data/{peerid}": { - "get": { - "description": "Retrieves data for a specific node identified by peer ID", - "consumes": [ - "application/json" - ], - "produces": [ - "application/json" - ], - "tags": [ - "Node" - ], - "summary": "Get Node Data by Peer ID", - "parameters": [ - { - "type": "string", - "description": "Peer ID", - "name": "peerid", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "Successfully retrieved node data by peer ID", - "schema": { - "$ref": "#/definitions/NodeDataResponse" - } - }, - "400": { - "description": "Error retrieving node data by peer ID", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "security": [ - { - "Bearer": [] - } - ] + "count": { + "type": "integer", + "description": "Number of tweets to return" + } } + } + } + ], + "responses": { + "200": { + "description": "List of recent tweets", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/Tweet" + } + } }, - "/sentiment/tweets": { - "post": { - "description": "Searches for tweets and analyzes their sentiment", - "consumes": [ - "application/json" - ], - "produces": [ - "application/json" - ], - "tags": [ - "Sentiment" - ], - "summary": "Analyze Sentiment of Tweets", - "parameters": [ - { - "in": "body", - "name": "body", - "description": "Sentiment analysis request body", - "required": true, - "schema": { - "type": "object", - "properties": { - "query": { - "type": "string", - "description": "Search Query" - }, - "count": { - "type": "integer", - "description": "Number of tweets to analyze" - }, - "model": { - "type": "string", - "description": "Sentiment analysis model to use" - } - } - } - } - ], - "responses": { - "200": { - "description": "Successfully analyzed sentiment of tweets", - "schema": { - "$ref": "#/definitions/SentimentAnalysisResponse" - } - }, - "400": { - "description": "Error analyzing sentiment of tweets", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - } - } + "400": { + "description": "Invalid query or error fetching tweets", + "schema": { + "$ref": "#/definitions/ErrorResponse" + } + } + }, + "security": [ + { + "Bearer": [] + } + ] + } + }, + "/data/twitter/tweets/trends": { + "get": { + "description": "Retrieves the latest Twitter trending topics", + "consumes": ["application/json"], + "produces": ["application/json"], + "tags": ["Twitter"], + "summary": "Twitter Trends", + "responses": { + "200": { + "description": "List of trending topics", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/Trend" + } + } }, - "/sentiment/telegram": { - "post": { - "description": "Searches for Telegram messages and analyzes their sentiment", - "tags": ["Sentiment"], - "summary": "Analyze Sentiment of Telegram Messages", - "consumes": ["application/json"], - "produces": ["application/json"], - "parameters": [ - { - "name": "query", - "in": "body", - "description": "Search Query", - "required": true, - "schema": { - "type": "object", - "properties": { - "query": { - "type": "string" - } - } - } - } - ], - "responses": { - "200": { - "description": "Successfully analyzed sentiment of Telegram messages", - "schema": { - "$ref": "#/definitions/SentimentAnalysisResponse" - } - }, - "400": { - "description": "Error analyzing sentiment of Telegram messages", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - } - } + "400": { + "description": "Error fetching Twitter trends", + "schema": { + "$ref": "#/definitions/ErrorResponse" + } + } + }, + "security": [ + { + "Bearer": [] + } + ] + } + }, + "/data/discord/profile/{userID}": { + "get": { + "description": "Retrieves a Discord user profile by user ID.", + "consumes": ["application/json"], + "produces": ["application/json"], + "tags": ["Discord"], + "summary": "Search Discord Profile", + "parameters": [ + { + "name": "userID", + "in": "path", + "description": "Discord User ID", + "required": true, + "type": "string" + } + ], + "responses": { + "200": { + "description": "Successfully retrieved Discord user profile", + "schema": { + "$ref": "#/definitions/UserProfile" + } }, - "/sentiment/discord": { - "post": { - "description": "Searches for Discord messages and analyzes their sentiment", - "consumes": [ - "application/json" - ], - "produces": [ - "application/json" - ], - "tags": [ - "Sentiment" - ], - "summary": "Analyze Sentiment of Discord Messages", - "parameters": [ - { - "in": "body", - "name": "body", - "description": "Sentiment analysis request body for Discord messages", - "required": true, - "schema": { - "type": "object", - "properties": { - "channelID": { - "type": "string", - "description": "Discord Channel ID" - }, - "prompt": { - "type": "string", - "description": "Prompt to enter" - }, - "model": { - "type": "string", - "description": "Sentiment analysis model to use" - } - }, - "required": ["channelID", "model"] - } - } - ], - "responses": { - "200": { - "description": "Successfully analyzed sentiment of Discord messages", - "schema": { - "$ref": "#/definitions/SentimentAnalysisResponse" - } - }, - "400": { - "description": "Error analyzing sentiment of Discord messages", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - } - } + "400": { + "description": "Invalid user ID or error fetching profile", + "schema": { + "$ref": "#/definitions/ErrorResponse" + } + } + }, + "security": [ + { + "Bearer": [] + } + ] + } + }, + "/data/discord/channels/{channelID}/messages": { + "get": { + "description": "Retrieves messages from a specified Discord channel.", + "consumes": ["application/json"], + "produces": ["application/json"], + "tags": ["Discord"], + "summary": "Get messages from a Discord channel", + "parameters": [ + { + "name": "channelID", + "in": "path", + "description": "Discord Channel ID", + "required": true, + "type": "string" }, - "/auth": { - "get": { - "description": "Retrieves the API key for the node", - "produces": [ - "application/json" - ], - "tags": [ - "Authentication" - ], - "summary": "Get Node API Key", - "responses": { - "200": { - "description": "Successfully retrieved API key", - "schema": { - "type": "object", - "additionalProperties": { - "type": "string" - } - } - }, - "500": { - "description": "Error generating API key", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - } - } + { + "name": "limit", + "in": "query", + "description": "The maximum number of messages to return", + "required": false, + "type": "integer", + "format": "int32" }, - "/auth/telegram/start": { - "post": { - "description": "Initiates the authentication process with Telegram by sending a code to the provided phone number.", - "tags": ["Authentication"], - "summary": "Start Telegram Authentication", - "consumes": ["application/json"], - "produces": ["application/json"], - "parameters": [ - { - "name": "phone_number", - "in": "body", - "description": "Phone Number", - "required": true, - "schema": { - "type": "object", - "properties": { - "phone_number": { - "type": "string" - } - } - } - } - ], - "responses": { - "200": { - "description": "Successfully sent authentication code", - "schema": { - "type": "object", - "additionalProperties": { - "type": "string" - } - } - }, - "400": { - "description": "Invalid request body", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "Failed to initialize Telegram client or to start authentication", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - } - } + { + "name": "before", + "in": "query", + "description": "A message ID to return messages posted before this message", + "required": false, + "type": "string" + } + ], + "responses": { + "200": { + "description": "Successfully retrieved messages from the Discord channel", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/ChannelMessage" + } + } }, - "/auth/telegram/complete": { - "post": { - "description": "Completes the authentication process with Telegram using the code sent to the phone number.", - "tags": ["Authentication"], - "summary": "Complete Telegram Authentication", - "consumes": ["application/json"], - "produces": ["application/json"], - "parameters": [ - { - "name": "phone_number", - "in": "body", - "description": "Phone Number", - "required": true, - "schema": { - "type": "object", - "properties": { - "phone_number": { - "type": "string" - }, - "code": { - "type": "string" - }, - "phone_code_hash": { - "type": "string" - } - }, - "required": ["phone_number", "code", "phone_code_hash"] - } - } - ], - "responses": { - "200": { - "description": "Successfully authenticated", - "schema": { - "type": "object", - "additionalProperties": { - "type": "string" - } - } - }, - "400": { - "description": "Invalid request body", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "401": { - "description": "Two-factor authentication is required", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "Failed to initialize Telegram client or to complete authentication", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - } - } + "400": { + "description": "Invalid channel ID or error fetching messages", + "schema": { + "$ref": "#/definitions/ErrorResponse" + } + } + }, + "security": [ + { + "Bearer": [] + } + ] + } + }, + "/data/discord/guilds/{guildID}/channels": { + "get": { + "description": "Retrieves channels from a specified Discord guild.", + "tags": ["Discord"], + "summary": "Get channels from a Discord guild", + "parameters": [ + { + "name": "guildID", + "in": "path", + "description": "Discord Guild ID", + "required": true, + "type": "string" + } + ], + "responses": { + "200": { + "description": "Successfully retrieved channels from the Discord guild", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/GuildChannel" + } + } }, - "/sentiment/web": { - "post": { - "description": "Searches for web content and analyzes its sentiment", - "consumes": [ - "application/json" - ], - "produces": [ - "application/json" - ], - "tags": [ - "Sentiment" - ], - "summary": "Analyze Sentiment of Web Content", - "parameters": [ - { - "in": "body", - "name": "body", - "description": "Sentiment analysis request body", - "required": true, - "schema": { - "type": "object", - "properties": { - "url": { - "type": "string", - "description": "URL of the web content" - }, - "depth": { - "type": "integer", - "description": "Depth of web crawling" - }, - "model": { - "type": "string", - "description": "Sentiment analysis model to use" - } - } - } - } - ], - "responses": { - "200": { - "description": "Successfully analyzed sentiment of web content", - "schema": { - "$ref": "#/definitions/SentimentAnalysisResponse" - } - }, - "400": { - "description": "Error analyzing sentiment of web content", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - } - } + "400": { + "description": "Invalid guild ID or error fetching channels", + "schema": { + "$ref": "#/definitions/ErrorResponse" + } + } + }, + "security": [ + { + "Bearer": [] + } + ] + } + }, + "/data/discord/user/guilds": { + "get": { + "description": "Retrieves guilds from a specified Discord user.", + "tags": ["Discord"], + "summary": "Get guilds from a Discord user", + "responses": { + "200": { + "description": "Successfully retrieved guilds from the Discord user", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/UserGuild" + } + } }, - }, - "DHTResponse": { - "type": "object", - "properties": { - "key": { - "type": "string" - }, - "value": { - "type": "string" - } + "400": { + "description": "Invalid user ID or error fetching guilds", + "schema": { + "$ref": "#/definitions/ErrorResponse" + } } - }, - "SuccessResponse": { - "type": "object", - "properties": { - "message": { - "type": "string" - } + }, + "security": [ + { + "Bearer": [] } - }, - "WebDataRequest": { - "type": "object", - "properties": { - "query": { - "type": "string" - }, - "url": { - "type": "string" - }, - "depth": { - "type": "integer" - } + ] + } + }, + "/data/discord/guilds/all": { + "get": { + "description": "Retrieves all guilds that all the Discord workers are apart of.", + "consumes": ["application/json"], + "produces": ["application/json"], + "tags": ["Discord"], + "summary": "Get all guilds", + "responses": { + "200": { + "description": "Successfully retrieved all guilds for the Discord user", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/Guild" + } + } + }, + "400": { + "description": "Error fetching guilds or invalid access token", + "schema": { + "$ref": "#/definitions/ErrorResponse" + } + } + }, + "security": [ + { + "Bearer": [] } - }, - "WebDataResponse": { + ] + } + } + }, + "definitions": { + "UserProfile": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "username": { + "type": "string" + }, + "discriminator": { + "type": "string" + }, + "avatar": { + "type": "string" + } + } + }, + "SuccessResponse": { + "type": "object", + "properties": { + "message": { + "type": "string" + } + } + }, + "ErrorResponse": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + } + }, + "Tweet": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "text": { + "type": "string" + }, + "created_at": { + "type": "string" + }, + "user": { "type": "object", "properties": { - "data": { - "type": "string" - } + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "screen_name": { + "type": "string" + } } - }, - "SentimentAnalysisResponse": { + } + } + }, + "ChannelMessage": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "channelID": { + "type": "string" + }, + "author": { "type": "object", "properties": { - "sentiment": { - "type": "string" - }, - "data": { - "type": "string" - } - } - }, - "definitions": { - "ChatResponse": { - "type": "object", - "properties": { - "message": { - "type": "string" - } - } - }, - "DHTResponse": { - "type": "object", - "properties": { - "key": { - "type": "string" - }, - "value": { - "type": "string" - } - } - }, - "UserProfile": { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "username": { - "type": "string" - }, - "discriminator": { - "type": "string" - }, - "avatar": { - "type": "string" - } - } - }, - "ErrorResponse": { - "type": "object", - "properties": { - "error": { - "type": "string" - } - } - }, - "Tweet": { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "text": { - "type": "string" - }, - "created_at": { - "type": "string" - }, - "user": { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "name": { - "type": "string" - }, - "screen_name": { - "type": "string" - } - } - } - } - }, - "ChannelMessage": { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "channelID": { - "type": "string" - }, - "author": { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "username": { - "type": "string" - }, - "discriminator": { - "type": "string" - }, - "avatar": { - "type": "string" - } - } - }, - "content": { - "type": "string" - }, - "timestamp": { - "type": "string" - } - } + "id": { + "type": "string" }, - "GuildChannel": { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "guildID": { - "type": "string" - }, - "name": { - "type": "string" - }, - "type": { - "type": "integer" - } - } + "username": { + "type": "string" }, - "Guild": { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "name": { - "type": "string" - }, - "icon": { - "type": "string" - }, - "owner": { - "type": "boolean" - }, - "permissions": { - "type": "string" - } - } + "discriminator": { + "type": "string" }, - "Trend": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "url": { - "type": "string" - }, - "tweet_volume": { - "type": "integer" - } - } - }, - "SentimentAnalysisResponse": { - "type": "object", - "properties": { - "sentiment": { - "type": "string" - }, - "data": { - "type": "string" - } - } - }, - "WebDataResponse": { - "type": "object", - "properties": { - "data": { - "type": "string" - } - } - }, - "LLMModelsResponse": { - "type": "object", - "properties": { - "models": { - "type": "array", - "items": { - "type": "string" - } - } - } - }, - "NodeDataResponse": { - "type": "object", - "properties": { - "peer_id": { - "type": "string" - }, - "data": { - "type": "string" - } - } + "avatar": { + "type": "string" + } } + }, + "content": { + "type": "string" + }, + "timestamp": { + "type": "string" + } + } + }, + "GuildChannel": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "guildID": { + "type": "string" + }, + "name": { + "type": "string" + }, + "type": { + "type": "integer" + } + } + }, + "Guild": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "icon": { + "type": "string" + }, + "owner": { + "type": "boolean" + }, + "permissions": { + "type": "string" + } + } + }, + "Trend": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "url": { + "type": "string" + }, + "tweet_volume": { + "type": "integer" + } + } + }, + "Profile": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "username": { + "type": "string" + }, + "discriminator": { + "type": "string" + }, + "avatar": { + "type": "string" + } + } + }, + "UserGuild": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "icon": { + "type": "string" + }, + "owner": { + "type": "boolean" + }, + "permissions": { + "type": "string" + } } -}` + } + } + }` // SwaggerInfo holds exported Swagger Info so clients can modify it var SwaggerInfo = &swag.Spec{ diff --git a/go.mod b/go.mod index b3ca6763..7865648e 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,6 @@ go 1.22.0 toolchain go1.22.2 require ( - github.com/asynkron/protoactor-go v0.0.0-20240413045429-76c172a71a16 github.com/cenkalti/backoff/v4 v4.3.0 github.com/chyeh/pubip v0.0.0-20170203095919-b7e679cf541c github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 @@ -42,7 +41,6 @@ require ( github.com/swaggo/files v1.0.1 github.com/swaggo/gin-swagger v1.6.0 github.com/swaggo/swag v1.16.3 - google.golang.org/protobuf v1.34.2 ) require ( @@ -50,12 +48,10 @@ require ( github.com/KyleBanks/depth v1.2.1 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/PuerkitoBio/goquery v1.9.1 // indirect - github.com/Workiva/go-datastructures v1.1.3 // indirect github.com/andybalholm/cascadia v1.3.2 // indirect github.com/antchfx/htmlquery v1.2.3 // indirect github.com/antchfx/xmlquery v1.3.1 // indirect github.com/antchfx/xpath v1.1.10 // indirect - github.com/asynkron/gofun v0.0.0-20220329210725-34fed760f4c2 // indirect github.com/benbjohnson/clock v1.3.5 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bits-and-blooms/bitset v1.13.0 // indirect @@ -82,7 +78,6 @@ require ( github.com/docker/go-units v0.5.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/elastic/gosigar v0.14.2 // indirect - github.com/emirpasic/gods v1.18.1 // indirect github.com/ethereum/c-kzg-4844 v1.0.0 // indirect github.com/flynn/noise v1.1.0 // indirect github.com/francoispqt/gojay v1.2.13 // indirect @@ -152,8 +147,6 @@ require ( github.com/libp2p/go-reuseport v0.4.0 // indirect github.com/libp2p/go-yamux/v4 v4.0.1 // indirect github.com/libp2p/zeroconf/v2 v2.2.0 // indirect - github.com/lithammer/shortuuid/v4 v4.0.0 // indirect - github.com/lmittmann/tint v1.0.3 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mailru/easyjson v0.7.7 // indirect @@ -182,7 +175,6 @@ require ( github.com/onsi/ginkgo/v2 v2.16.0 // indirect github.com/opencontainers/runtime-spec v1.2.0 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect - github.com/orcaman/concurrent-map v1.0.0 // indirect github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/pion/datachannel v1.5.6 // indirect @@ -231,7 +223,6 @@ require ( github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.6.1 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect - github.com/twmb/murmur3 v1.1.8 // indirect github.com/ugorji/go/codec v1.2.12 // indirect github.com/urfave/cli/v2 v2.27.1 // indirect github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirect @@ -239,10 +230,7 @@ require ( github.com/yusufpapurcu/wmi v1.2.4 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/otel v1.28.0 // indirect - go.opentelemetry.io/otel/exporters/prometheus v0.44.0 // indirect go.opentelemetry.io/otel/metric v1.28.0 // indirect - go.opentelemetry.io/otel/sdk v1.28.0 // indirect - go.opentelemetry.io/otel/sdk/metric v1.28.0 // indirect go.opentelemetry.io/otel/trace v1.28.0 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/dig v1.17.1 // indirect @@ -262,8 +250,7 @@ require ( golang.org/x/tools v0.22.0 // indirect gonum.org/v1/gonum v0.15.0 // indirect google.golang.org/appengine v1.6.8 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240513163218-0867130af1f8 // indirect - google.golang.org/grpc v1.63.2 // indirect + google.golang.org/protobuf v1.34.2 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect lukechampine.com/blake3 v1.2.1 // indirect diff --git a/go.sum b/go.sum index a5fde0b3..7e4fc34e 100644 --- a/go.sum +++ b/go.sum @@ -23,8 +23,6 @@ github.com/PuerkitoBio/goquery v1.9.1 h1:mTL6XjbJTZdpfL+Gwl5U2h1l9yEkJjhmlTeV9VP github.com/PuerkitoBio/goquery v1.9.1/go.mod h1:cW1n6TmIMDoORQU5IU/P1T3tGFunOeXEpGP2WHRwkbY= github.com/VictoriaMetrics/fastcache v1.12.2 h1:N0y9ASrJ0F6h0QaC3o6uJb3NIZ9VKLjCM7NQbSmF7WI= github.com/VictoriaMetrics/fastcache v1.12.2/go.mod h1:AmC+Nzz1+3G2eCPapF6UcsnkThDcMsQicp4xDukwJYI= -github.com/Workiva/go-datastructures v1.1.3 h1:LRdRrug9tEuKk7TGfz/sct5gjVj44G9pfqDt4qm7ghw= -github.com/Workiva/go-datastructures v1.1.3/go.mod h1:1yZL+zfsztete+ePzZz/Zb1/t5BnDuE2Ya2MMGhzP6A= github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= github.com/andybalholm/cascadia v1.2.0/go.mod h1:YCyR8vOZT9aZ1CHEd8ap0gMVm2aFgxBp0T0eFw1RUQY= github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss= @@ -40,10 +38,6 @@ github.com/antchfx/xpath v1.1.8/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNY github.com/antchfx/xpath v1.1.10 h1:cJ0pOvEdN/WvYXxvRrzQH9x5QWKpzHacYO8qzCcDYAg= github.com/antchfx/xpath v1.1.10/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= -github.com/asynkron/gofun v0.0.0-20220329210725-34fed760f4c2 h1:jEsFZ9d/ieJGVrx3fSPi8oe/qv21fRmyUL5cS3ZEn5A= -github.com/asynkron/gofun v0.0.0-20220329210725-34fed760f4c2/go.mod h1:5GMOSqaYxNWwuVRWyampTPJEntwz7Mj9J8v1a7gSU2E= -github.com/asynkron/protoactor-go v0.0.0-20240413045429-76c172a71a16 h1:WcgLv2PuooiG5+WmeJAaWevD5RZH3HMVxyTZX0xofJM= -github.com/asynkron/protoactor-go v0.0.0-20240413045429-76c172a71a16/go.mod h1:HTx47MGokOrouz8nrUmjyLLOVu+/kRNN6KKVG0XjQ3E= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz5o= @@ -149,8 +143,6 @@ github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+m github.com/elastic/gosigar v0.12.0/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs= github.com/elastic/gosigar v0.14.2 h1:Dg80n8cr90OZ7x+bAax/QjoW/XqTI11RmA79ZwIm9/4= github.com/elastic/gosigar v0.14.2/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs= -github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= -github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -303,7 +295,6 @@ github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm4 github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -456,10 +447,6 @@ github.com/libp2p/go-yamux/v4 v4.0.1 h1:FfDR4S1wj6Bw2Pqbc8Uz7pCxeRBPbwsBbEdfwiCy github.com/libp2p/go-yamux/v4 v4.0.1/go.mod h1:NWjl8ZTLOGlozrXSOZ/HlfG++39iKNnM5wwmtQP1YB4= github.com/libp2p/zeroconf/v2 v2.2.0 h1:Cup06Jv6u81HLhIj1KasuNM/RHHrJ8T7wOTS4+Tv53Q= github.com/libp2p/zeroconf/v2 v2.2.0/go.mod h1:fuJqLnUwZTshS3U/bMRJ3+ow/v9oid1n0DmyYyNO1Xs= -github.com/lithammer/shortuuid/v4 v4.0.0 h1:QRbbVkfgNippHOS8PXDkti4NaWeyYfcBTHtw7k08o4c= -github.com/lithammer/shortuuid/v4 v4.0.0/go.mod h1:Zs8puNcrvf2rV9rTH51ZLLcj7ZXqQI3lv67aw4KiB1Y= -github.com/lmittmann/tint v1.0.3 h1:W5PHeA2D8bBJVvabNfQD/XW9HPLZK1XoPZH0cq8NouQ= -github.com/lmittmann/tint v1.0.3/go.mod h1:HIS3gSy7qNwGCj+5oRjAutErFBl4BzdQP6cJZ0NfMwE= github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= @@ -568,14 +555,11 @@ github.com/opencontainers/runtime-spec v1.2.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/ github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8= -github.com/orcaman/concurrent-map v1.0.0 h1:I/2A2XPCb4IuQWcQhBhSwGfiuybl/J0ev9HDbW65HOY= -github.com/orcaman/concurrent-map v1.0.0/go.mod h1:Lu3tH6HLW3feq74c2GC+jIMS/K2CFcDWnWD9XkenwhI= github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0= github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= -github.com/philhofer/fwd v1.1.1/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= github.com/pion/datachannel v1.5.6 h1:1IxKJntfSlYkpUj8LlYRSWpYiTTC02nUrOE8T3DqGeg= github.com/pion/datachannel v1.5.6/go.mod h1:1eKT6Q85pRnr2mHiWHxJwO50SfZRtWHTsNIVb/NfGW4= github.com/pion/dtls/v2 v2.2.7/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s= @@ -766,16 +750,12 @@ github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45 github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= github.com/temoto/robotstxt v1.1.1 h1:Gh8RCs8ouX3hRSxxK7B1mO5RFByQ4CmJZDwgom++JaA= github.com/temoto/robotstxt v1.1.1/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo= -github.com/tinylib/msgp v1.1.5/go.mod h1:eQsjooMTnV42mHu917E26IogZ2930nFyBQdofk10Udg= github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= -github.com/ttacon/chalk v0.0.0-20160626202418-22c06c80ed31/go.mod h1:onvgF043R+lC5RZ8IT9rBXDaEDnpnw/Cl+HFiw+v/7Q= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= -github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg= -github.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/tyler-smith/go-bip39 v1.1.0 h1:5eUemwrMargf3BSLRRCalXT93Ns6pQJIjYQN2nyfOP8= github.com/tyler-smith/go-bip39 v1.1.0/go.mod h1:gUYDtqQw1JS3ZJ8UWVcGTGqqr6YIN3CWg+kkNaLt55U= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= @@ -805,14 +785,8 @@ go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/otel v1.28.0 h1:/SqNcYk+idO0CxKEUOtKQClMK/MimZihKYMruSMViUo= go.opentelemetry.io/otel v1.28.0/go.mod h1:q68ijF8Fc8CnMHKyzqL6akLO46ePnjkgfIMIjUIX9z4= -go.opentelemetry.io/otel/exporters/prometheus v0.44.0 h1:08qeJgaPC0YEBu2PQMbqU3rogTlyzpjhCI2b58Yn00w= -go.opentelemetry.io/otel/exporters/prometheus v0.44.0/go.mod h1:ERL2uIeBtg4TxZdojHUwzZfIFlUIjZtxubT5p4h1Gjg= go.opentelemetry.io/otel/metric v1.28.0 h1:f0HGvSl1KRAU1DLgLGFjrwVyismPlnuU6JD6bOeuA5Q= go.opentelemetry.io/otel/metric v1.28.0/go.mod h1:Fb1eVBFZmLVTMb6PPohq3TO9IIhUisDsbJoL/+uQW4s= -go.opentelemetry.io/otel/sdk v1.28.0 h1:b9d7hIry8yZsgtbmM0DKyPWMMUMlK9NEKuIG4aBqWyE= -go.opentelemetry.io/otel/sdk v1.28.0/go.mod h1:oYj7ClPUA7Iw3m+r7GeEjz0qckQRJK2B8zjcZEfu7Pg= -go.opentelemetry.io/otel/sdk/metric v1.28.0 h1:OkuaKgKrgAbYrrY0t92c+cC+2F6hsFNnCQArXCKlg08= -go.opentelemetry.io/otel/sdk/metric v1.28.0/go.mod h1:cWPjykihLAPvXKi4iZc1dpER3Jdq2Z0YLse3moQUCpg= go.opentelemetry.io/otel/trace v1.28.0 h1:GhQ9cUuQGmNDd5BTCP2dAvv75RdMxEfTmYejp+lkx9g= go.opentelemetry.io/otel/trace v1.28.0/go.mod h1:jPyXzNPg6da9+38HEwElrQiHlVMTnVfM3/yv2OlIHaI= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= @@ -1031,7 +1005,6 @@ golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20201022035929-9cf592e881e9/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= @@ -1061,8 +1034,6 @@ google.golang.org/genproto v0.0.0-20181202183823-bd91e49a0898/go.mod h1:7Ep/1NZk google.golang.org/genproto v0.0.0-20190306203927-b5d61aea6440/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240513163218-0867130af1f8 h1:mxSlqyb8ZAHsYDCfiXN1EDdNTdvjUJSLY+OnAUtYNYA= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240513163218-0867130af1f8/go.mod h1:I7Y+G38R2bu5j1aLzfFmQfTcU/WnFuqDwLZAbvKTKpM= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= @@ -1071,8 +1042,6 @@ google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyac google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.63.2 h1:MUeiw1B2maTVZthpU5xvASfTh3LDbxHd6IJ6QQVU+xM= -google.golang.org/grpc v1.63.2/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= diff --git a/internal/versioning/version.go b/internal/versioning/version.go index c1f58485..4e9d3286 100644 --- a/internal/versioning/version.go +++ b/internal/versioning/version.go @@ -5,5 +5,5 @@ var ( // XXX: Bump this value only when there are protocol changes that makes the oracle // incompatible between version! - ProtocolVersion = `v0.5.1.2` + ProtocolVersion = `v0.6.0` ) diff --git a/pkg/api/config.go b/pkg/api/config.go new file mode 100644 index 00000000..4ae6f362 --- /dev/null +++ b/pkg/api/config.go @@ -0,0 +1,22 @@ +package api + +import "time" + +// APIConfig contains configuration settings for the API +type APIConfig struct { + WorkerResponseTimeout time.Duration + // Add other API-specific configuration fields here +} + +var DefaultConfig = APIConfig{ + WorkerResponseTimeout: 60 * time.Second, + // Set default values for other fields here +} + +// LoadConfig loads the API configuration +// This can be expanded later to load from environment variables or a file +func LoadConfig() (*APIConfig, error) { + // For now, we'll just return the default config + config := DefaultConfig + return &config, nil +} diff --git a/pkg/api/handlers_data.go b/pkg/api/handlers_data.go index d53d4fc4..4e7e7504 100644 --- a/pkg/api/handlers_data.go +++ b/pkg/api/handlers_data.go @@ -26,6 +26,7 @@ import ( "github.com/masa-finance/masa-oracle/pkg/scrapers/discord" "github.com/masa-finance/masa-oracle/pkg/scrapers/telegram" "github.com/masa-finance/masa-oracle/pkg/workers" + "github.com/masa-finance/masa-oracle/pkg/workers/types" ) type LLMChat struct { @@ -42,7 +43,42 @@ func IsBase64(s string) bool { return err == nil } -// publishWorkRequest sends a work request to the PubSubManager for processing by a worker. +// SendWorkRequest sends a work request to a worker for processing. +// It marshals the request details into JSON and sends it over a libp2p stream. +// It is currently re-using the response channel map for this; however, it could be a simple synchronous call +// in which case the worker handlers would be responseible for preparing the data to be sent back to the client +// +// Parameters: +// - api: The API instance containing the Node and PubSubManager. +// - requestID: A unique identifier for the request. +// - workType: The type of work to be performed by the worker. +// - bodyBytes: The request body in byte slice format. +// +// Returns: +// - error: An error object if the request could not be sent or processed, otherwise nil. +func SendWorkRequest(api *API, requestID string, workType data_types.WorkerType, bodyBytes []byte, wg *sync.WaitGroup) error { + request := data_types.WorkRequest{ + WorkType: workType, + RequestId: requestID, + Data: bodyBytes, + } + response := workers.GetWorkHandlerManager().DistributeWork(api.Node, request) + responseChannel, exists := workers.GetResponseChannelMap().Get(requestID) + if !exists { + return fmt.Errorf("response channel not found") + } + select { + case responseChannel <- response: + wg.Add(1) + // Successfully sent JSON response to the response channel + default: + // Log an error if the channel is blocking for debugging purposes + logrus.Errorf("response channel is blocking for request ID: %s", requestID) + } + return nil +} + +// SendWorkRequest sends a work request to the PubSubManager for processing by a worker. // It marshals the request details into JSON and publishes it to the configured topic. // // Parameters: @@ -53,7 +89,7 @@ func IsBase64(s string) bool { // // Returns: // - error: An error object if the request could not be published, otherwise nil. -func publishWorkRequest(api *API, requestID string, request workers.WorkerType, bodyBytes []byte) error { +func publishWorkRequest(api *API, requestID string, request data_types.WorkerType, bodyBytes []byte) error { workRequest := map[string]string{ "request": string(request), "request_id": requestID, @@ -74,18 +110,24 @@ func publishWorkRequest(api *API, requestID string, request workers.WorkerType, // Parameters: // - c: The gin.Context object, which provides the context for the HTTP request. // - responseCh: A channel that receives the worker's response as a byte slice. -func handleWorkResponse(c *gin.Context, responseCh chan []byte) { +func handleWorkResponse(c *gin.Context, responseCh chan data_types.WorkResponse, wg *sync.WaitGroup) { + cfg, err := LoadConfig() + if err != nil { + logrus.Errorf("Failed to load API cfg: %v", err) + c.JSON(http.StatusInternalServerError, gin.H{"error": "Internal server error"}) + return + } + for { select { case response := <-responseCh: - var result map[string]interface{} - if err := json.Unmarshal(response, &result); err != nil { - c.JSON(http.StatusExpectationFailed, gin.H{"error": err.Error()}) + if response.Error != "" { + c.JSON(http.StatusExpectationFailed, response) + wg.Done() return } - - if data, ok := result["data"].(string); ok && IsBase64(data) { - decodedData, err := base64.StdEncoding.DecodeString(result["data"].(string)) + if data, ok := response.Data.(string); ok && IsBase64(data) { + decodedData, err := base64.StdEncoding.DecodeString(response.Data.(string)) if err != nil { c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to decode base64 data"}) return @@ -96,14 +138,14 @@ func handleWorkResponse(c *gin.Context, responseCh chan []byte) { c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to parse JSON data"}) return } - result["data"] = jsonData + response.Data = jsonData } - - c.JSON(http.StatusOK, result) + response.WorkRequest = nil + c.JSON(http.StatusOK, response) + wg.Done() return - // teslashibe: adjust to timeout after 10 seconds for performance testing - case <-time.After(10 * time.Second): - c.JSON(http.StatusGatewayTimeout, gin.H{"error": "Request timed out, check that port 4001 TCP inbound is open."}) + case <-time.After(cfg.WorkerResponseTimeout): + c.JSON(http.StatusGatewayTimeout, gin.H{"error": "Request timed out in API layer"}) return case <-c.Done(): return @@ -186,14 +228,16 @@ func (api *API) SearchTweetsAndAnalyzeSentiment() gin.HandlerFunc { c.JSON(http.StatusBadRequest, gin.H{"error": wErr.Error()}) } requestID := uuid.New().String() - responseCh := pubsub2.GetResponseChannelMap().CreateChannel(requestID) - defer pubsub2.GetResponseChannelMap().Delete(requestID) - wErr = publishWorkRequest(api, requestID, workers.WORKER.TwitterSentiment, bodyBytes) + responseCh := workers.GetResponseChannelMap().CreateChannel(requestID) + wg := &sync.WaitGroup{} + defer workers.GetResponseChannelMap().Delete(requestID) + go handleWorkResponse(c, responseCh, wg) + + wErr = SendWorkRequest(api, requestID, data_types.TwitterSentiment, bodyBytes, wg) if wErr != nil { c.JSON(http.StatusBadRequest, gin.H{"error": wErr.Error()}) } - handleWorkResponse(c, responseCh) - // worker handler implementation + wg.Wait() } } @@ -236,14 +280,18 @@ func (api *API) SearchDiscordMessagesAndAnalyzeSentiment() gin.HandlerFunc { return } requestID := uuid.New().String() - responseCh := pubsub2.GetResponseChannelMap().CreateChannel(requestID) - defer pubsub2.GetResponseChannelMap().Delete(requestID) - wErr = publishWorkRequest(api, requestID, workers.WORKER.DiscordSentiment, bodyBytes) + responseCh := workers.GetResponseChannelMap().CreateChannel(requestID) + wg := &sync.WaitGroup{} + defer workers.GetResponseChannelMap().Delete(requestID) + go handleWorkResponse(c, responseCh, wg) + + wErr = SendWorkRequest(api, requestID, data_types.DiscordSentiment, bodyBytes, wg) if wErr != nil { c.JSON(http.StatusBadRequest, gin.H{"error": wErr.Error()}) return } - handleWorkResponse(c, responseCh) + wg.Wait() + } } @@ -286,14 +334,17 @@ func (api *API) SearchTelegramMessagesAndAnalyzeSentiment() gin.HandlerFunc { return } requestID := uuid.New().String() - responseCh := pubsub2.GetResponseChannelMap().CreateChannel(requestID) - defer pubsub2.GetResponseChannelMap().Delete(requestID) - wErr = publishWorkRequest(api, requestID, workers.WORKER.TelegramSentiment, bodyBytes) + responseCh := workers.GetResponseChannelMap().CreateChannel(requestID) + wg := &sync.WaitGroup{} + defer workers.GetResponseChannelMap().Delete(requestID) + go handleWorkResponse(c, responseCh, wg) + + wErr = SendWorkRequest(api, requestID, data_types.TelegramSentiment, bodyBytes, wg) if wErr != nil { c.JSON(http.StatusBadRequest, gin.H{"error": wErr.Error()}) return } - handleWorkResponse(c, responseCh) + wg.Wait() } } @@ -338,15 +389,18 @@ func (api *API) SearchWebAndAnalyzeSentiment() gin.HandlerFunc { if wErr != nil { c.JSON(http.StatusBadRequest, gin.H{"error": wErr.Error()}) } + requestID := uuid.New().String() - responseCh := pubsub2.GetResponseChannelMap().CreateChannel(requestID) - defer pubsub2.GetResponseChannelMap().Delete(requestID) - wErr = publishWorkRequest(api, requestID, workers.WORKER.WebSentiment, bodyBytes) + responseCh := workers.GetResponseChannelMap().CreateChannel(requestID) + wg := &sync.WaitGroup{} + defer workers.GetResponseChannelMap().Delete(requestID) + go handleWorkResponse(c, responseCh, wg) + + wErr = SendWorkRequest(api, requestID, data_types.WebSentiment, bodyBytes, wg) if wErr != nil { c.JSON(http.StatusBadRequest, gin.H{"error": wErr.Error()}) } - handleWorkResponse(c, responseCh) - // worker handler implementation + wg.Wait() } } @@ -372,14 +426,16 @@ func (api *API) SearchTweetsProfile() gin.HandlerFunc { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) } requestID := uuid.New().String() - responseCh := pubsub2.GetResponseChannelMap().CreateChannel(requestID) - defer pubsub2.GetResponseChannelMap().Delete(requestID) - err = publishWorkRequest(api, requestID, workers.WORKER.TwitterProfile, bodyBytes) + responseCh := workers.GetResponseChannelMap().CreateChannel(requestID) + wg := &sync.WaitGroup{} + defer workers.GetResponseChannelMap().Delete(requestID) + go handleWorkResponse(c, responseCh, wg) + + err = SendWorkRequest(api, requestID, data_types.TwitterProfile, bodyBytes, wg) if err != nil { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) } - handleWorkResponse(c, responseCh) - // worker handler implementation + wg.Wait() } } @@ -407,14 +463,16 @@ func (api *API) SearchDiscordProfile() gin.HandlerFunc { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) } requestID := uuid.New().String() - responseCh := pubsub2.GetResponseChannelMap().CreateChannel(requestID) - defer pubsub2.GetResponseChannelMap().Delete(requestID) - err = publishWorkRequest(api, requestID, workers.WORKER.DiscordProfile, bodyBytes) + responseCh := workers.GetResponseChannelMap().CreateChannel(requestID) + wg := &sync.WaitGroup{} + defer workers.GetResponseChannelMap().Delete(requestID) + go handleWorkResponse(c, responseCh, wg) + + err = SendWorkRequest(api, requestID, data_types.DiscordProfile, bodyBytes, wg) if err != nil { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) } - handleWorkResponse(c, responseCh) - // worker handler implementation + wg.Wait() } } @@ -451,16 +509,17 @@ func (api *API) SearchChannelMessages() gin.HandlerFunc { } requestID := uuid.New().String() - responseCh := pubsub2.GetResponseChannelMap().CreateChannel(requestID) - defer pubsub2.GetResponseChannelMap().Delete(requestID) + responseCh := workers.GetResponseChannelMap().CreateChannel(requestID) + wg := &sync.WaitGroup{} + defer workers.GetResponseChannelMap().Delete(requestID) + go handleWorkResponse(c, responseCh, wg) - err = publishWorkRequest(api, requestID, workers.WORKER.DiscordChannelMessages, bodyBytes) + err = SendWorkRequest(api, requestID, data_types.DiscordChannelMessages, bodyBytes, wg) if err != nil { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) return } - - handleWorkResponse(c, responseCh) + wg.Wait() } } @@ -484,13 +543,16 @@ func (api *API) SearchGuildChannels() gin.HandlerFunc { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) } requestID := uuid.New().String() - responseCh := pubsub2.GetResponseChannelMap().CreateChannel(requestID) - defer pubsub2.GetResponseChannelMap().Delete(requestID) - err = publishWorkRequest(api, requestID, workers.WORKER.DiscordGuildChannels, bodyBytes) + responseCh := workers.GetResponseChannelMap().CreateChannel(requestID) + wg := &sync.WaitGroup{} + defer workers.GetResponseChannelMap().Delete(requestID) + go handleWorkResponse(c, responseCh, wg) + + err = SendWorkRequest(api, requestID, data_types.DiscordGuildChannels, bodyBytes, wg) if err != nil { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) } - handleWorkResponse(c, responseCh) + wg.Wait() } } @@ -505,13 +567,16 @@ func (api *API) SearchUserGuilds() gin.HandlerFunc { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) } requestID := uuid.New().String() - responseCh := pubsub2.GetResponseChannelMap().CreateChannel(requestID) - defer pubsub2.GetResponseChannelMap().Delete(requestID) - err = publishWorkRequest(api, requestID, workers.WORKER.DiscordUserGuilds, bodyBytes) + responseCh := workers.GetResponseChannelMap().CreateChannel(requestID) + wg := &sync.WaitGroup{} + defer workers.GetResponseChannelMap().Delete(requestID) + go handleWorkResponse(c, responseCh, wg) + + err = SendWorkRequest(api, requestID, data_types.DiscordUserGuilds, bodyBytes, wg) if err != nil { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) } - handleWorkResponse(c, responseCh) + wg.Wait() } } @@ -561,7 +626,12 @@ func (api *API) SearchAllGuilds() gin.HandlerFunc { return } - defer resp.Body.Close() + defer func(Body io.ReadCloser) { + err := Body.Close() + if err != nil { + logrus.Error("[-] Error closing response body: ", err) + } + }(resp.Body) respBody, err := io.ReadAll(resp.Body) if err != nil { errCh <- fmt.Errorf("[-] Failed to read response body: %v", err) @@ -665,15 +735,16 @@ func (api *API) SearchTwitterFollowers() gin.HandlerFunc { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) } requestID := uuid.New().String() - responseCh := pubsub2.GetResponseChannelMap().CreateChannel(requestID) - defer pubsub2.GetResponseChannelMap().Delete(requestID) - err = publishWorkRequest(api, requestID, workers.WORKER.TwitterFollowers, bodyBytes) + responseCh := workers.GetResponseChannelMap().CreateChannel(requestID) + wg := &sync.WaitGroup{} + defer workers.GetResponseChannelMap().Delete(requestID) + go handleWorkResponse(c, responseCh, wg) + + err = SendWorkRequest(api, requestID, data_types.TwitterFollowers, bodyBytes, wg) if err != nil { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) } - handleWorkResponse(c, responseCh) - // worker handler implementation - + wg.Wait() } } @@ -705,14 +776,16 @@ func (api *API) SearchTweetsRecent() gin.HandlerFunc { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) } requestID := uuid.New().String() - responseCh := pubsub2.GetResponseChannelMap().CreateChannel(requestID) - defer pubsub2.GetResponseChannelMap().Delete(requestID) - err = publishWorkRequest(api, requestID, workers.WORKER.Twitter, bodyBytes) + responseCh := workers.GetResponseChannelMap().CreateChannel(requestID) + wg := &sync.WaitGroup{} + defer workers.GetResponseChannelMap().Delete(requestID) + go handleWorkResponse(c, responseCh, wg) + + err = SendWorkRequest(api, requestID, data_types.Twitter, bodyBytes, wg) if err != nil { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) } - handleWorkResponse(c, responseCh) - // worker handler implementation + wg.Wait() } } @@ -724,14 +797,16 @@ func (api *API) SearchTweetsTrends() gin.HandlerFunc { return func(c *gin.Context) { // worker handler implementation requestID := uuid.New().String() - responseCh := pubsub2.GetResponseChannelMap().CreateChannel(requestID) - defer pubsub2.GetResponseChannelMap().Delete(requestID) - err := publishWorkRequest(api, requestID, workers.WORKER.TwitterTrends, nil) + responseCh := workers.GetResponseChannelMap().CreateChannel(requestID) + wg := &sync.WaitGroup{} + defer workers.GetResponseChannelMap().Delete(requestID) + go handleWorkResponse(c, responseCh, wg) + + err := SendWorkRequest(api, requestID, data_types.TwitterTrends, nil, wg) if err != nil { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) } - handleWorkResponse(c, responseCh) - // worker handler implementation + wg.Wait() } } @@ -770,14 +845,17 @@ func (api *API) WebData() gin.HandlerFunc { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) } requestID := uuid.New().String() - responseCh := pubsub2.GetResponseChannelMap().CreateChannel(requestID) - err = publishWorkRequest(api, requestID, workers.WORKER.Web, bodyBytes) - defer pubsub2.GetResponseChannelMap().Delete(requestID) + responseCh := workers.GetResponseChannelMap().CreateChannel(requestID) + wg := &sync.WaitGroup{} + defer workers.GetResponseChannelMap().Delete(requestID) + go handleWorkResponse(c, responseCh, wg) + + err = SendWorkRequest(api, requestID, data_types.Web, bodyBytes, wg) + defer workers.GetResponseChannelMap().Delete(requestID) if err != nil { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) } - handleWorkResponse(c, responseCh) - // worker handler implementation + wg.Wait() } } @@ -853,13 +931,16 @@ func (api *API) GetChannelMessagesHandler() gin.HandlerFunc { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) } requestID := uuid.New().String() - responseCh := pubsub2.GetResponseChannelMap().CreateChannel(requestID) - defer pubsub2.GetResponseChannelMap().Delete(requestID) - err = publishWorkRequest(api, requestID, workers.WORKER.TelegramChannelMessages, bodyBytes) + responseCh := workers.GetResponseChannelMap().CreateChannel(requestID) + wg := &sync.WaitGroup{} + defer workers.GetResponseChannelMap().Delete(requestID) + go handleWorkResponse(c, responseCh, wg) + + err = SendWorkRequest(api, requestID, data_types.TelegramChannelMessages, bodyBytes, wg) if err != nil { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) } - handleWorkResponse(c, responseCh) + wg.Wait() } } @@ -906,14 +987,16 @@ func (api *API) LocalLlmChat() gin.HandlerFunc { } requestID := uuid.New().String() - responseCh := pubsub2.GetResponseChannelMap().CreateChannel(requestID) - defer pubsub2.GetResponseChannelMap().Delete(requestID) - err = publishWorkRequest(api, requestID, workers.WORKER.LLMChat, bodyBytes) + responseCh := workers.GetResponseChannelMap().CreateChannel(requestID) + wg := &sync.WaitGroup{} + defer workers.GetResponseChannelMap().Delete(requestID) + go handleWorkResponse(c, responseCh, wg) + + err = SendWorkRequest(api, requestID, data_types.LLMChat, bodyBytes, wg) if err != nil { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) } - handleWorkResponse(c, responseCh) - // worker handler implementation + wg.Wait() } } @@ -991,7 +1074,12 @@ func (api *API) CfLlmChat() gin.HandlerFunc { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) return } - defer resp.Body.Close() + defer func(Body io.ReadCloser) { + err := Body.Close() + if err != nil { + logrus.Error("[-] Error closing response body: ", err) + } + }(resp.Body) respBody, err := io.ReadAll(resp.Body) if err != nil { logrus.Error("[-] Error reading response body: ", err) diff --git a/pkg/config/app.go b/pkg/config/app.go index 4b6f20ca..2e662f85 100644 --- a/pkg/config/app.go +++ b/pkg/config/app.go @@ -114,8 +114,8 @@ func GetInstance() *AppConfig { once.Do(func() { instance = &AppConfig{} - instance.setEnvVariableConfig() instance.setDefaultConfig() + instance.setEnvVariableConfig() instance.setFileConfig(viper.GetString("FILE_PATH")) err := instance.setCommandLineConfig() if err != nil { @@ -196,6 +196,7 @@ func (c *AppConfig) setCommandLineConfig() error { pflag.StringVar(&c.StakeAmount, "stake", viper.GetString(StakeAmount), "Amount of tokens to stake") pflag.BoolVar(&c.Debug, "debug", viper.GetBool(Debug), "Override some protections for debugging (temporary)") pflag.StringVar(&c.Environment, "env", viper.GetString(Environment), "Environment to connect to") + pflag.StringVar(&c.Version, "version", viper.GetString("VERSION"), "application version") pflag.BoolVar(&c.AllowedPeer, "allowedPeer", viper.GetBool(AllowedPeer), "Set to true to allow setting this node as the allowed peer") pflag.StringVar(&c.PrivateKey, "privateKey", viper.GetString(PrivateKey), "The private key") pflag.StringVar(&c.PrivateKeyFile, "privKeyFile", viper.GetString(PrivKeyFile), "The private key file") @@ -224,6 +225,7 @@ func (c *AppConfig) setCommandLineConfig() error { pflag.BoolVar(&c.WebScraper, "webScraper", viper.GetBool(WebScraper), "WebScraper") pflag.BoolVar(&c.LlmServer, "llmServer", viper.GetBool(LlmServer), "Can service LLM requests") pflag.BoolVar(&c.Faucet, "faucet", viper.GetBool(Faucet), "Faucet") + pflag.Parse() // Bind command line flags to Viper (optional, if you want to use Viper for additional configuration) diff --git a/pkg/config/constants.go b/pkg/config/constants.go index f1e49b5f..d3fd5a1e 100644 --- a/pkg/config/constants.go +++ b/pkg/config/constants.go @@ -100,6 +100,7 @@ const ( MasaPrefix = "/masa" OracleProtocol = "oracle_protocol" + WorkerProtocol = "worker_protocol" NodeDataSyncProtocol = "nodeDataSync" NodeGossipTopic = "gossip" PublicKeyTopic = "bootNodePublicKey" diff --git a/pkg/llmbridge/sentiment.go b/pkg/llmbridge/sentiment.go index e4384241..5f780567 100644 --- a/pkg/llmbridge/sentiment.go +++ b/pkg/llmbridge/sentiment.go @@ -10,10 +10,11 @@ import ( "strings" "github.com/gotd/td/tg" - "github.com/masa-finance/masa-oracle/pkg/config" twitterscraper "github.com/masa-finance/masa-twitter-scraper" "github.com/ollama/ollama/api" "github.com/sirupsen/logrus" + + "github.com/masa-finance/masa-oracle/pkg/config" ) // AnalyzeSentimentTweets analyzes the sentiment of the provided tweets by sending them to the Claude API. @@ -194,8 +195,8 @@ func AnalyzeSentimentWeb(data string, model string, prompt string) (string, stri genReq := api.ChatRequest{ Model: strings.TrimPrefix(model, "ollama/"), Messages: []api.Message{ - {Role: "user", Content: data}, {Role: "assistant", Content: prompt}, + {Role: "user", Content: data}, }, Stream: &stream, Options: map[string]interface{}{ diff --git a/pkg/masacrypto/hash.go b/pkg/masacrypto/hash.go new file mode 100644 index 00000000..76a07fac --- /dev/null +++ b/pkg/masacrypto/hash.go @@ -0,0 +1,33 @@ +package masacrypto + +import ( + "github.com/ipfs/go-cid" + mh "github.com/multiformats/go-multihash" + "github.com/sirupsen/logrus" +) + +// ComputeSha256Cid calculates the CID (Content Identifier) for a given string. +// +// Parameters: +// - str: The input string for which to compute the CID. +// +// Returns: +// - string: The computed CID as a string. +// - error: An error, if any occurred during the CID computation. +// +// The function uses the multihash package to create a SHA2-256 hash of the input string. +// It then creates a CID (version 1) from the multihash and returns the CID as a string. +// If an error occurs during the multihash computation or CID creation, it is returned. +func ComputeSha256Cid(str string) (string, error) { + logrus.Infof("Computing CID for string: %s", str) + // Create a multihash from the string + mhHash, err := mh.Sum([]byte(str), mh.SHA2_256, -1) + if err != nil { + logrus.Errorf("Error computing multihash for string: %s, error: %v", str, err) + return "", err + } + // Create a CID from the multihash + cidKey := cid.NewCidV1(cid.Raw, mhHash).String() + logrus.Infof("Computed CID: %s", cidKey) + return cidKey, nil +} diff --git a/pkg/network/discover.go b/pkg/network/discover.go index 95f6af1a..5c25c385 100644 --- a/pkg/network/discover.go +++ b/pkg/network/discover.go @@ -8,9 +8,10 @@ import ( "github.com/cenkalti/backoff/v4" "github.com/libp2p/go-libp2p/core/peer" - "github.com/masa-finance/masa-oracle/pkg/config" "github.com/multiformats/go-multiaddr" + "github.com/masa-finance/masa-oracle/pkg/config" + dht "github.com/libp2p/go-libp2p-kad-dht" "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/core/network" @@ -39,7 +40,7 @@ func Discover(ctx context.Context, host host.Host, dht *dht.IpfsDHT, protocol pr logrus.Infof("[+] Successfully advertised protocol %s", protocolString) } - ticker := time.NewTicker(time.Second * 10) + ticker := time.NewTicker(time.Minute * 1) defer ticker.Stop() var peerChan <-chan peer.AddrInfo diff --git a/pkg/network/kdht.go b/pkg/network/kdht.go index 9b6cd1a1..5bb8a78e 100644 --- a/pkg/network/kdht.go +++ b/pkg/network/kdht.go @@ -30,7 +30,7 @@ func (dbValidator) Validate(_ string, _ []byte) error { return nil } func (dbValidator) Select(_ string, _ [][]byte) (int, error) { return 0, nil } func WithDht(ctx context.Context, host host.Host, bootstrapNodes []multiaddr.Multiaddr, - protocolId, prefix protocol.ID, peerChan chan PeerEvent, isStaked bool, removePeerCallback func(peer.ID)) (*dht.IpfsDHT, error) { + protocolId, prefix protocol.ID, peerChan chan PeerEvent, isStaked bool) (*dht.IpfsDHT, error) { options := make([]dht.Option, 0) options = append(options, dht.BucketSize(100)) // Adjust bucket size options = append(options, dht.Concurrency(100)) // Increase concurrency @@ -46,8 +46,6 @@ func WithDht(ctx context.Context, host host.Host, bootstrapNodes []multiaddr.Mul go monitorRoutingTable(ctx, kademliaDHT, time.Minute) kademliaDHT.RoutingTable().PeerAdded = func(p peer.ID) { - logrus.Infof("[+] Peer added to DHT: %s", p.String()) - pe := PeerEvent{ AddrInfo: peer.AddrInfo{ID: p}, Action: PeerAdded, @@ -57,16 +55,12 @@ func WithDht(ctx context.Context, host host.Host, bootstrapNodes []multiaddr.Mul } kademliaDHT.RoutingTable().PeerRemoved = func(p peer.ID) { - logrus.Infof("[-] Peer removed from DHT: %s", p) pe := PeerEvent{ AddrInfo: peer.AddrInfo{ID: p}, Action: PeerRemoved, Source: "kdht", } peerChan <- pe - if removePeerCallback != nil { - removePeerCallback(p) - } } if err = kademliaDHT.Bootstrap(ctx); err != nil { @@ -127,7 +121,14 @@ func WithDht(ctx context.Context, host host.Host, bootstrapNodes []multiaddr.Mul logrus.Errorf("[-] Error closing stream: %s", err) } }(stream) // Close the stream when done - _, err = stream.Write(pubsub.GetSelfNodeDataJson(host, isStaked)) + + multiaddr, err := GetMultiAddressesForHost(host) + if err != nil { + logrus.Errorf("[-] Error getting multiaddresses for host: %s", err) + return + } + multaddrString := GetPriorityAddress(multiaddr) + _, err = stream.Write(pubsub.GetSelfNodeDataJson(host, isStaked, multaddrString.String())) if err != nil { logrus.Errorf("[-] Error writing to stream: %s", err) return @@ -153,10 +154,6 @@ func monitorRoutingTable(ctx context.Context, dht *dht.IpfsDHT, interval time.Du routingTable := dht.RoutingTable() // Log the size of the routing table logrus.Infof("[+] Routing table size: %d", routingTable.Size()) - // Log the peer IDs in the routing table - for _, p := range routingTable.ListPeers() { - logrus.Debugf("[+] Peer in routing table: %s", p.String()) - } case <-ctx.Done(): // If the context is cancelled, stop the goroutine return diff --git a/pkg/oracle_node.go b/pkg/oracle_node.go index 0ec8a3fe..1089c56f 100644 --- a/pkg/oracle_node.go +++ b/pkg/oracle_node.go @@ -7,8 +7,6 @@ import ( "encoding/base64" "encoding/json" "fmt" - "io" - "log/slog" "net" "os" "reflect" @@ -16,15 +14,10 @@ import ( "sync" "time" - "github.com/asynkron/protoactor-go/actor" - "github.com/asynkron/protoactor-go/remote" - "github.com/chyeh/pubip" - "github.com/libp2p/go-libp2p" dht "github.com/libp2p/go-libp2p-kad-dht" "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/core/network" - "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/protocol" rcmgr "github.com/libp2p/go-libp2p/p2p/host/resource-manager" "github.com/libp2p/go-libp2p/p2p/muxer/yamux" @@ -37,8 +30,9 @@ import ( shell "github.com/ipfs/go-ipfs-api" pubsub "github.com/libp2p/go-libp2p-pubsub" + "github.com/masa-finance/masa-oracle/internal/versioning" - chain "github.com/masa-finance/masa-oracle/pkg/chain" + "github.com/masa-finance/masa-oracle/pkg/chain" "github.com/masa-finance/masa-oracle/pkg/config" "github.com/masa-finance/masa-oracle/pkg/masacrypto" myNetwork "github.com/masa-finance/masa-oracle/pkg/network" @@ -67,8 +61,6 @@ type OracleNode struct { StartTime time.Time WorkerTracker *pubsub2.WorkerEventTracker BlockTracker *BlockEventTracker - ActorEngine *actor.RootContext - ActorRemote *remote.Remote Blockchain *chain.Chain } @@ -89,8 +81,14 @@ func getOutboundIP() string { conn, err := net.Dial("udp", "8.8.8.8:80") if err != nil { fmt.Println("[-] Error getting outbound IP") + return "" } - defer conn.Close() + defer func(conn net.Conn) { + err := conn.Close() + if err != nil { + + } + }(conn) localAddr := conn.LocalAddr().String() idx := strings.LastIndex(localAddr, ":") return localAddr[0:idx] @@ -151,27 +149,6 @@ func NewOracleNode(ctx context.Context, isStaked bool) (*OracleNode, error) { return nil, err } - system := actor.NewActorSystemWithConfig(actor.Configure( - actor.ConfigOption(func(config *actor.Config) { - config.LoggerFactory = func(system *actor.ActorSystem) *slog.Logger { - return slog.New(slog.NewTextHandler(io.Discard, nil)) - } - }), - )) - engine := system.Root - - var ip any - if cfg.Environment == "local" { - ip = getOutboundIP() - } else { - ip, _ = pubip.Get() - } - conf := remote.Configure("0.0.0.0", 4001, - remote.WithAdvertisedHost(fmt.Sprintf("%s:4001", ip))) - - r := remote.NewRemote(system, conf) - go r.Start() - return &OracleNode{ Host: hst, PrivKey: masacrypto.KeyManagerInstance().EcdsaPrivKey, @@ -188,8 +165,6 @@ func NewOracleNode(ctx context.Context, isStaked bool) (*OracleNode, error) { IsTelegramScraper: cfg.TelegramScraper, IsWebScraper: cfg.WebScraper, IsLlmServer: cfg.LlmServer, - ActorEngine: engine, - ActorRemote: r, Blockchain: &chain.Chain{}, }, nil } @@ -217,11 +192,7 @@ func (node *OracleNode) Start() (err error) { go node.ListenToNodeTracker() go node.handleDiscoveredPeers() - removePeerCallback := func(p peer.ID) { - node.NodeTracker.RemoveNodeData(p.String()) - } - - node.DHT, err = myNetwork.WithDht(node.Context, node.Host, bootNodeAddrs, node.Protocol, config.MasaPrefix, node.PeerChan, node.IsStaked, removePeerCallback) + node.DHT, err = myNetwork.WithDht(node.Context, node.Host, bootNodeAddrs, node.Protocol, config.MasaPrefix, node.PeerChan, node.IsStaked) if err != nil { return err } @@ -235,7 +206,8 @@ func (node *OracleNode) Start() (err error) { nodeData := node.NodeTracker.GetNodeData(node.Host.ID().String()) if nodeData == nil { publicKeyHex := masacrypto.KeyManagerInstance().EthAddress - nodeData = pubsub2.NewNodeData(node.GetMultiAddrs(), node.Host.ID(), publicKeyHex, pubsub2.ActivityJoined) + ma := myNetwork.GetMultiAddressesForHostQuiet(node.Host) + nodeData = pubsub2.NewNodeData(ma[0], node.Host.ID(), publicKeyHex, pubsub2.ActivityJoined) nodeData.IsStaked = node.IsStaked nodeData.SelfIdentified = true nodeData.IsDiscordScraper = node.IsDiscordScraper @@ -505,7 +477,12 @@ func SubscribeToBlocks(ctx context.Context, node *OracleNode) { return } - go node.Blockchain.Init() + go func() { + err := node.Blockchain.Init() + if err != nil { + logrus.Error(err) + } + }() updateTicker := time.NewTicker(time.Second * 60) defer updateTicker.Stop() diff --git a/pkg/oracle_node_listener.go b/pkg/oracle_node_listener.go index 06598aa3..228df490 100644 --- a/pkg/oracle_node_listener.go +++ b/pkg/oracle_node_listener.go @@ -146,7 +146,7 @@ func (node *OracleNode) SendNodeData(peerID peer.ID) { logrus.Debugf("[-] Failed to close stream: %v", err) } }(stream) // Ensure the stream is closed after sending the data - logrus.Infof("[+] Sending %d node data records to %s", totalRecords, peerID) + logrus.Debugf("[+] Sending %d node data records to %s", totalRecords, peerID) for pageNumber := 0; pageNumber < totalPages; pageNumber++ { node.SendNodeDataPage(nodeData, stream, pageNumber) } diff --git a/pkg/pubsub/node_data.go b/pkg/pubsub/node_data.go index 6d2bea25..65ae139d 100644 --- a/pkg/pubsub/node_data.go +++ b/pkg/pubsub/node_data.go @@ -46,6 +46,7 @@ func (m *JSONMultiaddr) UnmarshalJSON(b []byte) error { type NodeData struct { Multiaddrs []JSONMultiaddr `json:"multiaddrs,omitempty"` + MultiaddrsString string `json:"multiaddrsString,omitempty"` PeerId peer.ID `json:"peerId"` FirstJoinedUnix int64 `json:"firstJoined,omitempty"` LastJoinedUnix int64 `json:"lastJoined,omitempty"` @@ -79,6 +80,7 @@ func NewNodeData(addr multiaddr.Multiaddr, peerId peer.ID, publicKey string, act return &NodeData{ PeerId: peerId, Multiaddrs: multiaddrs, + MultiaddrsString: addr.String(), LastUpdatedUnix: time.Now().Unix(), CurrentUptime: 0, AccumulatedUptime: 0, @@ -92,6 +94,10 @@ func NewNodeData(addr multiaddr.Multiaddr, peerId peer.ID, publicKey string, act // and peer ID in the format "/ip4/127.0.0.1/tcp/4001/p2p/QmcgpsyWgH8Y8ajJz1Cu72KnS5uo2Aa2LpzU7kinSupNKC". // This can be used by other nodes to connect to this node. func (n *NodeData) Address() string { + // Add a check for empty addresses + if len(n.Multiaddrs) == 0 { + return "" + } return fmt.Sprintf("%s/p2p/%s", n.Multiaddrs[0].String(), n.PeerId.String()) } @@ -118,16 +124,18 @@ func (n *NodeData) CanDoWork(workerType WorkerCategory) bool { logrus.Infof("[+] Skipping worker %s due to timeout", n.PeerId) return false } - + if !(n.IsStaked && n.IsActive) { + return false + } switch workerType { case CategoryTwitter: - return n.IsActive && n.IsTwitterScraper + return n.IsTwitterScraper case CategoryDiscord: - return n.IsActive && n.IsDiscordScraper + return n.IsDiscordScraper case CategoryTelegram: - return n.IsActive && n.IsTelegramScraper + return n.IsTelegramScraper case CategoryWeb: - return n.IsActive && n.IsWebScraper + return n.IsWebScraper default: return false } @@ -251,10 +259,11 @@ func (n *NodeData) UpdateAccumulatedUptime() { // It populates a NodeData struct with the node's ID, staking status, and Ethereum address. // The NodeData struct is then marshalled into a JSON byte array. // Returns nil if there is an error marshalling to JSON. -func GetSelfNodeDataJson(host host.Host, isStaked bool) []byte { +func GetSelfNodeDataJson(host host.Host, isStaked bool, multiaddrString string) []byte { // Create and populate NodeData nodeData := NodeData{ PeerId: host.ID(), + MultiaddrsString: multiaddrString, IsStaked: isStaked, EthAddress: masacrypto.KeyManagerInstance().EthAddress, IsTwitterScraper: config.GetInstance().TwitterScraper, diff --git a/pkg/pubsub/node_event_tracker.go b/pkg/pubsub/node_event_tracker.go index f5402bda..c19f7f59 100644 --- a/pkg/pubsub/node_event_tracker.go +++ b/pkg/pubsub/node_event_tracker.go @@ -249,7 +249,7 @@ func (net *NodeEventTracker) GetUpdatedNodes(since time.Time) []NodeData { } // GetEthAddress returns the Ethereum address for the given remote peer. -// It gets the peer's public key from the network's peerstore, converts +// It gets the peer's public key from the network's peer store, converts // it to a hex string, and converts that to an Ethereum address. // Returns an empty string if there is no public key for the peer. func GetEthAddress(remotePeer peer.ID, n network.Network) string { @@ -273,6 +273,18 @@ func GetEthAddress(remotePeer peer.ID, n network.Network) string { return publicKeyHex } +// GetEligibleWorkerNodes returns a slice of NodeData for nodes that are eligible to perform a specific category of work. +func (net *NodeEventTracker) GetEligibleWorkerNodes(category WorkerCategory) []NodeData { + logrus.Debugf("Getting eligible worker nodes for category: %s", category) + result := make([]NodeData, 0) + for _, nodeData := range net.GetAllNodeData() { + if nodeData.CanDoWork(category) { + result = append(result, nodeData) + } + } + return result +} + // IsStaked returns whether the node with the given peerID is marked as staked in the node data tracker. // Returns false if no node data is found for the given peerID. func (net *NodeEventTracker) IsStaked(peerID string) bool { @@ -366,7 +378,7 @@ func (net *NodeEventTracker) AddOrUpdateNodeData(nodeData *NodeData, forceGossip // entry, and if expired, processes the connect and removes the entry. func (net *NodeEventTracker) ClearExpiredBufferEntries() { for { - time.Sleep(30 * time.Second) // E.g., every 5 seconds + time.Sleep(1 * time.Minute) now := time.Now() for peerID, entry := range net.ConnectBuffer { if now.Sub(entry.ConnectTime) > time.Minute*1 { @@ -388,11 +400,13 @@ func (net *NodeEventTracker) ClearExpiredBufferEntries() { // // Parameters: // - peerID: A string representing the ID of the peer to be removed. -func (net *NodeEventTracker) RemoveNodeData(peerID string) { - net.nodeData.Delete(peerID) - delete(net.ConnectBuffer, peerID) - logrus.Infof("[+] Removed peer %s from NodeTracker", peerID) -} +// +// TODO: we should never remove node data from the internal map. Otherwise we lose all tracking of activity. +//func (net *NodeEventTracker) RemoveNodeData(peerID string) { +// net.nodeData.Delete(peerID) +// delete(net.ConnectBuffer, peerID) +// logrus.Infof("[+] Removed peer %s from NodeTracker", peerID) +//} // ClearExpiredWorkerTimeouts periodically checks and clears expired worker timeouts. // It runs in an infinite loop, sleeping for 5 minutes between each iteration. @@ -445,7 +459,6 @@ func (net *NodeEventTracker) cleanupStalePeers(hostId string) { if now.Sub(time.Unix(nodeData.LastUpdatedUnix, 0)) > maxDisconnectionTime { if nodeData.PeerId.String() != hostId { logrus.Infof("Removing stale peer: %s", nodeData.PeerId) - net.RemoveNodeData(nodeData.PeerId.String()) delete(net.ConnectBuffer, nodeData.PeerId.String()) // Notify about peer removal diff --git a/pkg/scrapers/discord/discordprofile.go b/pkg/scrapers/discord/discordprofile.go index 0e392386..b66d54e9 100644 --- a/pkg/scrapers/discord/discordprofile.go +++ b/pkg/scrapers/discord/discordprofile.go @@ -1,4 +1,3 @@ -// discordprofile.go package discord import ( diff --git a/pkg/scrapers/twitter/tweets.go b/pkg/scrapers/twitter/tweets.go index 8348d6de..c45d26bd 100644 --- a/pkg/scrapers/twitter/tweets.go +++ b/pkg/scrapers/twitter/tweets.go @@ -7,10 +7,11 @@ import ( _ "github.com/lib/pq" - "github.com/masa-finance/masa-oracle/pkg/config" - "github.com/masa-finance/masa-oracle/pkg/llmbridge" twitterscraper "github.com/masa-finance/masa-twitter-scraper" "github.com/sirupsen/logrus" + + "github.com/masa-finance/masa-oracle/pkg/config" + "github.com/masa-finance/masa-oracle/pkg/llmbridge" ) type TweetResult struct { @@ -151,7 +152,12 @@ func ScrapeTweetsByQuery(query string, count int) ([]*TweetResult, error) { } tweets = append(tweets, &tweet) } - return tweets, tweets[0].Error + + if len(tweets) == 0 { + return nil, fmt.Errorf("no tweets found for the given query") + } + + return tweets, nil } // ScrapeTweetsByTrends scrapes the current trending topics on Twitter. diff --git a/pkg/workers/config.go b/pkg/workers/config.go new file mode 100644 index 00000000..7da59bef --- /dev/null +++ b/pkg/workers/config.go @@ -0,0 +1,43 @@ +package workers + +import ( + "time" + + "github.com/sirupsen/logrus" +) + +type WorkerConfig struct { + WorkerTimeout time.Duration + WorkerResponseTimeout time.Duration + ConnectionTimeout time.Duration + MaxRetries int + MaxSpawnAttempts int + WorkerBufferSize int + MaxRemoteWorkers int +} + +var DefaultConfig = WorkerConfig{ + WorkerTimeout: 55 * time.Second, + WorkerResponseTimeout: 30 * time.Second, + ConnectionTimeout: 1 * time.Second, + MaxRetries: 1, + MaxSpawnAttempts: 1, + WorkerBufferSize: 100, + MaxRemoteWorkers: 1, +} + +var workerConfig *WorkerConfig + +func init() { + var err error + workerConfig, err = LoadConfig() + if err != nil { + logrus.Fatalf("Failed to load worker config: %v", err) + } +} + +func LoadConfig() (*WorkerConfig, error) { + // For now, we'll just return the default config + config := DefaultConfig + return &config, nil +} diff --git a/pkg/workers/handler.go b/pkg/workers/handler.go deleted file mode 100644 index c7bf94e0..00000000 --- a/pkg/workers/handler.go +++ /dev/null @@ -1,219 +0,0 @@ -package workers - -import ( - "context" - "encoding/json" - "fmt" - "net" - "strings" - - "github.com/asynkron/protoactor-go/actor" - pubsub2 "github.com/libp2p/go-libp2p-pubsub" - masa "github.com/masa-finance/masa-oracle/pkg" - "github.com/masa-finance/masa-oracle/pkg/config" - "github.com/masa-finance/masa-oracle/pkg/scrapers/discord" - "github.com/masa-finance/masa-oracle/pkg/scrapers/telegram" - "github.com/masa-finance/masa-oracle/pkg/scrapers/twitter" - "github.com/masa-finance/masa-oracle/pkg/scrapers/web" - "github.com/masa-finance/masa-oracle/pkg/workers/messages" - "github.com/multiformats/go-multiaddr" - "github.com/sirupsen/logrus" -) - -type LLMChatBody struct { - Model string `json:"model,omitempty"` - Messages []struct { - Role string `json:"role"` - Content string `json:"content"` - } `json:"messages,omitempty"` - Stream bool `json:"stream"` -} - -// getPeers is a function that takes an OracleNode as an argument and returns a slice of actor.PID pointers. -// These actor.PID pointers represent the peers of the given OracleNode in the network. -func getPeers(node *masa.OracleNode) []*actor.PID { - var actors []*actor.PID - peers := node.Host.Network().Peers() - for _, p := range peers { - conns := node.Host.Network().ConnsToPeer(p) - for _, conn := range conns { - addr := conn.RemoteMultiaddr() - ipAddr, _ := addr.ValueForProtocol(multiaddr.P_IP4) - if p.String() != node.Host.ID().String() { - spawned, err := node.ActorRemote.SpawnNamed(fmt.Sprintf("%s:4001", ipAddr), "worker", "peer", -1) - if err != nil { - if strings.Contains(err.Error(), "future: dead letter") { - logrus.Debugf("Ignoring dead letter error for peer %s: %v", p.String(), err) - continue - } - logrus.Debugf("Spawned error %v", err) - } else { - actors = append(actors, spawned.Pid) - } - } - } - } - return actors -} - -// HandleConnect is a method of the Worker struct that handles the connection of a worker. -// It takes in an actor context and a Connect message as parameters. -func (a *Worker) HandleConnect(ctx actor.Context, m *messages.Connect) { - logrus.Infof("[+] Worker %v connected", m.Sender) - clients.Add(m.Sender) -} - -// HandleLog is a method of the Worker struct that handles logging. -// It takes in an actor context and a string message as parameters. -func (a *Worker) HandleLog(ctx actor.Context, l string) { - logrus.Info(l) -} - -// HandleWork is a method of the Worker struct that handles the work assigned to a worker. -// It takes in an actor context and a Work message as parameters. -func (a *Worker) HandleWork(ctx actor.Context, m *messages.Work, node *masa.OracleNode) { - var resp interface{} - var err error - - var workData map[string]string - err = json.Unmarshal([]byte(m.Data), &workData) - if err != nil { - logrus.Errorf("[-] Error parsing work data: %v", err) - return - } - - var bodyData map[string]interface{} - if workData["body"] != "" { - if err := json.Unmarshal([]byte(workData["body"]), &bodyData); err != nil { - logrus.Errorf("[-] Error unmarshalling body: %v", err) - return - } - } - - switch workData["request"] { - case string(WORKER.DiscordProfile): - userID := bodyData["userID"].(string) - resp, err = discord.GetUserProfile(userID) - case string(WORKER.DiscordChannelMessages): - channelID := bodyData["channelID"].(string) - resp, err = discord.GetChannelMessages(channelID, bodyData["limit"].(string), bodyData["before"].(string)) - case string(WORKER.DiscordSentiment): - logrus.Infof("[+] Discord Channel Messages %s %s", m.Data, m.Sender) - channelID := bodyData["channelID"].(string) - _, resp, err = discord.ScrapeDiscordMessagesForSentiment(channelID, bodyData["model"].(string), bodyData["prompt"].(string)) - case string(WORKER.TelegramChannelMessages): - logrus.Infof("[+] Telegram Channel Messages %s %s", m.Data, m.Sender) - username := bodyData["username"].(string) - resp, err = telegram.FetchChannelMessages(context.Background(), username) // Removed the underscore placeholder - case string(WORKER.TelegramSentiment): - logrus.Infof("[+] Telegram Channel Messages %s %s", m.Data, m.Sender) - username := bodyData["username"].(string) - _, resp, err = telegram.ScrapeTelegramMessagesForSentiment(context.Background(), username, bodyData["model"].(string), bodyData["prompt"].(string)) - case string(WORKER.DiscordGuildChannels): - guildID := bodyData["guildID"].(string) - resp, err = discord.GetGuildChannels(guildID) - case string(WORKER.DiscordUserGuilds): - resp, err = discord.GetUserGuilds() - case string(WORKER.LLMChat): - uri := config.GetInstance().LLMChatUrl - if uri == "" { - logrus.Error("[-] Missing env variable LLM_CHAT_URL") - return - } - bodyBytes, _ := json.Marshal(bodyData) - headers := map[string]string{ - "Content-Type": "application/json", - } - resp, _ = Post(uri, bodyBytes, headers) - case string(WORKER.Twitter): - query := bodyData["query"].(string) - count := int(bodyData["count"].(float64)) - resp, err = twitter.ScrapeTweetsByQuery(query, count) - case string(WORKER.TwitterFollowers): - username := bodyData["username"].(string) - count := int(bodyData["count"].(float64)) - resp, err = twitter.ScrapeFollowersForProfile(username, count) - case string(WORKER.TwitterProfile): - username := bodyData["username"].(string) - resp, err = twitter.ScrapeTweetsProfile(username) - case string(WORKER.TwitterSentiment): - count := int(bodyData["count"].(float64)) - _, resp, err = twitter.ScrapeTweetsForSentiment(bodyData["query"].(string), count, bodyData["model"].(string)) - case string(WORKER.TwitterTrends): - resp, err = twitter.ScrapeTweetsByTrends() - case string(WORKER.Web): - depth := int(bodyData["depth"].(float64)) - resp, err = web.ScrapeWebData([]string{bodyData["url"].(string)}, depth) - case string(WORKER.WebSentiment): - depth := int(bodyData["depth"].(float64)) - _, resp, err = web.ScrapeWebDataForSentiment([]string{bodyData["url"].(string)}, depth, bodyData["model"].(string)) - case string(WORKER.Test): - count := int(bodyData["count"].(float64)) - resp, err = func(count int) (interface{}, error) { - return count, err - }(count) - default: - logrus.Warningf("[+] Received unknown message: %T", m) - return - } - - if err != nil { - host, _, err := net.SplitHostPort(m.Sender.Address) - addrs := node.Host.Addrs() - isLocalHost := false - for _, addr := range addrs { - addrStr := addr.String() - if strings.HasPrefix(addrStr, "/ip4/") { - ipStr := strings.Split(strings.Split(addrStr, "/")[2], "/")[0] - if host == ipStr { - isLocalHost = true - break - } - } - } - - if isLocalHost { - logrus.Errorf("[-] Local node: Error processing request: %s", err.Error()) - } else { - logrus.Errorf("[-] Remote node %s: Error processing request: %s", m.Sender, err.Error()) - } - - chanResponse := ChanResponse{ - Response: map[string]interface{}{"error": err.Error()}, - ChannelId: workData["request_id"], - } - val := &pubsub2.Message{ - ValidatorData: chanResponse, - ID: m.Id, - } - jsn, err := json.Marshal(val) - if err != nil { - logrus.Errorf("[-] Error marshalling response: %v", err) - return - } - ctx.Respond(&messages.Response{RequestId: workData["request_id"], Value: string(jsn)}) - } else { - chanResponse := ChanResponse{ - Response: map[string]interface{}{"data": resp}, - ChannelId: workData["request_id"], - } - val := &pubsub2.Message{ - ValidatorData: chanResponse, - ID: m.Id, - } - jsn, err := json.Marshal(val) - if err != nil { - logrus.Errorf("[-] Error marshalling response: %v", err) - return - } - cfg := config.GetInstance() - - if cfg.TwitterScraper || cfg.DiscordScraper || cfg.TelegramScraper || cfg.WebScraper { - ctx.Respond(&messages.Response{RequestId: workData["request_id"], Value: string(jsn)}) - } - for _, pid := range getPeers(node) { - ctx.Send(pid, &messages.Response{RequestId: workData["request_id"], Value: string(jsn)}) - } - } - ctx.Poison(ctx.Self()) -} diff --git a/pkg/workers/handlers/discord.go b/pkg/workers/handlers/discord.go new file mode 100644 index 00000000..4dfd3d14 --- /dev/null +++ b/pkg/workers/handlers/discord.go @@ -0,0 +1,89 @@ +package handlers + +import ( + "fmt" + + "github.com/sirupsen/logrus" + + "github.com/masa-finance/masa-oracle/pkg/scrapers/discord" + "github.com/masa-finance/masa-oracle/pkg/workers/types" +) + +type DiscordProfileHandler struct{} +type DiscordChannelHandler struct{} +type DiscordSentimentHandler struct{} +type DiscordGuildHandler struct{} +type DiscoreUserGuildsHandler struct{} + +// HandleWork implements the WorkHandler interface for DiscordProfileHandler. +func (h *DiscordProfileHandler) HandleWork(data []byte) data_types.WorkResponse { + logrus.Infof("[+] DiscordProfileHandler %s", data) + dataMap, err := JsonBytesToMap(data) + if err != nil { + return data_types.WorkResponse{Error: fmt.Sprintf("unable to parse discord json data: %v", err)} + } + userID := dataMap["userID"].(string) + resp, err := discord.GetUserProfile(userID) + if err != nil { + return data_types.WorkResponse{Data: resp, Error: fmt.Sprintf("unable to get discord user profile: %v", err)} + } + return data_types.WorkResponse{Data: resp} +} + +// HandleWork implements the WorkHandler interface for DiscordChannelHandler. +func (h *DiscordChannelHandler) HandleWork(data []byte) data_types.WorkResponse { + logrus.Infof("[+] DiscordChannelHandler %s", data) + dataMap, err := JsonBytesToMap(data) + if err != nil { + return data_types.WorkResponse{Error: fmt.Sprintf("unable to parse discord json data: %v", err)} + } + channelID := dataMap["channelID"].(string) + limit := dataMap["limit"].(string) + prompt := dataMap["prompt"].(string) + resp, err := discord.GetChannelMessages(channelID, limit, prompt) + if err != nil { + return data_types.WorkResponse{Error: fmt.Sprintf("unable to get discord channel messages: %v", err)} + } + return data_types.WorkResponse{Data: resp} +} + +// HandleWork implements the WorkHandler interface for DiscordSentimentHandler. +func (h *DiscordSentimentHandler) HandleWork(data []byte) data_types.WorkResponse { + logrus.Infof("[+] DiscordSentimentHandler %s", data) + dataMap, err := JsonBytesToMap(data) + if err != nil { + return data_types.WorkResponse{Error: fmt.Sprintf("unable to parse discord json data: %v", err)} + } + channelID := dataMap["channelID"].(string) + model := dataMap["model"].(string) + prompt := dataMap["prompt"].(string) + _, resp, err := discord.ScrapeDiscordMessagesForSentiment(channelID, model, prompt) + if err != nil { + return data_types.WorkResponse{Error: fmt.Sprintf("unable to get discord channel messages: %v", err)} + } + return data_types.WorkResponse{Data: resp} +} + +// HandleWork implements the WorkHandler interface for DiscordGuildHandler. +func (h *DiscordGuildHandler) HandleWork(data []byte) data_types.WorkResponse { + logrus.Infof("[+] DiscordGuildHandler %s", data) + dataMap, err := JsonBytesToMap(data) + if err != nil { + return data_types.WorkResponse{Error: fmt.Sprintf("unable to parse discord json data: %v", err)} + } + guildID := dataMap["guildID"].(string) + resp, err := discord.GetGuildChannels(guildID) + if err != nil { + return data_types.WorkResponse{Error: fmt.Sprintf("unable to get discord guild channels: %v", err)} + } + return data_types.WorkResponse{Data: resp} +} + +func (h *DiscoreUserGuildsHandler) HandleWork(data []byte) data_types.WorkResponse { + logrus.Infof("[+] DiscordUserGuildsHandler %s", data) + resp, err := discord.GetUserGuilds() + if err != nil { + return data_types.WorkResponse{Error: fmt.Sprintf("unable to get discord user guilds: %v", err)} + } + return data_types.WorkResponse{Data: resp} +} diff --git a/pkg/workers/handlers/helper.go b/pkg/workers/handlers/helper.go new file mode 100644 index 00000000..331dd716 --- /dev/null +++ b/pkg/workers/handlers/helper.go @@ -0,0 +1,14 @@ +package handlers + +import ( + "encoding/json" +) + +func JsonBytesToMap(jsonBytes []byte) (map[string]interface{}, error) { + var jsonMap map[string]interface{} + err := json.Unmarshal(jsonBytes, &jsonMap) + if err != nil { + return nil, err + } + return jsonMap, nil +} diff --git a/pkg/workers/client.go b/pkg/workers/handlers/http_client.go similarity index 99% rename from pkg/workers/client.go rename to pkg/workers/handlers/http_client.go index b4e9f16f..50f873b2 100644 --- a/pkg/workers/client.go +++ b/pkg/workers/handlers/http_client.go @@ -1,4 +1,4 @@ -package workers +package handlers import ( "bytes" diff --git a/pkg/workers/handlers/llm.go b/pkg/workers/handlers/llm.go new file mode 100644 index 00000000..658fcf77 --- /dev/null +++ b/pkg/workers/handlers/llm.go @@ -0,0 +1,48 @@ +package handlers + +import ( + "encoding/json" + "fmt" + + "github.com/sirupsen/logrus" + + "github.com/masa-finance/masa-oracle/pkg/config" + "github.com/masa-finance/masa-oracle/pkg/workers/types" +) + +// TODO: LLMChatBody isn't used anywhere in the codebase. Remove after testing +type LLMChatBody struct { + Model string `json:"model,omitempty"` + Messages []struct { + Role string `json:"role"` + Content string `json:"content"` + } `json:"messages,omitempty"` + Stream bool `json:"stream"` +} + +type LLMChatHandler struct{} + +// HandleWork implements the WorkHandler interface for LLMChatHandler. +// It contains the logic for processing LLMChat work. +func (h *LLMChatHandler) HandleWork(data []byte) data_types.WorkResponse { + logrus.Infof("[+] LLM Chat %s", data) + uri := config.GetInstance().LLMChatUrl + if uri == "" { + return data_types.WorkResponse{Error: "missing env variable LLM_CHAT_URL"} + } + + var dataMap map[string]interface{} + if err := json.Unmarshal(data, &dataMap); err != nil { + return data_types.WorkResponse{Error: fmt.Sprintf("unable to parse LLM chat data: %v", err)} + } + + jsnBytes, err := json.Marshal(dataMap) + if err != nil { + return data_types.WorkResponse{Error: fmt.Sprintf("unable to marshal LLM chat data: %v", err)} + } + resp, err := Post(uri, jsnBytes, nil) + if err != nil { + return data_types.WorkResponse{Error: fmt.Sprintf("unable to post LLM chat data: %v", err)} + } + return data_types.WorkResponse{Data: resp} +} diff --git a/pkg/workers/handlers/telegram.go b/pkg/workers/handlers/telegram.go new file mode 100644 index 00000000..234e3ee0 --- /dev/null +++ b/pkg/workers/handlers/telegram.go @@ -0,0 +1,46 @@ +package handlers + +import ( + "context" + "fmt" + + "github.com/sirupsen/logrus" + + "github.com/masa-finance/masa-oracle/pkg/scrapers/telegram" + "github.com/masa-finance/masa-oracle/pkg/workers/types" +) + +type TelegramSentimentHandler struct{} +type TelegramChannelHandler struct{} + +// HandleWork implements the WorkHandler interface for TelegramSentimentHandler. +func (h *TelegramSentimentHandler) HandleWork(data []byte) data_types.WorkResponse { + logrus.Infof("[+] TelegramSentimentHandler %s", data) + dataMap, err := JsonBytesToMap(data) + if err != nil { + return data_types.WorkResponse{Error: fmt.Sprintf("unable to parse telegram json data: %v", err)} + } + userName := dataMap["username"].(string) + model := dataMap["model"].(string) + prompt := dataMap["prompt"].(string) + _, resp, err := telegram.ScrapeTelegramMessagesForSentiment(context.Background(), userName, model, prompt) + if err != nil { + return data_types.WorkResponse{Error: fmt.Sprintf("unable to get telegram sentiment: %v", err)} + } + return data_types.WorkResponse{Data: resp} +} + +// HandleWork implements the WorkHandler interface for TelegramChannelHandler. +func (h *TelegramChannelHandler) HandleWork(data []byte) data_types.WorkResponse { + logrus.Infof("[+] TelegramChannelHandler %s", data) + dataMap, err := JsonBytesToMap(data) + if err != nil { + return data_types.WorkResponse{Error: fmt.Sprintf("unable to parse telegram json data: %v", err)} + } + userName := dataMap["username"].(string) + resp, err := telegram.FetchChannelMessages(context.Background(), userName) + if err != nil { + return data_types.WorkResponse{Error: fmt.Sprintf("unable to get telegram channel messages: %v", err)} + } + return data_types.WorkResponse{Data: resp} +} diff --git a/pkg/workers/handlers/twitter.go b/pkg/workers/handlers/twitter.go new file mode 100644 index 00000000..6db38d2d --- /dev/null +++ b/pkg/workers/handlers/twitter.go @@ -0,0 +1,85 @@ +package handlers + +import ( + "fmt" + + "github.com/sirupsen/logrus" + + "github.com/masa-finance/masa-oracle/pkg/scrapers/twitter" + "github.com/masa-finance/masa-oracle/pkg/workers/types" +) + +type TwitterQueryHandler struct{} +type TwitterFollowersHandler struct{} +type TwitterProfileHandler struct{} +type TwitterSentimentHandler struct{} +type TwitterTrendsHandler struct{} + +func (h *TwitterQueryHandler) HandleWork(data []byte) data_types.WorkResponse { + logrus.Infof("[+] TwitterQueryHandler %s", data) + dataMap, err := JsonBytesToMap(data) + if err != nil { + return data_types.WorkResponse{Error: fmt.Sprintf("unable to parse twitter query data: %v", err)} + } + count := int(dataMap["count"].(float64)) + query := dataMap["query"].(string) + resp, err := twitter.ScrapeTweetsByQuery(query, count) + if err != nil { + return data_types.WorkResponse{Error: fmt.Sprintf("unable to get twitter query: %v", err)} + } + return data_types.WorkResponse{Data: resp} +} + +func (h *TwitterFollowersHandler) HandleWork(data []byte) data_types.WorkResponse { + logrus.Infof("[+] TwitterFollowersHandler %s", data) + dataMap, err := JsonBytesToMap(data) + if err != nil { + return data_types.WorkResponse{Error: fmt.Sprintf("unable to parse twitter followers data: %v", err)} + } + username := dataMap["username"].(string) + count := int(dataMap["count"].(float64)) + resp, err := twitter.ScrapeFollowersForProfile(username, count) + if err != nil { + return data_types.WorkResponse{Error: fmt.Sprintf("unable to get twitter followers: %v", err)} + } + return data_types.WorkResponse{Data: resp} +} + +func (h *TwitterProfileHandler) HandleWork(data []byte) data_types.WorkResponse { + logrus.Infof("[+] TwitterProfileHandler %s", data) + dataMap, err := JsonBytesToMap(data) + if err != nil { + return data_types.WorkResponse{Error: fmt.Sprintf("unable to parse twitter profile data: %v", err)} + } + username := dataMap["username"].(string) + resp, err := twitter.ScrapeTweetsProfile(username) + if err != nil { + return data_types.WorkResponse{Error: fmt.Sprintf("unable to get twitter profile: %v", err)} + } + return data_types.WorkResponse{Data: resp} +} + +func (h *TwitterSentimentHandler) HandleWork(data []byte) data_types.WorkResponse { + logrus.Infof("[+] TwitterSentimentHandler %s", data) + dataMap, err := JsonBytesToMap(data) + if err != nil { + return data_types.WorkResponse{Error: fmt.Sprintf("unable to parse twitter sentiment data: %v", err)} + } + count := int(dataMap["count"].(float64)) + query := dataMap["query"].(string) + model := dataMap["model"].(string) + _, resp, err := twitter.ScrapeTweetsForSentiment(query, count, model) + if err != nil { + return data_types.WorkResponse{Error: fmt.Sprintf("unable to get twitter sentiment: %v", err)} + } + return data_types.WorkResponse{Data: resp} +} + +func (h *TwitterTrendsHandler) HandleWork(data []byte) data_types.WorkResponse { + logrus.Infof("[+] TwitterTrendsHandler %s", data) + resp, err := twitter.ScrapeTweetsByTrends() + if err != nil { + return data_types.WorkResponse{Error: fmt.Sprintf("unable to get twitter trends: %v", err)} + } + return data_types.WorkResponse{Data: resp} +} diff --git a/pkg/workers/handlers/web.go b/pkg/workers/handlers/web.go new file mode 100644 index 00000000..70c2ef3b --- /dev/null +++ b/pkg/workers/handlers/web.go @@ -0,0 +1,45 @@ +package handlers + +import ( + "fmt" + + "github.com/sirupsen/logrus" + + "github.com/masa-finance/masa-oracle/pkg/scrapers/web" + "github.com/masa-finance/masa-oracle/pkg/workers/types" +) + +// WebHandler - All the web handlers implement the WorkHandler interface. +type WebHandler struct{} +type WebSentimentHandler struct{} + +func (h *WebHandler) HandleWork(data []byte) data_types.WorkResponse { + logrus.Infof("[+] WebHandler %s", data) + dataMap, err := JsonBytesToMap(data) + if err != nil { + return data_types.WorkResponse{Error: fmt.Sprintf("unable to parse web data: %v", err)} + } + depth := int(dataMap["depth"].(float64)) + urls := []string{dataMap["url"].(string)} + resp, err := web.ScrapeWebData(urls, depth) + if err != nil { + return data_types.WorkResponse{Error: fmt.Sprintf("unable to get web data: %v", err)} + } + return data_types.WorkResponse{Data: resp} +} + +func (h *WebSentimentHandler) HandleWork(data []byte) data_types.WorkResponse { + logrus.Infof("[+] WebSentimentHandler %s", data) + dataMap, err := JsonBytesToMap(data) + if err != nil { + return data_types.WorkResponse{Error: fmt.Sprintf("unable to parse web sentiment data: %v", err)} + } + depth := int(dataMap["depth"].(float64)) + urls := []string{dataMap["url"].(string)} + model := dataMap["model"].(string) + _, resp, err := web.ScrapeWebDataForSentiment(urls, depth, model) + if err != nil { + return data_types.WorkResponse{Error: fmt.Sprintf("unable to get web sentiment: %v", err)} + } + return data_types.WorkResponse{Data: resp} +} diff --git a/pkg/workers/messages/build.sh b/pkg/workers/messages/build.sh deleted file mode 100755 index 279c821b..00000000 --- a/pkg/workers/messages/build.sh +++ /dev/null @@ -1,2 +0,0 @@ -protoc -I="/Users/john/Projects/masa/protoactor-go/actor" --go_out=. --go_opt=paths=source_relative --proto_path=. protos.proto - diff --git a/pkg/workers/messages/protos.pb.go b/pkg/workers/messages/protos.pb.go deleted file mode 100644 index b7fd1348..00000000 --- a/pkg/workers/messages/protos.pb.go +++ /dev/null @@ -1,374 +0,0 @@ -// Code generated by protoc-gen-go. DO NOT EDIT. -// versions: -// protoc-gen-go v1.33.0 -// protoc v5.26.1 -// source: protos.proto - -package messages - -import ( - actor "github.com/asynkron/protoactor-go/actor" - protoreflect "google.golang.org/protobuf/reflect/protoreflect" - protoimpl "google.golang.org/protobuf/runtime/protoimpl" - reflect "reflect" - sync "sync" -) - -const ( - // Verify that this generated code is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) - // Verify that runtime/protoimpl is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) -) - -type Connect struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Sender *actor.PID `protobuf:"bytes,1,opt,name=Sender,proto3" json:"Sender,omitempty"` -} - -func (x *Connect) Reset() { - *x = Connect{} - if protoimpl.UnsafeEnabled { - mi := &file_protos_proto_msgTypes[0] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *Connect) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*Connect) ProtoMessage() {} - -func (x *Connect) ProtoReflect() protoreflect.Message { - mi := &file_protos_proto_msgTypes[0] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use Connect.ProtoReflect.Descriptor instead. -func (*Connect) Descriptor() ([]byte, []int) { - return file_protos_proto_rawDescGZIP(), []int{0} -} - -func (x *Connect) GetSender() *actor.PID { - if x != nil { - return x.Sender - } - return nil -} - -type Connected struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Message string `protobuf:"bytes,1,opt,name=Message,proto3" json:"Message,omitempty"` -} - -func (x *Connected) Reset() { - *x = Connected{} - if protoimpl.UnsafeEnabled { - mi := &file_protos_proto_msgTypes[1] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *Connected) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*Connected) ProtoMessage() {} - -func (x *Connected) ProtoReflect() protoreflect.Message { - mi := &file_protos_proto_msgTypes[1] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use Connected.ProtoReflect.Descriptor instead. -func (*Connected) Descriptor() ([]byte, []int) { - return file_protos_proto_rawDescGZIP(), []int{1} -} - -func (x *Connected) GetMessage() string { - if x != nil { - return x.Message - } - return "" -} - -type Work struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Sender *actor.PID `protobuf:"bytes,1,opt,name=Sender,proto3" json:"Sender,omitempty"` - Data string `protobuf:"bytes,2,opt,name=Data,proto3" json:"Data,omitempty"` - Id string `protobuf:"bytes,3,opt,name=Id,proto3" json:"Id,omitempty"` - Type int64 `protobuf:"varint,4,opt,name=Type,proto3" json:"Type,omitempty"` -} - -func (x *Work) Reset() { - *x = Work{} - if protoimpl.UnsafeEnabled { - mi := &file_protos_proto_msgTypes[2] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *Work) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*Work) ProtoMessage() {} - -func (x *Work) ProtoReflect() protoreflect.Message { - mi := &file_protos_proto_msgTypes[2] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use Work.ProtoReflect.Descriptor instead. -func (*Work) Descriptor() ([]byte, []int) { - return file_protos_proto_rawDescGZIP(), []int{2} -} - -func (x *Work) GetSender() *actor.PID { - if x != nil { - return x.Sender - } - return nil -} - -func (x *Work) GetData() string { - if x != nil { - return x.Data - } - return "" -} - -func (x *Work) GetId() string { - if x != nil { - return x.Id - } - return "" -} - -func (x *Work) GetType() int64 { - if x != nil { - return x.Type - } - return 0 -} - -type Response struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Value string `protobuf:"bytes,1,opt,name=Value,proto3" json:"Value,omitempty"` - RequestId string `protobuf:"bytes,2,opt,name=RequestId,proto3" json:"RequestId,omitempty"` -} - -func (x *Response) Reset() { - *x = Response{} - if protoimpl.UnsafeEnabled { - mi := &file_protos_proto_msgTypes[3] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *Response) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*Response) ProtoMessage() {} - -func (x *Response) ProtoReflect() protoreflect.Message { - mi := &file_protos_proto_msgTypes[3] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use Response.ProtoReflect.Descriptor instead. -func (*Response) Descriptor() ([]byte, []int) { - return file_protos_proto_rawDescGZIP(), []int{3} -} - -func (x *Response) GetValue() string { - if x != nil { - return x.Value - } - return "" -} - -func (x *Response) GetRequestId() string { - if x != nil { - return x.RequestId - } - return "" -} - -var File_protos_proto protoreflect.FileDescriptor - -var file_protos_proto_rawDesc = []byte{ - 0x0a, 0x0c, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x08, - 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x1a, 0x0b, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x2d, 0x0a, 0x07, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, - 0x12, 0x22, 0x0a, 0x06, 0x53, 0x65, 0x6e, 0x64, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x0a, 0x2e, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x50, 0x49, 0x44, 0x52, 0x06, 0x53, 0x65, - 0x6e, 0x64, 0x65, 0x72, 0x22, 0x25, 0x0a, 0x09, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, - 0x64, 0x12, 0x18, 0x0a, 0x07, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x07, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x62, 0x0a, 0x04, 0x57, - 0x6f, 0x72, 0x6b, 0x12, 0x22, 0x0a, 0x06, 0x53, 0x65, 0x6e, 0x64, 0x65, 0x72, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x0a, 0x2e, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x50, 0x49, 0x44, 0x52, - 0x06, 0x53, 0x65, 0x6e, 0x64, 0x65, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x44, 0x61, 0x74, 0x61, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x44, 0x61, 0x74, 0x61, 0x12, 0x0e, 0x0a, 0x02, 0x49, - 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x49, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x54, - 0x79, 0x70, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x54, 0x79, 0x70, 0x65, 0x22, - 0x3e, 0x0a, 0x08, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x56, - 0x61, 0x6c, 0x75, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x56, 0x61, 0x6c, 0x75, - 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x42, - 0x36, 0x5a, 0x34, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6d, 0x61, - 0x73, 0x61, 0x2d, 0x66, 0x69, 0x6e, 0x61, 0x6e, 0x63, 0x65, 0x2f, 0x6d, 0x61, 0x73, 0x61, 0x2d, - 0x6f, 0x72, 0x61, 0x63, 0x6c, 0x65, 0x2f, 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x73, 0x2f, 0x6d, - 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, -} - -var ( - file_protos_proto_rawDescOnce sync.Once - file_protos_proto_rawDescData = file_protos_proto_rawDesc -) - -func file_protos_proto_rawDescGZIP() []byte { - file_protos_proto_rawDescOnce.Do(func() { - file_protos_proto_rawDescData = protoimpl.X.CompressGZIP(file_protos_proto_rawDescData) - }) - return file_protos_proto_rawDescData -} - -var file_protos_proto_msgTypes = make([]protoimpl.MessageInfo, 4) -var file_protos_proto_goTypes = []interface{}{ - (*Connect)(nil), // 0: messages.Connect - (*Connected)(nil), // 1: messages.Connected - (*Work)(nil), // 2: messages.Work - (*Response)(nil), // 3: messages.Response - (*actor.PID)(nil), // 4: actor.PID -} -var file_protos_proto_depIdxs = []int32{ - 4, // 0: messages.Connect.Sender:type_name -> actor.PID - 4, // 1: messages.Work.Sender:type_name -> actor.PID - 2, // [2:2] is the sub-list for method output_type - 2, // [2:2] is the sub-list for method input_type - 2, // [2:2] is the sub-list for extension type_name - 2, // [2:2] is the sub-list for extension extendee - 0, // [0:2] is the sub-list for field type_name -} - -func init() { file_protos_proto_init() } -func file_protos_proto_init() { - if File_protos_proto != nil { - return - } - if !protoimpl.UnsafeEnabled { - file_protos_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Connect); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_protos_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Connected); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_protos_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Work); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_protos_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Response); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - } - type x struct{} - out := protoimpl.TypeBuilder{ - File: protoimpl.DescBuilder{ - GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: file_protos_proto_rawDesc, - NumEnums: 0, - NumMessages: 4, - NumExtensions: 0, - NumServices: 0, - }, - GoTypes: file_protos_proto_goTypes, - DependencyIndexes: file_protos_proto_depIdxs, - MessageInfos: file_protos_proto_msgTypes, - }.Build() - File_protos_proto = out.File - file_protos_proto_rawDesc = nil - file_protos_proto_goTypes = nil - file_protos_proto_depIdxs = nil -} diff --git a/pkg/workers/messages/protos.proto b/pkg/workers/messages/protos.proto deleted file mode 100644 index 5fd0bc63..00000000 --- a/pkg/workers/messages/protos.proto +++ /dev/null @@ -1,24 +0,0 @@ -syntax = "proto3"; -package messages; -option go_package = "github.com/masa-finance/masa-oracle/workers/messages"; -import "actor.proto"; - -message Connect { - actor.PID Sender = 1; -} - -message Connected { - string Message = 1; -} - -message Work { - actor.PID Sender = 1; - string Data = 2; - string Id = 3; - int64 Type = 4; -} - -message Response { - string Value = 1; - string RequestId = 2; -} \ No newline at end of file diff --git a/pkg/pubsub/response_channel_map.go b/pkg/workers/response_channel_map.go similarity index 64% rename from pkg/pubsub/response_channel_map.go rename to pkg/workers/response_channel_map.go index 89340fd7..78a9f021 100644 --- a/pkg/pubsub/response_channel_map.go +++ b/pkg/workers/response_channel_map.go @@ -1,30 +1,34 @@ -package pubsub +package workers -import "sync" +import ( + "sync" + + "github.com/masa-finance/masa-oracle/pkg/workers/types" +) type ResponseChannelMap struct { mu sync.RWMutex - items map[string]chan []byte + items map[string]chan data_types.WorkResponse } var ( - instance *ResponseChannelMap - once sync.Once + rcmInstance *ResponseChannelMap + rcmOnce sync.Once ) -// GetResponseChannelMap returns the singleton instance of ResponseChannelMap. +// GetResponseChannelMap returns the singleton rcmInstance of ResponseChannelMap. func GetResponseChannelMap() *ResponseChannelMap { - once.Do(func() { - instance = &ResponseChannelMap{ - items: make(map[string]chan []byte), + rcmOnce.Do(func() { + rcmInstance = &ResponseChannelMap{ + items: make(map[string]chan data_types.WorkResponse), } }) - return instance + return rcmInstance } // Set associates the specified value with the specified key in the ResponseChannelMap. // It acquires a write lock to ensure thread-safety while setting the value. -func (drm *ResponseChannelMap) Set(key string, value chan []byte) { +func (drm *ResponseChannelMap) Set(key string, value chan data_types.WorkResponse) { drm.mu.Lock() defer drm.mu.Unlock() drm.items[key] = value @@ -34,7 +38,7 @@ func (drm *ResponseChannelMap) Set(key string, value chan []byte) { // It acquires a read lock to ensure thread-safety while reading the value. // If the key exists in the ResponseChannelMap, it returns the corresponding value and true. // If the key does not exist, it returns nil and false. -func (drm *ResponseChannelMap) Get(key string) (chan []byte, bool) { +func (drm *ResponseChannelMap) Get(key string) (chan data_types.WorkResponse, bool) { drm.mu.RLock() defer drm.mu.RUnlock() value, ok := drm.items[key] @@ -57,8 +61,8 @@ func (drm *ResponseChannelMap) Len() int { return len(drm.items) } -func (drm *ResponseChannelMap) CreateChannel(key string) chan []byte { - ch := make(chan []byte) +func (drm *ResponseChannelMap) CreateChannel(key string) chan data_types.WorkResponse { + ch := make(chan data_types.WorkResponse) drm.Set(key, ch) return ch } diff --git a/pkg/workers/types/request_response.go b/pkg/workers/types/request_response.go new file mode 100644 index 00000000..359c63cb --- /dev/null +++ b/pkg/workers/types/request_response.go @@ -0,0 +1,29 @@ +package data_types + +import ( + "github.com/libp2p/go-libp2p/core/peer" + + masa "github.com/masa-finance/masa-oracle/pkg" + "github.com/masa-finance/masa-oracle/pkg/pubsub" +) + +type Worker struct { + IsLocal bool + IPAddr string + AddrInfo *peer.AddrInfo + NodeData pubsub.NodeData + Node *masa.OracleNode +} + +type WorkRequest struct { + WorkType WorkerType `json:"workType,omitempty"` + RequestId string `json:"requestId,omitempty"` + Data []byte `json:"data,omitempty"` +} + +type WorkResponse struct { + WorkRequest *WorkRequest `json:"workRequest,omitempty"` + Data interface{} `json:"data,omitempty"` + Error string `json:"error,omitempty"` + WorkerPeerId string `json:"workerPeerId,omitempty"` +} diff --git a/pkg/workers/types/work_types.go b/pkg/workers/types/work_types.go new file mode 100644 index 00000000..e84fe439 --- /dev/null +++ b/pkg/workers/types/work_types.go @@ -0,0 +1,51 @@ +package data_types + +import ( + "github.com/sirupsen/logrus" + + "github.com/masa-finance/masa-oracle/pkg/pubsub" +) + +type WorkerType string + +const ( + Discord WorkerType = "discord" + DiscordProfile WorkerType = "discord-profile" + DiscordChannelMessages WorkerType = "discord-channel-messages" + DiscordSentiment WorkerType = "discord-sentiment" + TelegramSentiment WorkerType = "telegram-sentiment" + TelegramChannelMessages WorkerType = "telegram-channel-messages" + DiscordGuildChannels WorkerType = "discord-guild-channels" + DiscordUserGuilds WorkerType = "discord-user-guilds" + LLMChat WorkerType = "llm-chat" + Twitter WorkerType = "twitter" + TwitterFollowers WorkerType = "twitter-followers" + TwitterProfile WorkerType = "twitter-profile" + TwitterSentiment WorkerType = "twitter-sentiment" + TwitterTrends WorkerType = "twitter-trends" + Web WorkerType = "web" + WebSentiment WorkerType = "web-sentiment" + Test WorkerType = "test" +) + +// WorkerTypeToCategory maps WorkerType to WorkerCategory +func WorkerTypeToCategory(wt WorkerType) pubsub.WorkerCategory { + logrus.Infof("Mapping WorkerType %s to WorkerCategory", wt) + switch wt { + case Discord, DiscordProfile, DiscordChannelMessages, DiscordSentiment, DiscordGuildChannels, DiscordUserGuilds: + logrus.Info("WorkerType is related to Discord") + return pubsub.CategoryDiscord + case TelegramSentiment, TelegramChannelMessages: + logrus.Info("WorkerType is related to Telegram") + return pubsub.CategoryTelegram + case Twitter, TwitterFollowers, TwitterProfile, TwitterSentiment, TwitterTrends: + logrus.Info("WorkerType is related to Twitter") + return pubsub.CategoryTwitter + case Web, WebSentiment: + logrus.Info("WorkerType is related to Web") + return pubsub.CategoryWeb + default: + logrus.Warn("WorkerType is invalid or not recognized") + return -1 // Invalid category + } +} diff --git a/pkg/workers/worker_manager.go b/pkg/workers/worker_manager.go new file mode 100644 index 00000000..d4966cc6 --- /dev/null +++ b/pkg/workers/worker_manager.go @@ -0,0 +1,299 @@ +package workers + +import ( + "context" + "encoding/binary" + "encoding/json" + "errors" + "fmt" + "io" + "sync" + "time" + + "github.com/libp2p/go-libp2p/core/network" + "github.com/sirupsen/logrus" + + masa "github.com/masa-finance/masa-oracle/pkg" + "github.com/masa-finance/masa-oracle/pkg/config" + "github.com/masa-finance/masa-oracle/pkg/workers/handlers" + data_types "github.com/masa-finance/masa-oracle/pkg/workers/types" +) + +var ( + instance *WorkHandlerManager + once sync.Once +) + +func GetWorkHandlerManager() *WorkHandlerManager { + once.Do(func() { + instance = &WorkHandlerManager{ + handlers: make(map[data_types.WorkerType]*WorkHandlerInfo), + } + instance.setupHandlers() + }) + return instance +} + +// ErrHandlerNotFound is an error returned when a work handler cannot be found. +var ErrHandlerNotFound = errors.New("work handler not found") + +// WorkHandler defines the interface for handling different types of work. +type WorkHandler interface { + HandleWork(data []byte) data_types.WorkResponse +} + +// WorkHandlerInfo contains information about a work handler, including metrics. +type WorkHandlerInfo struct { + Handler WorkHandler + CallCount int64 + TotalRuntime time.Duration +} + +// WorkHandlerManager manages work handlers and tracks their execution metrics. +type WorkHandlerManager struct { + handlers map[data_types.WorkerType]*WorkHandlerInfo + mu sync.RWMutex +} + +func (whm *WorkHandlerManager) setupHandlers() { + cfg := config.GetInstance() + if cfg.TwitterScraper { + whm.addWorkHandler(data_types.Twitter, &handlers.TwitterQueryHandler{}) + whm.addWorkHandler(data_types.TwitterFollowers, &handlers.TwitterFollowersHandler{}) + whm.addWorkHandler(data_types.TwitterProfile, &handlers.TwitterProfileHandler{}) + whm.addWorkHandler(data_types.TwitterSentiment, &handlers.TwitterSentimentHandler{}) + whm.addWorkHandler(data_types.TwitterTrends, &handlers.TwitterTrendsHandler{}) + } + if cfg.WebScraper { + whm.addWorkHandler(data_types.Web, &handlers.WebHandler{}) + whm.addWorkHandler(data_types.WebSentiment, &handlers.WebSentimentHandler{}) + } + if cfg.LlmServer { + whm.addWorkHandler(data_types.LLMChat, &handlers.LLMChatHandler{}) + } + if cfg.DiscordScraper { + whm.addWorkHandler(data_types.Discord, &handlers.DiscordProfileHandler{}) + } +} + +// addWorkHandler registers a new work handler under a specific name. +func (whm *WorkHandlerManager) addWorkHandler(wType data_types.WorkerType, handler WorkHandler) { + whm.mu.Lock() + defer whm.mu.Unlock() + whm.handlers[wType] = &WorkHandlerInfo{Handler: handler} +} + +// getWorkHandler retrieves a registered work handler by name. +func (whm *WorkHandlerManager) getWorkHandler(wType data_types.WorkerType) (WorkHandler, bool) { + whm.mu.RLock() + defer whm.mu.RUnlock() + info, exists := whm.handlers[wType] + if !exists { + return nil, false + } + return info.Handler, true +} + +func (whm *WorkHandlerManager) DistributeWork(node *masa.OracleNode, workRequest data_types.WorkRequest) (response data_types.WorkResponse) { + category := data_types.WorkerTypeToCategory(workRequest.WorkType) + remoteWorkers, localWorker := GetEligibleWorkers(node, category, workerConfig) + + remoteWorkersAttempted := 0 + logrus.Info("Starting round-robin worker selection") + + // Try remote workers first, up to MaxRemoteWorkers + for _, worker := range remoteWorkers { + if remoteWorkersAttempted >= workerConfig.MaxRemoteWorkers { + logrus.Infof("Reached maximum remote workers (%d), stopping remote worker attempts", workerConfig.MaxRemoteWorkers) + break + } + remoteWorkersAttempted++ + logrus.Infof("Attempting remote worker %s (attempt %d/%d)", worker.NodeData.PeerId, remoteWorkersAttempted, workerConfig.MaxRemoteWorkers) + response = whm.sendWorkToWorker(node, worker, workRequest) + if response.Error != "" { + logrus.Errorf("error sending work to worker: %s: %s", response.WorkerPeerId, response.Error) + logrus.Infof("Remote worker %s failed, moving to next worker", worker.NodeData.PeerId) + continue + } + return response + } + // Fallback to local execution if local worker is eligible + if localWorker != nil { + return whm.ExecuteWork(workRequest) + } + if response.Error == "" { + response.Error = "no eligible workers found" + } else { + response.Error = fmt.Sprintf("no workers could process: remote attempt failed due to: %s", response.Error) + } + return response +} + +func (whm *WorkHandlerManager) sendWorkToWorker(node *masa.OracleNode, worker data_types.Worker, workRequest data_types.WorkRequest) (response data_types.WorkResponse) { + ctxWithTimeout, cancel := context.WithTimeout(context.Background(), workerConfig.WorkerResponseTimeout) + defer cancel() // Cancel the context when done to release resources + + if err := node.Host.Connect(ctxWithTimeout, *worker.AddrInfo); err != nil { + response.Error = fmt.Sprintf("failed to connect to remote peer %s: %v", worker.AddrInfo.ID.String(), err) + return + } else { + logrus.Debugf("[+] Connection established with node: %s", worker.AddrInfo.ID.String()) + stream, err := node.Host.NewStream(ctxWithTimeout, worker.AddrInfo.ID, config.ProtocolWithVersion(config.WorkerProtocol)) + if err != nil { + response.Error = fmt.Sprintf("error opening stream: %v", err) + return + } + // the stream should be closed by the receiver, but keeping this here just in case + defer func(stream network.Stream) { + err := stream.Close() + if err != nil { + logrus.Debugf("[-] Error closing stream: %s", err) + } + }(stream) // Close the stream when done + + // Write the request to the stream with length prefix + bytes, err := json.Marshal(workRequest) + if err != nil { + response.Error = fmt.Sprintf("error marshaling work request: %v", err) + return + } + lengthBuf := make([]byte, 4) + binary.BigEndian.PutUint32(lengthBuf, uint32(len(bytes))) + _, err = stream.Write(lengthBuf) + if err != nil { + response.Error = fmt.Sprintf("error writing length to stream: %v", err) + return + } + _, err = stream.Write(bytes) + if err != nil { + response.Error = fmt.Sprintf("error writing to stream: %v", err) + return + } + + // Read the response length + lengthBuf = make([]byte, 4) + _, err = io.ReadFull(stream, lengthBuf) + if err != nil { + response.Error = fmt.Sprintf("error reading response length: %v", err) + return + } + responseLength := binary.BigEndian.Uint32(lengthBuf) + + // Read the actual response + responseBuf := make([]byte, responseLength) + _, err = io.ReadFull(stream, responseBuf) + if err != nil { + response.Error = fmt.Sprintf("error reading response: %v", err) + return + } + err = json.Unmarshal(responseBuf, &response) + if err != nil { + response.Error = fmt.Sprintf("error unmarshaling response: %v", err) + return + } + } + return response +} + +// ExecuteWork finds and executes the work handler associated with the given name. +// It tracks the call count and execution duration for the handler. +func (whm *WorkHandlerManager) ExecuteWork(workRequest data_types.WorkRequest) (response data_types.WorkResponse) { + handler, exists := whm.getWorkHandler(workRequest.WorkType) + if !exists { + return data_types.WorkResponse{Error: ErrHandlerNotFound.Error()} + } + + // Create a context with a 30-second timeout + ctx, cancel := context.WithTimeout(context.Background(), workerConfig.WorkerResponseTimeout) + defer cancel() + + // Channel to receive the work response + responseChan := make(chan data_types.WorkResponse, 1) + + // Execute the work in a separate goroutine + go func() { + startTime := time.Now() + workResponse := handler.HandleWork(workRequest.Data) + if workResponse.Error == "" { + duration := time.Since(startTime) + whm.mu.Lock() + handlerInfo := whm.handlers[workRequest.WorkType] + handlerInfo.CallCount++ + handlerInfo.TotalRuntime += duration + whm.mu.Unlock() + } + responseChan <- workResponse + }() + + select { + case <-ctx.Done(): + // Context timed out + return data_types.WorkResponse{Error: "work execution timed out"} + case response = <-responseChan: + // Work completed within the timeout + return response + } +} + +func (whm *WorkHandlerManager) HandleWorkerStream(stream network.Stream) { + defer func(stream network.Stream) { + err := stream.Close() + if err != nil { + logrus.Errorf("[-] Error closing stream in handler: %s", err) + } + }(stream) + + // Read the length of the message + lengthBuf := make([]byte, 4) + _, err := io.ReadFull(stream, lengthBuf) + if err != nil { + logrus.Errorf("error reading message length: %v", err) + return + } + messageLength := binary.BigEndian.Uint32(lengthBuf) + + // Read the actual message + messageBuf := make([]byte, messageLength) + _, err = io.ReadFull(stream, messageBuf) + if err != nil { + logrus.Errorf("error reading message: %v", err) + return + } + + var workRequest data_types.WorkRequest + err = json.Unmarshal(messageBuf, &workRequest) + if err != nil { + logrus.Errorf("error unmarshaling work request: %v", err) + return + } + peerId := stream.Conn().LocalPeer().String() + workResponse := whm.ExecuteWork(workRequest) + if workResponse.Error != "" { + logrus.Errorf("error from remote worker %s: executing work: %s", peerId, workResponse.Error) + } + workResponse.WorkerPeerId = peerId + + // Write the response to the stream + responseBytes, err := json.Marshal(workResponse) + if err != nil { + logrus.Errorf("error marshaling work response: %v", err) + return + } + + // Prefix the response with its length + responseLength := uint32(len(responseBytes)) + lengthBuf = make([]byte, 4) + binary.BigEndian.PutUint32(lengthBuf, responseLength) + + _, err = stream.Write(lengthBuf) + if err != nil { + logrus.Errorf("error writing response length to stream: %v", err) + return + } + + _, err = stream.Write(responseBytes) + if err != nil { + logrus.Errorf("error writing response to stream: %v", err) + return + } +} diff --git a/pkg/workers/worker_selection.go b/pkg/workers/worker_selection.go new file mode 100644 index 00000000..9e1d6dcb --- /dev/null +++ b/pkg/workers/worker_selection.go @@ -0,0 +1,67 @@ +package workers + +import ( + "context" + "math/rand/v2" + "time" + + "github.com/libp2p/go-libp2p/core/peer" + "github.com/multiformats/go-multiaddr" + "github.com/sirupsen/logrus" + + masa "github.com/masa-finance/masa-oracle/pkg" + "github.com/masa-finance/masa-oracle/pkg/pubsub" + "github.com/masa-finance/masa-oracle/pkg/workers/types" +) + +// GetEligibleWorkers Uses the new NodeTracker method to get the eligible workers for a given message type +// I'm leaving this returning an array so that we can easily increase the number of workers in the future +func GetEligibleWorkers(node *masa.OracleNode, category pubsub.WorkerCategory, config *WorkerConfig) ([]data_types.Worker, *data_types.Worker) { + + var workers []data_types.Worker + nodes := node.NodeTracker.GetEligibleWorkerNodes(category) + var localWorker *data_types.Worker + + // Shuffle the node list first to avoid always selecting the same node + rand.Shuffle(len(nodes), func(i, j int) { + nodes[i], nodes[j] = nodes[j], nodes[i] + }) + + logrus.Info("checking connections to eligible workers") + start := time.Now() + for _, eligible := range nodes { + if eligible.PeerId.String() == node.Host.ID().String() { + localWorker = &data_types.Worker{IsLocal: true, NodeData: eligible} + continue + } + addr, err := multiaddr.NewMultiaddr(eligible.MultiaddrsString) + if err != nil { + logrus.Errorf("error creating multiaddress: %s", err.Error()) + continue + } + peerInfo, err := peer.AddrInfoFromP2pAddr(addr) + if err != nil { + logrus.Errorf("Failed to get peer info: %s", err) + continue + } + ctxWithTimeout, cancel := context.WithTimeout(context.Background(), config.ConnectionTimeout) + defer cancel() // Cancel the context when done to release resources + if err := node.Host.Connect(ctxWithTimeout, *peerInfo); err != nil { + logrus.Debugf("Failed to connect to peer: %v", err) + continue + } + workers = append(workers, data_types.Worker{IsLocal: false, NodeData: eligible, AddrInfo: peerInfo}) + // print duration of worker selection in seconds with floating point precision + dur := time.Since(start).Milliseconds() + logrus.Infof("Worker selection took %v milliseconds", dur) + break + } + // make sure we get the local node in the list + if localWorker == nil { + nd := node.NodeTracker.GetNodeData(node.Host.ID().String()) + if nd.CanDoWork(category) { + localWorker = &data_types.Worker{IsLocal: true, NodeData: *nd} + } + } + return workers, localWorker +} diff --git a/pkg/workers/workers.go b/pkg/workers/workers.go deleted file mode 100644 index f7f862e4..00000000 --- a/pkg/workers/workers.go +++ /dev/null @@ -1,491 +0,0 @@ -package workers - -import ( - "context" - "encoding/json" - "fmt" - "sync" - "time" - - "github.com/libp2p/go-libp2p/core/peer" - - masa "github.com/masa-finance/masa-oracle/pkg" - "github.com/masa-finance/masa-oracle/pkg/config" - "github.com/masa-finance/masa-oracle/pkg/db" - "github.com/masa-finance/masa-oracle/pkg/pubsub" - "github.com/masa-finance/masa-oracle/pkg/workers/messages" - - "github.com/multiformats/go-multiaddr" - - "github.com/asynkron/protoactor-go/actor" - - "github.com/ipfs/go-cid" - - pubsub2 "github.com/libp2p/go-libp2p-pubsub" - mh "github.com/multiformats/go-multihash" - "github.com/sirupsen/logrus" -) - -type WorkerType string - -const ( - Discord WorkerType = "discord" - DiscordProfile WorkerType = "discord-profile" - DiscordChannelMessages WorkerType = "discord-channel-messages" - DiscordSentiment WorkerType = "discord-sentiment" - TelegramSentiment WorkerType = "telegram-sentiment" - TelegramChannelMessages WorkerType = "telegram-channel-messages" - DiscordGuildChannels WorkerType = "discord-guild-channels" - DiscordUserGuilds WorkerType = "discord-user-guilds" - LLMChat WorkerType = "llm-chat" - Twitter WorkerType = "twitter" - TwitterFollowers WorkerType = "twitter-followers" - TwitterProfile WorkerType = "twitter-profile" - TwitterSentiment WorkerType = "twitter-sentiment" - TwitterTrends WorkerType = "twitter-trends" - Web WorkerType = "web" - WebSentiment WorkerType = "web-sentiment" - Test WorkerType = "test" -) - -var WORKER = struct { - Discord, DiscordProfile, DiscordChannelMessages, DiscordSentiment, TelegramSentiment, TelegramChannelMessages, DiscordGuildChannels, DiscordUserGuilds, LLMChat, Twitter, TwitterFollowers, TwitterProfile, TwitterSentiment, TwitterTrends, Web, WebSentiment, Test WorkerType -}{ - Discord: Discord, - DiscordProfile: DiscordProfile, - DiscordChannelMessages: DiscordChannelMessages, - DiscordSentiment: DiscordSentiment, - TelegramSentiment: TelegramSentiment, - TelegramChannelMessages: TelegramChannelMessages, - DiscordGuildChannels: DiscordGuildChannels, - DiscordUserGuilds: DiscordUserGuilds, - LLMChat: LLMChat, - Twitter: Twitter, - TwitterFollowers: TwitterFollowers, - TwitterProfile: TwitterProfile, - TwitterSentiment: TwitterSentiment, - TwitterTrends: TwitterTrends, - Web: Web, - WebSentiment: WebSentiment, - Test: Test, -} - -var ( - clients = actor.NewPIDSet() - workerStatusCh = make(chan *pubsub2.Message) - workerDoneCh = make(chan *pubsub2.Message) -) - -// WorkerTypeToCategory maps WorkerType to WorkerCategory -func WorkerTypeToCategory(wt WorkerType) pubsub.WorkerCategory { - switch wt { - case Discord, DiscordProfile, DiscordChannelMessages, DiscordSentiment, DiscordGuildChannels, DiscordUserGuilds: - return pubsub.CategoryDiscord - case TelegramSentiment, TelegramChannelMessages: - return pubsub.CategoryTelegram - case Twitter, TwitterFollowers, TwitterProfile, TwitterSentiment, TwitterTrends: - return pubsub.CategoryTwitter - case Web, WebSentiment: - return pubsub.CategoryWeb - default: - return -1 // Invalid category - } -} - -type ChanResponse struct { - Response map[string]interface{} - ChannelId string -} - -type Worker struct { - Node *masa.OracleNode -} - -// NewWorker creates a new instance of the Worker actor. -// It implements the actor.Receiver interface, allowing it to receive and handle messages. -// -// Returns: -// - An instance of the Worker struct that implements the actor.Receiver interface. -func NewWorker(node *masa.OracleNode) actor.Producer { - return func() actor.Actor { - return &Worker{Node: node} - } -} - -// Receive is the message handling method for the Worker actor. -// It receives messages through the actor context and processes them based on their type. -func (a *Worker) Receive(ctx actor.Context) { - switch m := ctx.Message().(type) { - case *messages.Connect: - a.HandleConnect(ctx, m) - case *actor.Started: - if a.Node.IsWorker() { - a.HandleLog(ctx, "[+] Actor started") - } - case *actor.Stopping: - if a.Node.IsWorker() { - a.HandleLog(ctx, "[+] Actor stopping") - } - case *actor.Stopped: - if a.Node.IsWorker() { - a.HandleLog(ctx, "[+] Actor stopped") - } - case *messages.Work: - if a.Node.IsWorker() { - logrus.Infof("[+] Received Work") - a.HandleWork(ctx, m, a.Node) - } - case *messages.Response: - logrus.Infof("[+] Received Response") - msg := &pubsub2.Message{} - err := json.Unmarshal([]byte(m.Value), msg) - if err != nil { - msg, err = getResponseMessage(m) - if err != nil { - logrus.Errorf("[-] Error getting response message: %v", err) - return - } - } - workerDoneCh <- msg - ctx.Poison(ctx.Self()) - default: - logrus.Warningf("[+] Received unknown message in workers: %T, message: %+v", m, m) - } -} - -// computeCid calculates the CID (Content Identifier) for a given string. -// -// Parameters: -// - str: The input string for which to compute the CID. -// -// Returns: -// - string: The computed CID as a string. -// - error: An error, if any occurred during the CID computation. -// -// The function uses the multihash package to create a SHA2-256 hash of the input string. -// It then creates a CID (version 1) from the multihash and returns the CID as a string. -// If an error occurs during the multihash computation or CID creation, it is returned. -func computeCid(str string) (string, error) { - // Create a multihash from the string - mhHash, err := mh.Sum([]byte(str), mh.SHA2_256, -1) - if err != nil { - return "", err - } - // Create a CID from the multihash - cidKey := cid.NewCidV1(cid.Raw, mhHash).String() - return cidKey, nil -} - -// getResponseMessage converts a messages.Response object into a pubsub2.Message object. -// It unmarshals the JSON-encoded response value into a map and then constructs a new pubsub2.Message -// using the extracted data. -// -// Parameters: -// - response: A pointer to a messages.Response object containing the JSON-encoded response data. -// -// Returns: -// - A pointer to a pubsub2.Message object constructed from the response data. -// - An error if there is an issue with unmarshalling the response data. -func getResponseMessage(response *messages.Response) (*pubsub2.Message, error) { - responseData := map[string]interface{}{} - - err := json.Unmarshal([]byte(response.Value), &responseData) - if err != nil { - return nil, err - } - msg := &pubsub2.Message{ - ID: responseData["ID"].(string), - ReceivedFrom: peer.ID(responseData["ReceivedFrom"].(string)), - ValidatorData: responseData["ValidatorData"], - Local: responseData["Local"].(bool), - } - return msg, nil -} - -// SendWork is a function that sends work to a node. It takes two parameters: -// node: A pointer to a masa.OracleNode object. This is the node to which the work will be sent. -// m: A pointer to a pubsub2.Message object. This is the message that contains the work to be sent. -func SendWork(node *masa.OracleNode, m *pubsub2.Message) { - var wg sync.WaitGroup - props := actor.PropsFromProducer(NewWorker(node)) - pid := node.ActorEngine.Spawn(props) - message := &messages.Work{Data: string(m.Data), Sender: pid, Id: m.ReceivedFrom.String(), Type: int64(pubsub.CategoryTwitter)} - n := 0 - - responseCollector := make(chan *pubsub2.Message, 100) // Buffered channel to collect responses - timeout := time.After(8 * time.Second) - - // Local worker - if node.IsStaked && node.IsWorker() { - wg.Add(1) - go func() { - defer wg.Done() - future := node.ActorEngine.RequestFuture(pid, message, 60*time.Second) // Increase timeout from 30 to 60 seconds - result, err := future.Result() - if err != nil { - logrus.Errorf("[-] Error receiving response from local worker: %v", err) - responseCollector <- &pubsub2.Message{ - ValidatorData: map[string]interface{}{"error": err.Error()}, - } - return - } - response := result.(*messages.Response) - msg := &pubsub2.Message{} - rErr := json.Unmarshal([]byte(response.Value), msg) - if rErr != nil { - gMsg, gErr := getResponseMessage(result.(*messages.Response)) - if gErr != nil { - logrus.Errorf("[-] Error getting response message: %v", gErr) - responseCollector <- &pubsub2.Message{ - ValidatorData: map[string]interface{}{"error": gErr.Error()}, - } - return - } - msg = gMsg - } - responseCollector <- msg - n++ - }() - } - - // Remote workers - peers := node.NodeTracker.GetAllNodeData() - for _, p := range peers { - for _, addr := range p.Multiaddrs { - ipAddr, _ := addr.ValueForProtocol(multiaddr.P_IP4) - if (p.PeerId.String() != node.Host.ID().String()) && - p.IsStaked && - node.NodeTracker.GetNodeData(p.PeerId.String()).CanDoWork(pubsub.WorkerCategory(message.Type)) { - logrus.Infof("[+] Worker Address: %s", ipAddr) - wg.Add(1) - go func(p pubsub.NodeData) { - defer wg.Done() - spawned, err := node.ActorRemote.SpawnNamed(fmt.Sprintf("%s:4001", ipAddr), "worker", "peer", -1) - if err != nil { - logrus.Debugf("[-] Error spawning remote worker: %v", err) - responseCollector <- &pubsub2.Message{ - ValidatorData: map[string]interface{}{"error": err.Error()}, - } - return - } - spawnedPID := spawned.Pid - logrus.Infof("[+] Worker Address: %s", spawnedPID) - if spawnedPID == nil { - logrus.Errorf("[-] Spawned PID is nil for IP: %s", ipAddr) - responseCollector <- &pubsub2.Message{ - ValidatorData: map[string]interface{}{"error": "Spawned PID is nil"}, - } - return - } - client := node.ActorEngine.Spawn(props) - node.ActorEngine.Send(spawnedPID, &messages.Connect{Sender: client}) - future := node.ActorEngine.RequestFuture(spawnedPID, message, 30*time.Second) - result, fErr := future.Result() - if fErr != nil { - logrus.Debugf("[-] Error receiving response from remote worker: %v", fErr) - responseCollector <- &pubsub2.Message{ - ValidatorData: map[string]interface{}{"error": fErr.Error()}, - } - return - } - response := result.(*messages.Response) - msg := &pubsub2.Message{} - rErr := json.Unmarshal([]byte(response.Value), &msg) - if rErr != nil { - gMsg, gErr := getResponseMessage(response) - if gErr != nil { - logrus.Errorf("[-] Error getting response message: %v", gErr) - responseCollector <- &pubsub2.Message{ - ValidatorData: map[string]interface{}{"error": gErr.Error()}, - } - return - } - if gMsg != nil { - msg = gMsg - } - } - responseCollector <- msg - n++ - // cap at 3 for performance - if n == len(peers) || n == 3 { - logrus.Info("[+] All workers have responded") - responseCollector <- msg - } - }(p) - } - } - } - - // Queue responses and send to workerDoneCh - go func() { - var responses []*pubsub2.Message - for { - select { - case response := <-responseCollector: - responses = append(responses, response) - case <-timeout: - for _, resp := range responses { - workerDoneCh <- resp - } - return - } - } - }() - - wg.Wait() -} - -// MonitorWorkers monitors worker data by subscribing to the completed work topic, -// computing a CID for each received data, and writing the data to the database. -// -// Parameters: -// - ctx: The context for the monitoring operation. -// - node: A pointer to the OracleNode instance. -// -// The function uses a ticker to periodically log a debug message every 60 seconds. -// It subscribes to the completed work topic using the PubSubManager and handles the received data. -// For each received data, it computes a CID using the computeCid function, logs the CID, -// marshals the data to JSON, and writes it to the database using the WriteData function. -// The monitoring continues until the context is done. -func MonitorWorkers(ctx context.Context, node *masa.OracleNode) { - node.WorkerTracker = &pubsub.WorkerEventTracker{WorkerStatusCh: workerStatusCh} - err := node.PubSubManager.AddSubscription(config.TopicWithVersion(config.WorkerTopic), node.WorkerTracker, true) - if err != nil { - logrus.Errorf("[-] Subscribe error %v", err) - } - - // Register self as a remote node for the network - node.ActorRemote.Register("peer", actor.PropsFromProducer(NewWorker(node))) - - if node.WorkerTracker == nil { - logrus.Error("[-] MonitorWorkers: WorkerTracker is nil") - return - } - - if node.WorkerTracker.WorkerStatusCh == nil { - logrus.Error("[-] MonitorWorkers: WorkerStatusCh is nil") - return - } - - ticker := time.NewTicker(time.Second * 15) - defer ticker.Stop() - rcm := pubsub.GetResponseChannelMap() - var startTime time.Time - - for { - select { - case work := <-node.WorkerTracker.WorkerStatusCh: - logrus.Info("[+] Sending work to network") - var workData map[string]string - - err := json.Unmarshal(work.Data, &workData) - if err != nil { - logrus.Error("[-] Error unmarshalling work: ", err) - continue - } - startTime = time.Now() - go SendWork(node, work) - case data := <-workerDoneCh: - validatorDataMap, ok := data.ValidatorData.(map[string]interface{}) - if !ok { - logrus.Errorf("[-] Error asserting type: %v", ok) - continue - } - - if ch, ok := rcm.Get(validatorDataMap["ChannelId"].(string)); ok { - validatorData, err := json.Marshal(validatorDataMap["Response"]) - if err != nil { - logrus.Errorf("[-] Error marshalling data.ValidatorData: %v", err) - continue - } - ch <- validatorData - defer close(ch) - } else { - logrus.Debugf("Error processing data.ValidatorData: %v", data.ValidatorData) - continue - } - - processValidatorData(data, validatorDataMap, &startTime, node) - - case <-ticker.C: - logrus.Info("[+] worker tick") - - case <-ctx.Done(): - return - } - } -} - -/** - * Processes the validator data received from the network. - * - * @param {pubsub2.Message} data - The message data received from the network. - * @param {map[string]interface{}} validatorDataMap - The map containing validator data. - * @param {time.Time} startTime - The start time of the work. - * @param {masa.OracleNode} node - The OracleNode instance. - */ -func processValidatorData(data *pubsub2.Message, validatorDataMap map[string]interface{}, startTime *time.Time, node *masa.OracleNode) { - //logrus.Infof("[+] Work validatorDataMap %s", validatorDataMap) - if response, ok := validatorDataMap["Response"].(map[string]interface{}); ok { - if _, ok := response["error"].(string); ok { - logrus.Infof("[+] Work failed %s", response["error"]) - - // Set WorkerTimeout for the node - nodeData := node.NodeTracker.GetNodeData(data.ReceivedFrom.String()) - if nodeData != nil { - nodeData.WorkerTimeout = time.Now() - node.NodeTracker.AddOrUpdateNodeData(nodeData, true) - } - - } else if work, ok := response["data"].(string); ok { - processWork(data, work, startTime, node) - - } else if w, ok := response["data"].(map[string]interface{}); ok { - work, err := json.Marshal(w) - - if err != nil { - logrus.Errorf("[-] Error marshalling data.ValidatorData: %v", err) - return - } - - processWork(data, string(work), startTime, node) - } else { - work, err := json.Marshal(response["data"]) - if err != nil { - logrus.Errorf("[-] Error marshalling data.ValidatorData: %v", err) - return - } - - processWork(data, string(work), startTime, node) - - } - } -} - -/** - * Processes the work received from the network. - * - * @param {pubsub2.Message} data - The message data received from the network. - * @param {string} work - The work data as a string. - * @param {time.Time} startTime - The start time of the work. - * @param {masa.OracleNode} node - The OracleNode instance. - */ -func processWork(data *pubsub2.Message, work string, startTime *time.Time, node *masa.OracleNode) { - key, _ := computeCid(work) - logrus.Infof("[+] Work done %s", key) - - endTime := time.Now() - duration := endTime.Sub(*startTime) - - workEvent := db.WorkEvent{ - CID: key, - PeerId: data.ID, - Payload: []byte(work), - Duration: duration.Seconds(), - Timestamp: time.Now().Unix(), - } - logrus.Infof("[+] Publishing work event : %v for Peer %s", workEvent.CID, workEvent.PeerId) - logrus.Debugf("[+] Publishing work event : %v", workEvent) - - _ = node.PubSubManager.Publish(config.TopicWithVersion(config.BlockTopic), workEvent.Payload) -} From e81008baba6ff229c281c657747cdd2f090150b3 Mon Sep 17 00:00:00 2001 From: Nolan Jacobson <50815660+nolanjacobson@users.noreply.github.com> Date: Fri, 16 Aug 2024 16:26:44 -0400 Subject: [PATCH 04/10] chore: updates to docs (#495) * chore: updates to docs * chore: updates to docs * chore: updates to docs --------- Co-authored-by: Brendan Playford <34052452+teslashibe@users.noreply.github.com> --- docs/oracle-node/telegram-sentiment.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/oracle-node/telegram-sentiment.md b/docs/oracle-node/telegram-sentiment.md index 8764ea15..5c399367 100644 --- a/docs/oracle-node/telegram-sentiment.md +++ b/docs/oracle-node/telegram-sentiment.md @@ -5,15 +5,15 @@ title: Telegram Sentiment ## Masa Node Telegram Sentiment Analysis Feature -The Masa Node introduces a powerful feature for analyzing the sentiment of telegram messages. This functionality leverages advanced language models to interpret the sentiment behind a collection of tweets, providing valuable insights into public perception and trends. +The Masa Node introduces a powerful feature for analyzing the sentiment of telegram messages. This functionality leverages advanced language models to interpret the sentiment behind a collection of Telegram messages, providing valuable insights into public perception and trends. ## Overview -The Telegram sentiment analysis feature is part of the broader capabilities of the Masa Node, designed to interact with Telegram messages data in a meaningful way. It uses state-of-the-art language models to evaluate the sentiment of tweets, categorizing them into positive, negative, or neutral sentiments. +The Telegram sentiment analysis feature is part of the broader capabilities of the Masa Node, designed to interact with Telegram messages data in a meaningful way. It uses state-of-the-art language models to evaluate the sentiment of Telegram messages, categorizing them into positive, negative, or neutral sentiments. ## How It Works -The sentiment analysis process involves fetching tweets based on specific queries, and then analyzing these tweets using selected language models. The system supports various models, including Claude and GPT variants, allowing for flexible and powerful sentiment analysis. +The sentiment analysis process involves fetching Telegram messages based on specific queries, and then analyzing these messages using selected language models. The system supports various models, including Claude and GPT variants, allowing for flexible and powerful sentiment analysis. ### Models @@ -64,7 +64,7 @@ const ( #### Masa cli or code integration -Tweets are fetched using the Twitter Scraper library, as seen in the [llmbridge](file:///Users/john/Projects/masa/masa-oracle/pkg/llmbridge/sentiment.go#) package. This process does not require Telegram API keys, making it accessible and straightforward. +Messages are fetched using the Telegram Scraper library, as seen in the [llmbridge](/masa-oracle/pkg/llmbridge/sentiment.go#) package. This process does not require Telegram API keys, making it accessible and straightforward. ```go func AnalyzeSentimentTelegram(messages []*tg.Message, model string, prompt string) (string, string, error) { @@ -72,4 +72,4 @@ func AnalyzeSentimentTelegram(messages []*tg.Message, model string, prompt strin ### Analyzing Sentiment -Once tweets are fetched, they are sent to the chosen language model for sentiment analysis. The system currently supports models prefixed with "claude-" and "gpt-", catering to a range of analysis needs. \ No newline at end of file +Once Telegram Messages are fetched, they are sent to the chosen language model for sentiment analysis. The system currently supports models prefixed with "claude-" and "gpt-", catering to a range of analysis needs. \ No newline at end of file From 452cecb9ff674d884fa79dcf8c99527a7ced2d9a Mon Sep 17 00:00:00 2001 From: Nolan Jacobson <50815660+nolanjacobson@users.noreply.github.com> Date: Fri, 16 Aug 2024 16:32:25 -0400 Subject: [PATCH 05/10] chore: 2fa fixes (#503) * chore: 2fa initial fixes * chore: support for Telegram 2FA * chore: update swagger * chore: proper error checking * docs: update to docs to support 2fa * fix where app was using ApplicationVersion instead of ProtocolVersion * Feat/libp2p worker (#504) * feat: refactor sendWork function in workers.go to make modular - Separate out localWorker to send local work requests - Separate out remoteWorker to enable selection and handling of remote work requests - Lay foundation for remote worker selection - even though queue is limited all remoteWorkers still get the same request which is inefficient - Log number of responses in the queue - Make logs verbose for easy debugging of localWork and remoteWork * feat: worker-sub selection refactor * feat: Implement round-robin worker selection for distributed task processing - Add round-robin iterator for fair worker distribution - Prioritize local worker when eligible - Cycle through remote workers in subsequent calls - Improve error handling and logging for worker responses - Enhance code readability and maintainability This update ensures a balanced workload across all available workers over time, while still prioritizing local processing when possible. * Improve error handling and resilience in send_work.go - Add maxSpawnAttempts constant to limit remote worker spawn attempts - Implement retry mechanism for spawning remote workers with exponential backoff - Introduce spawnRemoteWorker function for better organization and error handling - Enhance logging for better visibility into worker spawning and processing - Improve handling of dead letters and timeouts in remote worker operations - Refactor handleRemoteWorker to be more robust against transient failures - Update tryWorker function to handle both local and remote worker scenarios - Implement round-robin worker selection with retries in SendWork function These changes aim to increase the reliability of the worker system, particularly when dealing with remote workers, and provide better insights into error scenarios for easier debugging and monitoring. * feat: add config.go to worker package to simplify settings * refactor(workers): move worker selection logic to separate file - Created new file worker_selection.go to house worker selection logic - Moved GetEligibleWorkers, isEligibleRemoteWorker, and RoundRobinIterator to worker_selection.go - Updated send_work.go to use new exported functions from worker_selection.go - Renamed newRoundRobinIterator to NewRoundRobinIterator for proper exporting - Updated imports and function calls in send_work.go to reflect new structure - Improved code organization and modularity * fix: duplication of getEligibleWorkers func * feat: Enhance worker selection process with configurable remote worker limit - Add MaxRemoteWorkers to WorkerConfig in config.go - Update tryWorkersRoundRobin function in send_work.go to respect MaxRemoteWorkers limit - Implement fallback mechanism to local worker when remote workers fail or are unavailable - Add detailed logging throughout the worker selection process for better debugging - Ensure a last resort local worker is always available if no other workers are found * fix: fine tune timeouts with testing * Refactor worker selection and add eligibility check Refactored worker selection to use NodeTracker's new GetEligibleWorkerNodes method and introduced an eligibility check for staked workers. Added a new utility function for converting strings to WorkerTypes and a method in NodeEventTracker to retrieve eligible nodes based on work categories. * fixed case where err was reassigned and hiding the original * fix: local work is not timing out for long queries >20s * added more error handling and bubbled up to the tryWorker level * chore: swagger updates (#494) * ci: update workflow for deploy based on tag. (#485) * update workflow for deploy based on tag. * give workflow better name, fix env. refs. * ci: remove unneeded / untested workflows (#487) * update workflow for deploy based on tag. * give workflow better name, fix env. refs. * this needs work, not functional now. * handy cleanup script to clear out old workflows * give the workflow for running Go tests a more descriptive name * remove duplicate workflow. * update cleanup script to handle multiple workflows with same name * fix auth pattern to GCP * fix: repair deploy portion of the build and deploy to gcp bootnodes workflow (#488) * update workflow for deploy based on tag. * give workflow better name, fix env. refs. * this needs work, not functional now. * handy cleanup script to clear out old workflows * give the workflow for running Go tests a more descriptive name * remove duplicate workflow. * update cleanup script to handle multiple workflows with same name * fix auth pattern to GCP * update workflow to deploy to test env. in GCP * chore: 2fa initial fixes * stash * chore: updates to docs.go for swagger --------- Co-authored-by: J2D3 <156010594+5u6r054@users.noreply.github.com> * Enable randomized node selection and improve config settings Shuffle eligible nodes before selecting workers. Added context-based connection attempts to ensure we can connect to selected worker before returning. Adjusted AppConfig initialization order and added version flag for CLI configuration. * x: add error handling for peer info creation and fine tune timeouts - Check and log errors from NewMultiaddr and AddrInfoFromP2pAddr - Resolve "value of err is never used" linter warning - Change timeouts on workers to 20s and 15s respectively * chore: add configurable connection timeout to config.go - Add ConnectionTimeout to WorkerConfig in config.go - Update GetEligibleWorkers to use the configurable timeout * chore: revert makefile version tagger * Adjust time intervals and remove unnecessary peer removal Updated sleep durations to one minute for efficiency and commented out peer removal logic to retain node activity tracking. Also removed the callback for peer removal in DHT initialization. * resolving comments in PR * fix: cleanup logs and tune timeouts * Add worker management for distributed task handling Implemented `WorkHandlerManager` to manage different work handlers, and integrated it for handling remote and local worker tasks. Introduced new functions in the workers package to execute and manage work streams. Updated API layer to utilize the new worker management system for request distribution and response handling. * fix multiaddress and local worker assignment * fix incorrect node reference * Add length-prefix to stream communication This update adds a length-prefix to both request and response messages in the stream communication to ensure proper message boundary handling. It includes encoding the length of the messages using big-endian encoding before writing to the stream and reading the length before processing the actual messages. This change improves the robustness and reliability of streaming operations. * update logging and pass back original error it happened on remote work. * Remove actor framework and work distribution logic Deleted all files and scripts related to the actor-based framework and work distribution. This includes the removal of `workers.go`, `handlers.go`, and corresponding protobuf messages and build scripts. Restructured the package and broke up the handlers into smaller more manageable files * Refactor error handling and JSON tags in worker types Updated the error field in WorkResponse to a string as it should have been. Also, added JSON tags to struct fields for better JSON serialization. updated assignment of Error in handler functions to use formatted strings to conform to the data type change. * Fixed introduced bug with error in remote worker also fixed logging to correctly display peerId when errors occur. * fixed incorrectly scoped assignment to response it needed to be the method declared not a new variable * fixing error messages * set request to nil so that it is not in the resulting JSON * fix: empty tweet case causing panic * fix: linting error and tidy * chore: Version 0.6.0 --------- Co-authored-by: Bob Stevens <35038919+restevens402@users.noreply.github.com> Co-authored-by: Nolan Jacobson <50815660+nolanjacobson@users.noreply.github.com> Co-authored-by: J2D3 <156010594+5u6r054@users.noreply.github.com> * chore: updates to docs (#495) * chore: updates to docs * chore: updates to docs * chore: updates to docs --------- Co-authored-by: Brendan Playford <34052452+teslashibe@users.noreply.github.com> * chore: 2fa initial fixes * chore: support for Telegram 2FA * chore: update swagger * chore: proper error checking * docs: update to docs to support 2fa --------- Co-authored-by: J2D3 <156010594+5u6r054@users.noreply.github.com> Co-authored-by: Brendan Playford <34052452+teslashibe@users.noreply.github.com> Co-authored-by: Bob Stevens <35038919+restevens402@users.noreply.github.com> --- docs/docs.go | 1206 ++++++++++++++++++++++ docs/worker-node/telegram-worker.md | 2 + pkg/api/handlers_data.go | 3 +- pkg/scrapers/telegram/telegram_client.go | 19 +- 4 files changed, 1225 insertions(+), 5 deletions(-) diff --git a/docs/docs.go b/docs/docs.go index 28b9199e..94d35e82 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -48,6 +48,1212 @@ const docTemplate = `{ "name": "username", "in": "path", "required": true + "schemes": {{ marshal .Schemes }}, + "swagger": "2.0", + "info": { + "description": "{{escape .Description}}", + "title": "{{.Title}}", + "contact": { + "name": "Masa API Support", + "url": "https://masa.ai", + "email": "support@masa.ai" + }, + "license": { + "name": "MIT", + "url": "https://opensource.org/license/mit" + }, + "version": "{{.Version}}" + }, + "host": "{{.Host}}", + "basePath": "{{.BasePath}}", + "securityDefinitions": { + "Bearer": { + "type": "apiKey", + "name": "Authorization", + "in": "header" + } + }, + "security": [ + { + "Bearer": [] + } + ], + "paths": { + "/peers": { + "get": { + "description": "Retrieves a list of peers connected to the node", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "Peers" + ], + "summary": "Get list of peers", + "responses": { + "200": { + "description": "List of peer IDs", + "schema": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "security": [ + { + "Bearer": [] + } + ] + } + }, + "/peer/addresses": { + "get": { + "description": "Retrieves a list of peer addresses connected to the node", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "Peers" + ], + "summary": "Get peer addresses", + "responses": { + "200": { + "description": "List of peer addresses", + "schema": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "security": [ + { + "Bearer": [] + } + ] + } + }, + "/data/twitter/profile/{username}": { + "get": { + "description": "Retrieves tweets from a specific Twitter profile", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "Twitter" + ], + "summary": "Search Twitter Profile", + "parameters": [ + { + "type": "string", + "description": "Twitter Username", + "name": "username", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "List of tweets from the profile", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/Tweet" + } + } + }, + "400": { + "description": "Invalid username or error fetching tweets", + "schema": { + "$ref": "#/definitions/ErrorResponse" + } + } + }, + "security": [ + { + "Bearer": [] + } + ] + } + }, + "/data/twitter/followers/{username}": { + "get": { + "description": "Retrieves followers from a specific Twitter profile.", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "Twitter" + ], + "summary": "Search Followers by Twitter Username", + "parameters": [ + { + "type": "string", + "description": "Twitter Username", + "name": "username", + "in": "path", + "required": true + }, + { + "type": "integer", + "description": "Maximum number of users to return", + "name": "count", + "in": "query", + "required": false, + "default": 20 + } + ], + "responses": { + "200": { + "description": "Array of profiles a user has as followers", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/Profile" + } + } + }, + "400": { + "description": "Invalid username or error fetching followers", + "schema": { + "$ref": "#/definitions/ErrorResponse" + } + } + } + } + }, + "/data/twitter/tweets/recent": { + "post": { + "description": "Retrieves recent tweets based on query parameters", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "Twitter" + ], + "summary": "Search recent tweets", + "parameters": [ + { + "in": "body", + "name": "body", + "description": "Search parameters", + "required": true, + "schema": { + "type": "object", + "properties": { + "query": { + "type": "string", + "description": "Search Query" + }, + "count": { + "type": "integer", + "description": "Number of tweets to return" + } + } + } + } + ], + "responses": { + "200": { + "description": "List of recent tweets", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/Tweet" + } + } + }, + "400": { + "description": "Invalid query or error fetching tweets", + "schema": { + "$ref": "#/definitions/ErrorResponse" + } + } + }, + "security": [ + { + "Bearer": [] + } + ] + } + }, + "/data/twitter/tweets/trends": { + "get": { + "description": "Retrieves the latest Twitter trending topics", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "Twitter" + ], + "summary": "Twitter Trends", + "responses": { + "200": { + "description": "List of trending topics", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/Trend" + } + } + }, + "400": { + "description": "Error fetching Twitter trends", + "schema": { + "$ref": "#/definitions/ErrorResponse" + } + } + }, + "security": [ + { + "Bearer": [] + } + ] + } + }, + "/data/discord/profile/{userID}": { + "get": { + "description": "Retrieves a Discord user profile by user ID.", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "Discord" + ], + "summary": "Search Discord Profile", + "parameters": [ + { + "name": "userID", + "in": "path", + "description": "Discord User ID", + "required": true, + "type": "string" + } + ], + "responses": { + "200": { + "description": "Successfully retrieved Discord user profile", + "schema": { + "$ref": "#/definitions/UserProfile" + } + }, + "400": { + "description": "Invalid user ID or error fetching profile", + "schema": { + "$ref": "#/definitions/ErrorResponse" + } + } + }, + "security": [ + { + "Bearer": [] + } + ] + } + }, + "/data/discord/channels/{channelID}/messages": { + "get": { + "description": "Retrieves messages from a specified Discord channel.", + "tags": ["Discord"], + "summary": "Get messages from a Discord channel", + "parameters": [ + { + "name": "channelID", + "in": "path", + "description": "Discord Channel ID", + "required": true, + "type": "string" + }, + { + "name": "limit", + "in": "query", + "description": "The maximum number of messages to return", + "required": false, + "type": "integer", + "format": "int32" + }, + { + "name": "before", + "in": "query", + "description": "A message ID to return messages posted before this message", + "required": false, + "type": "string" + } + ], + "responses": { + "200": { + "description": "Successfully retrieved messages from the Discord channel", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/ChannelMessage" + } + } + }, + "400": { + "description": "Invalid channel ID or error fetching messages", + "schema": { + "$ref": "#/definitions/ErrorResponse" + } + } + }, + "security": [ + { + "Bearer": [] + } + ] + } + }, + "/data/discord/guilds/{guildID}/channels": { + "get": { + "description": "Retrieves channels from a specified Discord guild.", + "tags": ["Discord"], + "summary": "Get channels from a Discord guild", + "parameters": [ + { + "name": "guildID", + "in": "path", + "description": "Discord Guild ID", + "required": true, + "type": "string" + } + ], + "responses": { + "200": { + "description": "Successfully retrieved channels from the Discord guild", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/GuildChannel" + } + } + }, + "400": { + "description": "Invalid guild ID or error fetching channels", + "schema": { + "$ref": "#/definitions/ErrorResponse" + } + } + }, + "security": [ + { + "Bearer": [] + } + ] + } + }, + "/data/discord/user/guilds": { + "get": { + "description": "Retrieves guilds from a specified Discord user.", + "tags": ["Discord"], + "summary": "Get guilds from a Discord user", + "parameters": [ + ], + "responses": { + "200": { + "description": "Successfully retrieved guilds from the Discord user", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/UserGuild" + } + } + }, + "400": { + "description": "Invalid user ID or error fetching guilds", + "schema": { + "$ref": "#/definitions/ErrorResponse" + } + } + }, + "security": [ + { + "Bearer": [] + } + ] + } + }, + "/data/discord/guilds/all": { + "get": { + "description": "Retrieves all guilds that all the Discord workers are apart of.", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "Discord" + ], + "summary": "Get all guilds", + "responses": { + "200": { + "description": "Successfully retrieved all guilds for the Discord user", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/Guild" + } + } + }, + "400": { + "description": "Error fetching guilds or invalid access token", + "schema": { + "$ref": "#/definitions/ErrorResponse" + } + } + }, + "security": [ + { + "Bearer": [] + } + ] + } + }, + "/data/telegram/channel/messages": { + "post": { + "description": "Retrieves messages from a specified Telegram channel.", + "tags": ["Telegram"], + "summary": "Get Telegram Channel Messages", + "parameters": [ + { + "in": "body", + "name": "body", + "description": "Request body", + "required": true, + "schema": { + "type": "object", + "properties": { + "username": { + "type": "string", + "description": "Telegram Username" + } + }, + "required": ["username"] + } + } + ], + "responses": { + "200": { + "description": "Successfully retrieved messages", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/Message" + } + } + }, + "400": { + "description": "Invalid username or error fetching messages", + "schema": { + "$ref": "#/definitions/ErrorResponse" + } + } + }, + "security": [ + { + "Bearer": [] + } + ] + } + }, + "/data/web": { + "post": { + "description": "Retrieves data from the web", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "Web" + ], + "summary": "Web Data", + "parameters": [ + { + "in": "body", + "name": "body", + "description": "Search parameters", + "required": true, + "schema": { + "type": "object", + "properties": { + "url": { + "type": "string", + "description": "Url" + }, + "depth": { + "type": "integer", + "description": "Number of pages to scrape" + } + } + } + } + ], + "responses": { + "200": { + "description": "Successfully retrieved web data", + "schema": { + "$ref": "#/definitions/WebDataResponse" + } + }, + "400": { + "description": "Invalid query or error fetching web data", + "schema": { + "$ref": "#/definitions/ErrorResponse" + } + } + }, + "security": [ + { + "Bearer": [] + } + ] + } + }, + "/dht": { + "get": { + "description": "Retrieves data from the DHT (Distributed Hash Table)", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "DHT" + ], + "summary": "Get DHT Data", + "parameters": [ + { + "in": "query", + "name": "key", + "description": "Key to retrieve data for", + "required": true, + "type": "string" + } + ], + "responses": { + "200": { + "description": "Successfully retrieved data from DHT", + "schema": { + "$ref": "#/definitions/DHTResponse" + } + }, + "400": { + "description": "Error retrieving data from DHT", + "schema": { + "$ref": "#/definitions/ErrorResponse" + } + } + }, + "security": [ + { + "Bearer": [] + } + ] + }, + "post": { + "description": "Adds data to the DHT (Distributed Hash Table)", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "DHT" + ], + "summary": "Post to DHT", + "parameters": [ + { + "description": "Data to store in DHT", + "name": "data", + "in": "body", + "required": true, + "schema": { + "type": "object", + "properties": { + "key": { + "type": "string" + }, + "value": { + "type": "string" + } + } + } + } + ], + "responses": { + "200": { + "description": "Successfully added data to DHT", + "schema": { + "$ref": "#/definitions/SuccessResponse" + } + }, + "400": { + "description": "Error adding data to DHT", + "schema": { + "$ref": "#/definitions/ErrorResponse" + } + } + }, + "security": [ + { + "Bearer": [] + } + ] + } + }, + "/llm/models": { + "get": { + "description": "Retrieves the available LLM models", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "LLM" + ], + "summary": "Get LLM Models", + "responses": { + "200": { + "description": "Successfully retrieved LLM models", + "schema": { + "$ref": "#/definitions/LLMModelsResponse" + } + }, + "400": { + "description": "Error retrieving LLM models", + "schema": { + "$ref": "#/definitions/ErrorResponse" + } + } + }, + "security": [ + { + "Bearer": [] + } + ] + } + }, + "/chat": { + "post": { + "summary": "Chat with AI", + "description": "Initiates a chat session with an AI model.", + "consumes": ["application/json"], + "produces": ["application/json"], + "parameters": [ + { + "in": "body", + "name": "body", + "description": "Chat request payload", + "required": true, + "schema": { + "type": "object", + "properties": { + "model": { + "type": "string", + "example": "llama3" + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "properties": { + "role": { + "type": "string", + "example": "user" + }, + "content": { + "type": "string", + "example": "why is the sky blue?" + } + } + } + }, + "stream": { + "type": "boolean", + "example": false + } + }, + "required": ["model", "messages", "stream"] + } + } + ], + "responses": { + "200": { + "description": "Successfully received response from AI", + "schema": { + "$ref": "#/definitions/ChatResponse" + } + }, + "400": { + "description": "Error communicating with AI", + "schema": { + "$ref": "#/definitions/ErrorResponse" + } + } + } + } + }, + "/node/data": { + "get": { + "description": "Retrieves data from the node", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "Node" + ], + "summary": "Node Data", + "responses": { + "200": { + "description": "Successfully retrieved node data", + "schema": { + "$ref": "#/definitions/NodeDataResponse" + } + }, + "400": { + "description": "Error retrieving node data", + "schema": { + "$ref": "#/definitions/ErrorResponse" + } + } + }, + "security": [ + { + "Bearer": [] + } + ] + } + }, + "/node/data/{peerid}": { + "get": { + "description": "Retrieves data for a specific node identified by peer ID", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "Node" + ], + "summary": "Get Node Data by Peer ID", + "parameters": [ + { + "type": "string", + "description": "Peer ID", + "name": "peerid", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "Successfully retrieved node data by peer ID", + "schema": { + "$ref": "#/definitions/NodeDataResponse" + } + }, + "400": { + "description": "Error retrieving node data by peer ID", + "schema": { + "$ref": "#/definitions/ErrorResponse" + } + } + }, + "security": [ + { + "Bearer": [] + } + ] + } + }, + "/sentiment/tweets": { + "post": { + "description": "Searches for tweets and analyzes their sentiment", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "Sentiment" + ], + "summary": "Analyze Sentiment of Tweets", + "parameters": [ + { + "in": "body", + "name": "body", + "description": "Sentiment analysis request body", + "required": true, + "schema": { + "type": "object", + "properties": { + "query": { + "type": "string", + "description": "Search Query" + }, + "count": { + "type": "integer", + "description": "Number of tweets to analyze" + }, + "model": { + "type": "string", + "description": "Sentiment analysis model to use" + } + } + } + } + ], + "responses": { + "200": { + "description": "Successfully analyzed sentiment of tweets", + "schema": { + "$ref": "#/definitions/SentimentAnalysisResponse" + } + }, + "400": { + "description": "Error analyzing sentiment of tweets", + "schema": { + "$ref": "#/definitions/ErrorResponse" + } + } + } + } + }, + "/sentiment/telegram": { + "post": { + "description": "Searches for Telegram messages and analyzes their sentiment", + "tags": ["Sentiment"], + "summary": "Analyze Sentiment of Telegram Messages", + "consumes": ["application/json"], + "produces": ["application/json"], + "parameters": [ + { + "name": "query", + "in": "body", + "description": "Search Query", + "required": true, + "schema": { + "type": "object", + "properties": { + "query": { + "type": "string" + } + } + } + } + ], + "responses": { + "200": { + "description": "Successfully analyzed sentiment of Telegram messages", + "schema": { + "$ref": "#/definitions/SentimentAnalysisResponse" + } + }, + "400": { + "description": "Error analyzing sentiment of Telegram messages", + "schema": { + "$ref": "#/definitions/ErrorResponse" + } + } + } + } + }, + "/sentiment/discord": { + "post": { + "description": "Searches for Discord messages and analyzes their sentiment", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "Sentiment" + ], + "summary": "Analyze Sentiment of Discord Messages", + "parameters": [ + { + "in": "body", + "name": "body", + "description": "Sentiment analysis request body for Discord messages", + "required": true, + "schema": { + "type": "object", + "properties": { + "channelID": { + "type": "string", + "description": "Discord Channel ID" + }, + "prompt": { + "type": "string", + "description": "Prompt to enter" + }, + "model": { + "type": "string", + "description": "Sentiment analysis model to use" + } + }, + "required": ["channelID", "model"] + } + } + ], + "responses": { + "200": { + "description": "Successfully analyzed sentiment of Discord messages", + "schema": { + "$ref": "#/definitions/SentimentAnalysisResponse" + } + }, + "400": { + "description": "Error analyzing sentiment of Discord messages", + "schema": { + "$ref": "#/definitions/ErrorResponse" + } + } + } + } + }, + "/auth": { + "get": { + "description": "Retrieves the API key for the node", + "produces": [ + "application/json" + ], + "tags": [ + "Authentication" + ], + "summary": "Get Node API Key", + "responses": { + "200": { + "description": "Successfully retrieved API key", + "schema": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + }, + "500": { + "description": "Error generating API key", + "schema": { + "$ref": "#/definitions/ErrorResponse" + } + } + } + } + }, + "/auth/telegram/start": { + "post": { + "description": "Initiates the authentication process with Telegram by sending a code to the provided phone number.", + "tags": ["Authentication"], + "summary": "Start Telegram Authentication", + "consumes": ["application/json"], + "produces": ["application/json"], + "parameters": [ + { + "name": "phone_number", + "in": "body", + "description": "Phone Number", + "required": true, + "schema": { + "type": "object", + "properties": { + "phone_number": { + "type": "string" + } + } + } + } + ], + "responses": { + "200": { + "description": "Successfully sent authentication code", + "schema": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + }, + "400": { + "description": "Invalid request body", + "schema": { + "$ref": "#/definitions/ErrorResponse" + } + }, + "500": { + "description": "Failed to initialize Telegram client or to start authentication", + "schema": { + "$ref": "#/definitions/ErrorResponse" + } + } + } + } + }, + "/auth/telegram/complete": { + "post": { + "description": "Completes the authentication process with Telegram using the code sent to the phone number.", + "tags": ["Authentication"], + "summary": "Complete Telegram Authentication", + "consumes": ["application/json"], + "produces": ["application/json"], + "parameters": [ + { + "name": "phone_number", + "in": "body", + "description": "Phone Number", + "required": true, + "schema": { + "type": "object", + "properties": { + "phone_number": { + "type": "string" + }, + "code": { + "type": "string" + }, + "phone_code_hash": { + "type": "string" + }, + "password": { + "type": "string", + "description": "Optional password for two-factor authentication" + } + }, + "required": ["phone_number", "code", "phone_code_hash"] + } + } + ], + "responses": { + "200": { + "description": "Successfully authenticated", + "schema": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + }, + "400": { + "description": "Invalid request body", + "schema": { + "$ref": "#/definitions/ErrorResponse" + } + }, + "401": { + "description": "Two-factor authentication is required", + "schema": { + "$ref": "#/definitions/ErrorResponse" + } + }, + "500": { + "description": "Failed to initialize Telegram client or to complete authentication", + "schema": { + "$ref": "#/definitions/ErrorResponse" + } + } + } + } + }, + "/sentiment/web": { + "post": { + "description": "Searches for web content and analyzes its sentiment", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "Sentiment" + ], + "summary": "Analyze Sentiment of Web Content", + "parameters": [ + { + "in": "body", + "name": "body", + "description": "Sentiment analysis request body", + "required": true, + "schema": { + "type": "object", + "properties": { + "url": { + "type": "string", + "description": "URL of the web content" + }, + "depth": { + "type": "integer", + "description": "Depth of web crawling" + }, + "model": { + "type": "string", + "description": "Sentiment analysis model to use" + } + } + } + } + ], + "responses": { + "200": { + "description": "Successfully analyzed sentiment of web content", + "schema": { + "$ref": "#/definitions/SentimentAnalysisResponse" + } + }, + "400": { + "description": "Error analyzing sentiment of web content", + "schema": { + "$ref": "#/definitions/ErrorResponse" + } + } + } + } + }, + }, + "DHTResponse": { + "type": "object", + "properties": { + "key": { + "type": "string" + }, + "value": { + "type": "string" + } } ], "responses": { diff --git a/docs/worker-node/telegram-worker.md b/docs/worker-node/telegram-worker.md index c9badcc4..323ac686 100644 --- a/docs/worker-node/telegram-worker.md +++ b/docs/worker-node/telegram-worker.md @@ -58,6 +58,8 @@ TELEGRAM_SCRAPER=true 1. Call the `/api/v1/auth/telegram/complete` endpoint with the code you received on your phone, your phone number, and the `phone_code_hash` from the previous step. +*Note* - If you have 2FA turned on, you will need to pass your 2FA password in the API call. + ### Verifying Node Configuration Ensure your node is correctly configured to handle Twitter data requests by checkint the initialization message: diff --git a/pkg/api/handlers_data.go b/pkg/api/handlers_data.go index 4e7e7504..c7c3950e 100644 --- a/pkg/api/handlers_data.go +++ b/pkg/api/handlers_data.go @@ -887,13 +887,14 @@ func (api *API) CompleteAuth() gin.HandlerFunc { PhoneNumber string `json:"phone_number"` Code string `json:"code"` PhoneCodeHash string `json:"phone_code_hash"` + Password string `json:"password"` } if err := c.ShouldBindJSON(&reqBody); err != nil { c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid request body"}) return } - auth, err := telegram.CompleteAuthentication(context.Background(), reqBody.PhoneNumber, reqBody.Code, reqBody.PhoneCodeHash) + auth, err := telegram.CompleteAuthentication(context.Background(), reqBody.PhoneNumber, reqBody.Code, reqBody.PhoneCodeHash, reqBody.Password) if err != nil { // Check if 2FA is required if err.Error() == "2FA required" { diff --git a/pkg/scrapers/telegram/telegram_client.go b/pkg/scrapers/telegram/telegram_client.go index 8bc42c8a..836d2ea3 100644 --- a/pkg/scrapers/telegram/telegram_client.go +++ b/pkg/scrapers/telegram/telegram_client.go @@ -8,6 +8,7 @@ import ( "os" "path/filepath" "strconv" + "strings" "sync" "github.com/gotd/contrib/bg" @@ -113,12 +114,13 @@ func StartAuthentication(ctx context.Context, phoneNumber string) (string, error } // CompleteAuthentication uses the provided code to authenticate with Telegram. -func CompleteAuthentication(ctx context.Context, phoneNumber, code, phoneCodeHash string) (*tg.AuthAuthorization, error) { +// CompleteAuthentication uses the provided code to authenticate with Telegram. +func CompleteAuthentication(ctx context.Context, phoneNumber, code, phoneCodeHash, password string) (*tg.AuthAuthorization, error) { // Initialize the Telegram client (if not already initialized) client, err := GetClient() if err != nil { logrus.Printf("Failed to initialize Telegram client: %v", err) - return nil, err // Edit: Added nil as the first return value + return nil, err } // Define a variable to hold the authentication result @@ -127,9 +129,18 @@ func CompleteAuthentication(ctx context.Context, phoneNumber, code, phoneCodeHas err = client.Run(ctx, func(ctx context.Context) error { // Use the provided code and phoneCodeHash to authenticate auth, err := client.Auth().SignIn(ctx, phoneNumber, code, phoneCodeHash) + if err != nil { - log.Printf("Error during SignIn: %v", err) - return err + if strings.Contains(err.Error(), "2FA required") { + auth, err = client.Auth().Password(ctx, password) + if err != nil { + log.Printf("Error during 2FA SignIn: %v", err) + return err + } + } else { + log.Printf("Error during SignIn: %v", err) + return err + } } // At this point, authentication was successful, and you have the user's Telegram auth data. From 2b336549ada18e2c2d9a6be4e1cf7ac4caee6663 Mon Sep 17 00:00:00 2001 From: Nolan Jacobson <50815660+nolanjacobson@users.noreply.github.com> Date: Fri, 16 Aug 2024 16:35:06 -0400 Subject: [PATCH 06/10] chore: Go releaser workflow (#491) * chore: add goreleaser workflows Signed-off-by: mudler * chore: add test workflow Signed-off-by: mudler * fix: use github-native for release notes generation See also: https://goreleaser.com/customization/changelog/?h=changelog#changelog Signed-off-by: mudler --------- Signed-off-by: mudler Co-authored-by: mudler Co-authored-by: Brendan Playford <34052452+teslashibe@users.noreply.github.com> --- .github/workflows/build-goreleaser-test.yml | 25 ++++++++++++++ .github/workflows/goreleaser.yml | 30 ++++++++++++++++ .gitignore | 3 ++ .goreleaser.yml | 38 +++++++++++++++++++++ 4 files changed, 96 insertions(+) create mode 100644 .github/workflows/build-goreleaser-test.yml create mode 100644 .github/workflows/goreleaser.yml create mode 100644 .goreleaser.yml diff --git a/.github/workflows/build-goreleaser-test.yml b/.github/workflows/build-goreleaser-test.yml new file mode 100644 index 00000000..761f013d --- /dev/null +++ b/.github/workflows/build-goreleaser-test.yml @@ -0,0 +1,25 @@ +name: goreleaser-build-test + +on: + push: + pull_request: + +jobs: + goreleaser: # Add this new job for GoReleaser + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: 1.22 + - name: Run GoReleaser + uses: goreleaser/goreleaser-action@v6 + with: + version: v2.1.0 + args: build --clean --snapshot + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/goreleaser.yml b/.github/workflows/goreleaser.yml new file mode 100644 index 00000000..dccaa4f8 --- /dev/null +++ b/.github/workflows/goreleaser.yml @@ -0,0 +1,30 @@ +name: goreleaser + +on: + push: + tags: + - 'v*' # Add this line to trigger the workflow on tag pushes that match 'v*' + +permissions: + id-token: write + contents: read + +jobs: + goreleaser: # Add this new job for GoReleaser + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: 1.22 + - name: Run GoReleaser + uses: goreleaser/goreleaser-action@v6 + with: + version: v2.1.0 + args: release --clean + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore index 3fc6bb2b..eaa12583 100644 --- a/.gitignore +++ b/.gitignore @@ -69,3 +69,6 @@ docs/api-reference.md transcription.txt snippets.txt .env copy + +# Build result of goreleaser +dist/ diff --git a/.goreleaser.yml b/.goreleaser.yml new file mode 100644 index 00000000..230d00e2 --- /dev/null +++ b/.goreleaser.yml @@ -0,0 +1,38 @@ +# Make sure to check the documentation at http://goreleaser.com +version: 2 +builds: + - main: ./cmd/masa-node/main.go + ldflags: + - -w -s + - -X github.com/masa-finance/masa-oracle/internal.Version={{.Tag}} + - -X github.com/masa-finance/masa-oracle/internal.Commit={{.Commit}} + env: + - CGO_ENABLED=0 + goos: + - linux + - windows + - darwin + - freebsd + goarch: + - amd64 + - arm + - arm64 +source: + enabled: true + name_template: '{{ .ProjectName }}-{{ .Tag }}-source' +archives: + # Default template uses underscores instead of - + - name_template: >- + {{ .ProjectName }}-{{ .Tag }}- + {{- if eq .Os "freebsd" }}FreeBSD + {{- else }}{{- title .Os }}{{end}}- + {{- if eq .Arch "amd64" }}x86_64 + {{- else if eq .Arch "386" }}i386 + {{- else }}{{ .Arch }}{{end}} + {{- if .Arm }}v{{ .Arm }}{{ end }} +checksum: + name_template: '{{ .ProjectName }}-{{ .Tag }}-checksums.txt' +snapshot: + name_template: "{{ .Tag }}-next" +changelog: + use: github-native \ No newline at end of file From fb2b0cece26f6c3dcef408caad4fbde2b39f82cf Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Fri, 16 Aug 2024 22:35:37 +0200 Subject: [PATCH 07/10] ci: add static checker (#417) This adds static checker scanning to PRs Co-authored-by: J2D3 <156010594+5u6r054@users.noreply.github.com> Co-authored-by: Brendan Playford <34052452+teslashibe@users.noreply.github.com> --- .github/workflows/static.yml | 70 ++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 .github/workflows/static.yml diff --git a/.github/workflows/static.yml b/.github/workflows/static.yml new file mode 100644 index 00000000..1bc752c8 --- /dev/null +++ b/.github/workflows/static.yml @@ -0,0 +1,70 @@ +name: static check +on: pull_request + +jobs: + imports: + name: Imports + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@master + - name: check + uses: danhunsaker/golang-github-actions@v1.3.0 + with: + run: imports + token: ${{ secrets.GITHUB_TOKEN }} + + errcheck: + name: Errcheck + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@master + - name: check + uses: danhunsaker/golang-github-actions@v1.3.0 + with: + run: errcheck + token: ${{ secrets.GITHUB_TOKEN }} + + lint: + name: Lint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@master + - name: check + uses: danhunsaker/golang-github-actions@v1.3.0 + with: + run: lint + token: ${{ secrets.GITHUB_TOKEN }} + + shadow: + name: Shadow + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@master + - name: check + uses: danhunsaker/golang-github-actions@v1.3.0 + with: + run: shadow + token: ${{ secrets.GITHUB_TOKEN }} + + staticcheck: + name: StaticCheck + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@master + - name: check + uses: danhunsaker/golang-github-actions@v1.3.0 + with: + run: staticcheck + token: ${{ secrets.GITHUB_TOKEN }} + + sec: + name: Sec + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@master + - name: check + uses: danhunsaker/golang-github-actions@v1.3.0 + with: + run: sec + token: ${{ secrets.GITHUB_TOKEN }} + flags: "-exclude=G104" From 8ddc834ea6fc49d17c1345fd3838029eedbb5658 Mon Sep 17 00:00:00 2001 From: Nolan Jacobson <50815660+nolanjacobson@users.noreply.github.com> Date: Fri, 16 Aug 2024 16:36:27 -0400 Subject: [PATCH 08/10] docs: updates for Exact Search in docs (#506) Co-authored-by: Brendan Playford <34052452+teslashibe@users.noreply.github.com> --- docs/oracle-node/twitter-data.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/oracle-node/twitter-data.md b/docs/oracle-node/twitter-data.md index 4eba986d..a4c9a281 100644 --- a/docs/oracle-node/twitter-data.md +++ b/docs/oracle-node/twitter-data.md @@ -211,6 +211,18 @@ Example response: The Advanced Search feature allows users to perform more complex queries to filter tweets according to various criteria such as date ranges, specific users, hashtags, and more. Below you will find detailed information on how to construct advanced search queries. +### Exact Search +Search for tweets containing specific hashtags. + +**Syntax:** `"\"searchterm\"` + +**Example:** + +```bash +curl -X POST http://localhost:8080/api/v1/data/twitter/tweets/recent \ +-H "Content-Type: application/json" \ +-d '{"query": "\"masa\", "count": 10}' +``` ### Hashtag Search Search for tweets containing specific hashtags. From c53c66a1c3b66b27dde1a4ef90f77bd6ed456e25 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 16 Aug 2024 13:37:19 -0700 Subject: [PATCH 09/10] build(deps): bump github.com/ethereum/go-ethereum from 1.14.7 to 1.14.8 (#502) Bumps [github.com/ethereum/go-ethereum](https://github.com/ethereum/go-ethereum) from 1.14.7 to 1.14.8. - [Release notes](https://github.com/ethereum/go-ethereum/releases) - [Commits](https://github.com/ethereum/go-ethereum/compare/v1.14.7...v1.14.8) --- updated-dependencies: - dependency-name: github.com/ethereum/go-ethereum dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Brendan Playford <34052452+teslashibe@users.noreply.github.com> --- go.mod | 6 +++--- go.sum | 14 ++++++-------- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/go.mod b/go.mod index 7865648e..33eedf47 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/chyeh/pubip v0.0.0-20170203095919-b7e679cf541c github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 github.com/dgraph-io/badger v1.6.2 - github.com/ethereum/go-ethereum v1.14.7 + github.com/ethereum/go-ethereum v1.14.8 github.com/fatih/color v1.17.0 github.com/gdamore/tcell/v2 v2.7.4 github.com/gin-contrib/cors v1.7.2 @@ -56,7 +56,7 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/bits-and-blooms/bitset v1.13.0 // indirect github.com/blang/semver/v4 v4.0.0 // indirect - github.com/btcsuite/btcd/btcec/v2 v2.3.2 // indirect + github.com/btcsuite/btcd/btcec/v2 v2.3.4 // indirect github.com/btcsuite/btcd/chaincfg/chainhash v1.1.0 // indirect github.com/bytedance/sonic v1.11.6 // indirect github.com/bytedance/sonic/loader v0.1.1 // indirect @@ -117,7 +117,7 @@ require ( github.com/hashicorp/golang-lru v1.0.2 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/hashicorp/hcl v1.0.0 // indirect - github.com/holiman/uint256 v1.3.0 // indirect + github.com/holiman/uint256 v1.3.1 // indirect github.com/huin/goupnp v1.3.0 // indirect github.com/ipfs/boxo v0.18.0 // indirect github.com/ipfs/go-log v1.0.5 // indirect diff --git a/go.sum b/go.sum index 7e4fc34e..dd030d99 100644 --- a/go.sum +++ b/go.sum @@ -50,8 +50,8 @@ github.com/bits-and-blooms/bitset v1.13.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6 github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g= -github.com/btcsuite/btcd/btcec/v2 v2.3.2 h1:5n0X6hX0Zk+6omWcihdYvdAlGf2DfasC0GMf7DClJ3U= -github.com/btcsuite/btcd/btcec/v2 v2.3.2/go.mod h1:zYzJ8etWJQIv1Ogk7OzpWjowwOdXY1W/17j2MW85J04= +github.com/btcsuite/btcd/btcec/v2 v2.3.4 h1:3EJjcN70HCu/mwqlUsGK8GcNVyLVxFDlWurTXGPFfiQ= +github.com/btcsuite/btcd/btcec/v2 v2.3.4/go.mod h1:zYzJ8etWJQIv1Ogk7OzpWjowwOdXY1W/17j2MW85J04= github.com/btcsuite/btcd/chaincfg/chainhash v1.1.0 h1:59Kx4K6lzOW5w6nFlA0v5+lk/6sjybR934QNHSJZPTQ= github.com/btcsuite/btcd/chaincfg/chainhash v1.1.0/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc= github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= @@ -149,14 +149,12 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/ethereum/c-kzg-4844 v1.0.0 h1:0X1LBXxaEtYD9xsyj9B9ctQEZIpnvVDeoBx8aHEwTNA= github.com/ethereum/c-kzg-4844 v1.0.0/go.mod h1:VewdlzQmpT5QSrVhbBuGoCdFJkpaJlO1aQputP83wc0= -github.com/ethereum/go-ethereum v1.14.7 h1:EHpv3dE8evQmpVEQ/Ne2ahB06n2mQptdwqaMNhAT29g= -github.com/ethereum/go-ethereum v1.14.7/go.mod h1:Mq0biU2jbdmKSZoqOj29017ygFrMnB5/Rifwp980W4o= +github.com/ethereum/go-ethereum v1.14.8 h1:NgOWvXS+lauK+zFukEvi85UmmsS/OkV0N23UZ1VTIig= +github.com/ethereum/go-ethereum v1.14.8/go.mod h1:TJhyuDq0JDppAkFXgqjwpdlQApywnu/m10kFPxh8vvs= github.com/ethereum/go-verkle v0.1.1-0.20240306133620-7d920df305f0 h1:KrE8I4reeVvf7C1tm8elRjj4BdscTYzz/WAbYyf/JI4= github.com/ethereum/go-verkle v0.1.1-0.20240306133620-7d920df305f0/go.mod h1:D9AJLVXSyZQXJQVk8oh1EwjISE+sJTn2duYIZC0dy3w= github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4= github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI= -github.com/fjl/memsize v0.0.2 h1:27txuSD9or+NZlnOWdKUxeBzTAUkWCVh+4Gf2dWFOzA= -github.com/fjl/memsize v0.0.2/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/flynn/noise v1.1.0 h1:KjPQoQCEFdZDiP03phOvGi11+SVVhBG2wOWAorLsstg= github.com/flynn/noise v1.1.0/go.mod h1:xbMo+0i6+IGbYdJhF31t2eR1BIU0CYc12+BNAKwUTag= @@ -332,8 +330,8 @@ github.com/holiman/billy v0.0.0-20240216141850-2abb0c79d3c4 h1:X4egAf/gcS1zATw6w github.com/holiman/billy v0.0.0-20240216141850-2abb0c79d3c4/go.mod h1:5GuXa7vkL8u9FkFuWdVvfR5ix8hRB7DbOAaYULamFpc= github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZKVu0fao= github.com/holiman/bloomfilter/v2 v2.0.3/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iURXE7ZOP9L9hSkA= -github.com/holiman/uint256 v1.3.0 h1:4wdcm/tnd0xXdu7iS3ruNvxkWwrb4aeBQv19ayYn8F4= -github.com/holiman/uint256 v1.3.0/go.mod h1:EOMSn4q6Nyt9P6efbI3bueV4e1b3dGlUCXeiRV4ng7E= +github.com/holiman/uint256 v1.3.1 h1:JfTzmih28bittyHM8z360dCjIA9dbPIBlcTI6lmctQs= +github.com/holiman/uint256 v1.3.1/go.mod h1:EOMSn4q6Nyt9P6efbI3bueV4e1b3dGlUCXeiRV4ng7E= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huin/goupnp v1.3.0 h1:UvLUlWDNpoUdYzb2TCn+MuTWtcjXKSza2n6CBdQ0xXc= github.com/huin/goupnp v1.3.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= From f8291636de6c3f69775a61f96317f0604bf0f1f2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 16 Aug 2024 13:37:59 -0700 Subject: [PATCH 10/10] build(deps): bump github.com/ollama/ollama from 0.3.1 to 0.3.6 (#510) Bumps [github.com/ollama/ollama](https://github.com/ollama/ollama) from 0.3.1 to 0.3.6. - [Release notes](https://github.com/ollama/ollama/releases) - [Commits](https://github.com/ollama/ollama/compare/v0.3.1...v0.3.6) --- updated-dependencies: - dependency-name: github.com/ollama/ollama dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 5 +---- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 33eedf47..bec464b7 100644 --- a/go.mod +++ b/go.mod @@ -1,9 +1,6 @@ module github.com/masa-finance/masa-oracle go 1.22.0 - -toolchain go1.22.2 - require ( github.com/cenkalti/backoff/v4 v4.3.0 github.com/chyeh/pubip v0.0.0-20170203095919-b7e679cf541c @@ -31,7 +28,7 @@ require ( github.com/masa-finance/masa-twitter-scraper v0.0.0-20240515201201-b83fa3597a31 github.com/multiformats/go-multiaddr v0.13.0 github.com/multiformats/go-multihash v0.2.3 - github.com/ollama/ollama v0.3.1 + github.com/ollama/ollama v0.3.6 github.com/rivo/tview v0.0.0-20240505185119-ed116790de0f github.com/sashabaranov/go-openai v1.27.0 github.com/sirupsen/logrus v1.9.3 diff --git a/go.sum b/go.sum index dd030d99..646335e2 100644 --- a/go.sum +++ b/go.sum @@ -532,8 +532,8 @@ github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY= github.com/nxadm/tail v1.4.11/go.mod h1:OTaG3NK980DZzxbRq6lEuzgU+mug70nY11sMd4JXXHc= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= -github.com/ollama/ollama v0.3.1 h1:FvbhD9TxSB1F2xvQPFaGvYKLVxK9QJqfU+EUb3ftwkE= -github.com/ollama/ollama v0.3.1/go.mod h1:USAVO5xFaXAoVWJ0rkPYgCVhTxE/oJ81o7YGcJxvyp8= +github.com/ollama/ollama v0.3.6 h1:nA/N0AmjP327po5cZDGLqI40nl+aeei0pD0dLa92ypE= +github.com/ollama/ollama v0.3.6/go.mod h1:YrWoNkFnPOYsnDvsf/Ztb1wxU9/IXrNsQHqcxbY2r94= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=