Skip to content

Commit

Permalink
feat: telegram API integration, message scraping, sentiment analysis (#…
Browse files Browse the repository at this point in the history 
…456)

* feat: authentication for telegram

* chore: latest

* authentication is working

* chore: telegram fetchChannelMessages

* feat: sentiment analysis for telegram is working

* chore: fix discord docs

* updates to docs

* chore: updates to swagger docs

* stash

* chore: final touches to Telegram PR

* chore: saved file

* chore: update vyper

* chore: fixed merge conflicts with test

* chore: update teelgram messages handler

* chore: fix context issues

* chore: minor fixes

* chore: fixes

* chore: latest with ctx still closing

* new context for each call

* chore: context still failing

* Refactor Telegram integration and handle background connection

Removed the usage of the Run() method which automatically closes connections

Refactor Telegram initialization, updated logging to use logrus. Added context handling in Telegram authentication functions. Implemented TelegramStop function in configuration to manage the background connection shutdown when the system exits.

* chore: pr ready for merge

* chore: remove hardcoded values

---------

Co-authored-by: Bob Stevens <[email protected]>
  • Loading branch information
@nolanjacobson @restevens402
nolanjacobson and restevens402 authored Jul 30, 2024
1 parent dc5c840 commit 6ed7c4b
Show file tree
Hide file tree
Showing 20 changed files with 961 additions and 158 deletions.
9 changes: 8 additions & 1 deletion cmd/masa-node/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,13 @@ func main() {
nodeData.Left()
}
cancel()
// Call the global StopFunc to stop the Telegram background connection
cfg := config.GetInstance()
if cfg.TelegramStop != nil {
if err := cfg.TelegramStop(); err != nil {
logrus.Errorf("Error stopping the background connection: %v", err)
}
}
}()

router := api.SetupRoutes(node)
Expand All @@ -127,7 +134,7 @@ func main() {
multiAddr := node.GetMultiAddrs().String() // Get the multiaddress
ipAddr := node.Host.Addrs()[0].String() // Get the IP address
// Display the welcome message with the multiaddress and IP address
config.DisplayWelcomeMessage(multiAddr, ipAddr, keyManager.EthAddress, isStaked, isValidator, cfg.TwitterScraper, cfg.DiscordScraper, cfg.WebScraper, cfg.Version)
config.DisplayWelcomeMessage(multiAddr, ipAddr, keyManager.EthAddress, isStaked, isValidator, cfg.TwitterScraper, cfg.TelegramScraper, cfg.DiscordScraper, cfg.WebScraper, config.Version)

<-ctx.Done()
}
195 changes: 195 additions & 0 deletions docs/docs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -474,6 +474,53 @@ const docTemplate = `{
]
}
},
"/data/telegram/channel/messages": {
"post": {
"description": "Retrieves messages from a specified Telegram channel.",
"tags": ["Telegram"],
"summary": "Get Telegram Channel Messages",
"parameters": [
{
"in": "body",
"name": "body",
"description": "Request body",
"required": true,
"schema": {
"type": "object",
"properties": {
"username": {
"type": "string",
"description": "Telegram Username"
}
},
"required": ["username"]
}
}
],
"responses": {
"200": {
"description": "Successfully retrieved messages",
"schema": {
"type": "array",
"items": {
"$ref": "#/definitions/Message"
}
}
},
"400": {
"description": "Invalid username or error fetching messages",
"schema": {
"$ref": "#/definitions/ErrorResponse"
}
}
},
"security": [
{
"Bearer": []
}
]
}
},
"/data/web": {
"post": {
"description": "Retrieves data from the web",
Expand Down Expand Up @@ -848,6 +895,45 @@ const docTemplate = `{
}
}
},
"/sentiment/telegram": {
"post": {
"description": "Searches for Telegram messages and analyzes their sentiment",
"tags": ["Sentiment"],
"summary": "Analyze Sentiment of Telegram Messages",
"consumes": ["application/json"],
"produces": ["application/json"],
"parameters": [
{
"name": "query",
"in": "body",
"description": "Search Query",
"required": true,
"schema": {
"type": "object",
"properties": {
"query": {
"type": "string"
}
}
}
}
],
"responses": {
"200": {
"description": "Successfully analyzed sentiment of Telegram messages",
"schema": {
"$ref": "#/definitions/SentimentAnalysisResponse"
}
},
"400": {
"description": "Error analyzing sentiment of Telegram messages",
"schema": {
"$ref": "#/definitions/ErrorResponse"
}
}
}
}
},
"/sentiment/discord": {
"post": {
"description": "Searches for Discord messages and analyzes their sentiment",
Expand Down Expand Up @@ -932,6 +1018,115 @@ const docTemplate = `{
}
}
},
"/auth/telegram/start": {
"post": {
"description": "Initiates the authentication process with Telegram by sending a code to the provided phone number.",
"tags": ["Authentication"],
"summary": "Start Telegram Authentication",
"consumes": ["application/json"],
"produces": ["application/json"],
"parameters": [
{
"name": "phone_number",
"in": "body",
"description": "Phone Number",
"required": true,
"schema": {
"type": "object",
"properties": {
"phone_number": {
"type": "string"
}
}
}
}
],
"responses": {
"200": {
"description": "Successfully sent authentication code",
"schema": {
"type": "object",
"additionalProperties": {
"type": "string"
}
}
},
"400": {
"description": "Invalid request body",
"schema": {
"$ref": "#/definitions/ErrorResponse"
}
},
"500": {
"description": "Failed to initialize Telegram client or to start authentication",
"schema": {
"$ref": "#/definitions/ErrorResponse"
}
}
}
}
},
"/auth/telegram/complete": {
"post": {
"description": "Completes the authentication process with Telegram using the code sent to the phone number.",
"tags": ["Authentication"],
"summary": "Complete Telegram Authentication",
"consumes": ["application/json"],
"produces": ["application/json"],
"parameters": [
{
"name": "phone_number",
"in": "body",
"description": "Phone Number",
"required": true,
"schema": {
"type": "object",
"properties": {
"phone_number": {
"type": "string"
},
"code": {
"type": "string"
},
"phone_code_hash": {
"type": "string"
}
},
"required": ["phone_number", "code", "phone_code_hash"]
}
}
],
"responses": {
"200": {
"description": "Successfully authenticated",
"schema": {
"type": "object",
"additionalProperties": {
"type": "string"
}
}
},
"400": {
"description": "Invalid request body",
"schema": {
"$ref": "#/definitions/ErrorResponse"
}
},
"401": {
"description": "Two-factor authentication is required",
"schema": {
"$ref": "#/definitions/ErrorResponse"
}
},
"500": {
"description": "Failed to initialize Telegram client or to complete authentication",
"schema": {
"$ref": "#/definitions/ErrorResponse"
}
}
}
}
},
"/sentiment/web": {
"post": {
"description": "Searches for web content and analyzes its sentiment",
Expand Down
2 changes: 1 addition & 1 deletion docs/oracle-node/twitter-data.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -468,4 +468,4 @@ Upon collecting the tweets, CryptoSentimentAI processes and analyzes the text co

### Conclusion

The `/data/tweets` endpoint in the Masa Oracle Node API provides a rich foundation for developing decentralized applications that can interact with social media data in real-time. By leveraging this endpoint, developers are empowered to create innovative AI agents capable of analyzing social media trends and sentiments. These agents can uncover deep insights from the vast stream of social media conversations, offering valuable intelligence for a wide range of applications and decision-making processes.
The `/data/tweets` endpoint in the Masa Oracle Node API provides a rich foundation for developing decentralized applications that can interact with social media data in real-time. By leveraging this endpoint, developers are empowered to create innovative AI agents capable of analyzing social media trends and sentiments. These agents can uncover deep insights from the vast stream of social media conversations, offering valuable intelligence for a wide range of applications and decision-making processes.
43 changes: 27 additions & 16 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@ require (
github.com/gocolly/colly/v2 v2.1.0
github.com/golang-jwt/jwt/v4 v4.5.0
github.com/google/uuid v1.6.0
github.com/gotd/contrib v0.20.0
github.com/gotd/td v0.105.0
github.com/ipfs/go-cid v0.4.1
github.com/ipfs/go-datastore v0.6.0
github.com/ipfs/go-ds-leveldb v0.5.0
Expand Down Expand Up @@ -47,9 +49,9 @@ require (
github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 // indirect
github.com/KyleBanks/depth v1.2.1 // indirect
github.com/Microsoft/go-winio v0.6.2 // indirect
github.com/PuerkitoBio/goquery v1.5.1 // indirect
github.com/PuerkitoBio/goquery v1.9.1 // indirect
github.com/Workiva/go-datastructures v1.1.3 // indirect
github.com/andybalholm/cascadia v1.2.0 // indirect
github.com/andybalholm/cascadia v1.3.2 // indirect
github.com/antchfx/htmlquery v1.2.3 // indirect
github.com/antchfx/xmlquery v1.3.1 // indirect
github.com/antchfx/xpath v1.1.10 // indirect
Expand Down Expand Up @@ -88,7 +90,10 @@ require (
github.com/gabriel-vasile/mimetype v1.4.3 // indirect
github.com/gdamore/encoding v1.0.0 // indirect
github.com/gin-contrib/sse v0.1.0 // indirect
github.com/go-logr/logr v1.4.1 // indirect
github.com/go-faster/errors v0.7.1 // indirect
github.com/go-faster/jx v1.1.0 // indirect
github.com/go-faster/xor v1.0.0 // indirect
github.com/go-logr/logr v1.4.2 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-ole/go-ole v1.3.0 // indirect
github.com/go-openapi/jsonpointer v0.21.0 // indirect
Expand All @@ -110,6 +115,8 @@ require (
github.com/google/pprof v0.0.0-20240207164012-fb44976bdcd5 // indirect
github.com/gopherjs/gopherjs v0.0.0-20190812055157-5d271430af9f // indirect
github.com/gorilla/websocket v1.5.1 // indirect
github.com/gotd/ige v0.2.2 // indirect
github.com/gotd/neo v0.1.5 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/hashicorp/golang-lru v1.0.2 // indirect
Expand All @@ -128,7 +135,7 @@ require (
github.com/jpillora/backoff v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/kennygrant/sanitize v1.2.4 // indirect
github.com/klauspost/compress v1.17.8 // indirect
github.com/klauspost/compress v1.17.9 // indirect
github.com/klauspost/cpuid/v2 v2.2.7 // indirect
github.com/koron/go-ssdp v0.0.4 // indirect
github.com/leodido/go-urn v1.4.0 // indirect
Expand Down Expand Up @@ -209,6 +216,7 @@ require (
github.com/sagikazarmark/locafero v0.4.0 // indirect
github.com/sagikazarmark/slog-shim v0.1.0 // indirect
github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect
github.com/segmentio/asm v1.2.0 // indirect
github.com/shirou/gopsutil v3.21.11+incompatible // indirect
github.com/smartystreets/assertions v1.13.0 // indirect
github.com/sourcegraph/conc v0.3.0 // indirect
Expand All @@ -230,33 +238,36 @@ require (
github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 // indirect
github.com/yusufpapurcu/wmi v1.2.4 // indirect
go.opencensus.io v0.24.0 // indirect
go.opentelemetry.io/otel v1.24.0 // indirect
go.opentelemetry.io/otel v1.28.0 // indirect
go.opentelemetry.io/otel/exporters/prometheus v0.44.0 // indirect
go.opentelemetry.io/otel/metric v1.24.0 // indirect
go.opentelemetry.io/otel/sdk v1.24.0 // indirect
go.opentelemetry.io/otel/sdk/metric v1.21.0 // indirect
go.opentelemetry.io/otel/trace v1.24.0 // indirect
go.opentelemetry.io/otel/metric v1.28.0 // indirect
go.opentelemetry.io/otel/sdk v1.28.0 // indirect
go.opentelemetry.io/otel/sdk/metric v1.28.0 // indirect
go.opentelemetry.io/otel/trace v1.28.0 // indirect
go.uber.org/atomic v1.11.0 // indirect
go.uber.org/dig v1.17.1 // indirect
go.uber.org/fx v1.21.1 // indirect
go.uber.org/mock v0.4.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
go.uber.org/zap v1.27.0 // indirect
golang.org/x/arch v0.8.0 // indirect
golang.org/x/crypto v0.23.0 // indirect
golang.org/x/crypto v0.24.0 // indirect
golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 // indirect
golang.org/x/mod v0.17.0 // indirect
golang.org/x/net v0.25.0 // indirect
golang.org/x/mod v0.18.0 // indirect
golang.org/x/net v0.26.0 // indirect
golang.org/x/sync v0.7.0 // indirect
golang.org/x/sys v0.21.0 // indirect
golang.org/x/term v0.20.0 // indirect
golang.org/x/text v0.15.0 // indirect
golang.org/x/tools v0.21.0 // indirect
golang.org/x/sys v0.22.0 // indirect
golang.org/x/term v0.21.0 // indirect
golang.org/x/text v0.16.0 // indirect
golang.org/x/tools v0.22.0 // indirect
gonum.org/v1/gonum v0.15.0 // indirect
google.golang.org/appengine v1.6.8 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240513163218-0867130af1f8 // indirect
google.golang.org/grpc v1.63.2 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
lukechampine.com/blake3 v1.2.1 // indirect
nhooyr.io/websocket v1.8.11 // indirect
rsc.io/qr v0.2.0 // indirect
rsc.io/tmplfunc v0.0.3 // indirect
)
Loading

0 comments on commit 6ed7c4b

Please sign in to comment.