From e3d42ae148602a8bf4546fce2d1925267d22cdd0 Mon Sep 17 00:00:00 2001
From: Nolan Jacobson <50815660+nolanjacobson@users.noreply.github.com>
Date: Fri, 12 Jul 2024 15:01:16 -0400
Subject: [PATCH 01/18] feat: authentication for telegram
---
cmd/telegram-cli/main.go | 168 ++++++++++++++++++++
go.mod | 43 +++--
go.sum | 90 +++++++----
pkg/api/handlers_data.go | 62 ++++++++
pkg/api/routes.go | 27 ++++
pkg/scrapers/telegram/getchannelmessages.go | 151 ++++++++++++++++++
pkg/scrapers/telegram/telegram_client.go | 128 +++++++++++++++
tg_public_keys.pem | 16 ++
8 files changed, 637 insertions(+), 48 deletions(-)
create mode 100644 cmd/telegram-cli/main.go
create mode 100644 pkg/scrapers/telegram/getchannelmessages.go
create mode 100644 pkg/scrapers/telegram/telegram_client.go
create mode 100644 tg_public_keys.pem
diff --git a/cmd/telegram-cli/main.go b/cmd/telegram-cli/main.go
new file mode 100644
index 00000000..dd4af330
--- /dev/null
+++ b/cmd/telegram-cli/main.go
@@ -0,0 +1,168 @@
+package main
+
+import (
+ "bufio"
+ "context"
+ "crypto/rand"
+ "fmt"
+ "os"
+ "path/filepath"
+ "strings"
+
+ "github.com/gotd/td/session"
+ "github.com/gotd/td/telegram"
+ "github.com/gotd/td/telegram/auth"
+ "github.com/gotd/td/tg"
+)
+
+// Fetch messages from a group
+func fetchChannelMessages(ctx context.Context, client *telegram.Client, channelID int64, accessHash int64) error {
+ inputPeer := &tg.InputPeerChannel{ // Use InputPeerChannel instead of InputChannel
+ ChannelID: channelID,
+ AccessHash: accessHash,
+ }
+ result, err := client.API().MessagesGetHistory(ctx, &tg.MessagesGetHistoryRequest{
+ Peer: inputPeer, // Pass inputPeer here
+ Limit: 100, // Adjust the number of messages to fetch
+ })
+ if err != nil {
+ return err
+ }
+
+ // Type assert the result to *tg.MessagesChannelMessages to access Messages field
+ messages, ok := result.(*tg.MessagesChannelMessages)
+ if !ok {
+ return fmt.Errorf("unexpected type %T", result)
+ }
+
+ // Process the messages
+ for _, m := range messages.Messages {
+ message, ok := m.(*tg.Message) // Type assert to *tg.Message
+ if !ok {
+ // Handle the case where the message is not a regular message (e.g., service message)
+ continue
+ }
+ fmt.Printf("Message ID: %d, Content: %s\n", message.ID, message.Message)
+ }
+
+ return nil
+}
+
+func resolveChannelUsername(ctx context.Context, client *telegram.Client, username string) (*tg.InputChannel, error) {
+ resolved, err := client.API().ContactsResolveUsername(ctx, username)
+ if err != nil {
+ return nil, err
+ }
+
+ channel := &tg.InputChannel{
+ ChannelID: resolved.Chats[0].GetID(),
+ AccessHash: resolved.Chats[0].(*tg.Channel).AccessHash,
+ }
+
+ fmt.Printf("Channel ID: %d, Access Hash: %d\n", channel.ChannelID, channel.AccessHash)
+ return channel, nil
+}
+
+func main() {
+ // Define your Telegram app credentials
+ appID := 28423325 // Your actual app ID
+ appHash := "c60c0a268973ea3f7d52e16e4ab2a0d3" // Your actual app hash
+
+ // Define the path to the directory where session data will be stored
+ sessionDir := filepath.Join(os.Getenv("HOME"), ".telegram-sessions")
+
+ // Create the session directory if it doesn't already exist
+ if err := os.MkdirAll(sessionDir, 0700); err != nil {
+ fmt.Fprintf(os.Stderr, "Failed to create session directory: %v\n", err)
+ os.Exit(1)
+ }
+
+ // Create a session storage
+ storage := &session.FileStorage{
+ Path: filepath.Join(sessionDir, "session.json"),
+ }
+
+ // Create a random seed for the client
+ seed := make([]byte, 32)
+ if _, err := rand.Read(seed); err != nil {
+ fmt.Fprintf(os.Stderr, "Failed to generate random seed: %v\n", err)
+ os.Exit(1)
+ }
+
+ client := telegram.NewClient(appID, appHash, telegram.Options{
+ SessionStorage: storage,
+ })
+
+ // Continue with the rest of your application logic
+ ctx := context.Background()
+
+ // Check if the session file already exists and has content
+ sessionExists := false
+ if info, err := os.Stat(filepath.Join(sessionDir, "session.json")); err == nil && info.Size() > 0 {
+ sessionExists = true
+ }
+
+ // Define the phone number and password (if 2FA is enabled)
+ phone := "+13053398321"
+ password := "" // Leave empty if 2FA is not enabled
+
+ // Define the code prompt function
+ codePrompt := func(ctx context.Context, sentCode *tg.AuthSentCode) (string, error) {
+ fmt.Print("Enter code: ")
+ code, err := bufio.NewReader(os.Stdin).ReadString('\n')
+ if err != nil {
+ return "", err
+ }
+ return strings.TrimSpace(code), nil
+ }
+
+ // Set up and perform the authentication flow only if no valid session exists
+ if !sessionExists {
+ authFlow := auth.NewFlow(
+ auth.Constant(phone, password, auth.CodeAuthenticatorFunc(codePrompt)),
+ auth.SendCodeOptions{},
+ )
+ if err := client.Run(ctx, func(ctx context.Context) error {
+ return authFlow.Run(ctx, client.Auth())
+ }); err != nil {
+ fmt.Fprintf(os.Stderr, "Authentication failed: %v\n", err)
+ os.Exit(1)
+ }
+ } else {
+ // If a session exists, simply start the client
+ if err := client.Run(ctx, func(ctx context.Context) error {
+ username := "coinlistofficialchannel" // Replace with the actual username of the channel
+ channel, err := resolveChannelUsername(ctx, client, username)
+ if err != nil {
+ fmt.Fprintf(os.Stderr, "Failed to resolve channel username: %v\n", err)
+ os.Exit(1)
+ }
+ if err := fetchChannelMessages(ctx, client, channel.ChannelID, channel.AccessHash); err != nil {
+ fmt.Fprintf(os.Stderr, "Failed to fetch channel messages: %v\n", err)
+ os.Exit(1)
+ }
+ return nil // No operation, just start the client
+ }); err != nil {
+ fmt.Fprintf(os.Stderr, "Failed to start client with existing session: %v\n", err)
+ os.Exit(1)
+ }
+ }
+
+ // The user is now authenticated, or a valid session is loaded
+ // You can use the client to interact with the Telegram API
+
+ // username := "coinlistofficialchannel" // Replace with the actual username of the channel
+ // channel, err := resolveChannelUsername(ctx, client, username)
+ // if err != nil {
+ // fmt.Fprintf(os.Stderr, "Failed to resolve channel username: %v\n", err)
+ // os.Exit(1)
+ // }
+
+ // // return channel
+
+ // if err := fetchChannelMessages(ctx, client, channel.ChannelID, channel.AccessHash); err != nil {
+ // fmt.Fprintf(os.Stderr, "Failed to fetch channel messages: %v\n", err)
+ // os.Exit(1)
+ // }
+
+}
diff --git a/go.mod b/go.mod
index b1a651ae..7fab7439 100644
--- a/go.mod
+++ b/go.mod
@@ -18,6 +18,7 @@ require (
github.com/gocolly/colly/v2 v2.1.0
github.com/golang-jwt/jwt/v4 v4.5.0
github.com/google/uuid v1.6.0
+ github.com/gotd/td v0.105.0
github.com/ipfs/go-cid v0.4.1
github.com/ipfs/go-datastore v0.6.0
github.com/ipfs/go-ds-leveldb v0.5.0
@@ -36,6 +37,7 @@ require (
github.com/spf13/pflag v1.0.5
github.com/spf13/viper v1.19.0
github.com/stretchr/testify v1.9.0
+ github.com/swaggo/files v1.0.1
github.com/swaggo/gin-swagger v1.6.0
github.com/swaggo/swag v1.16.3
google.golang.org/protobuf v1.34.2
@@ -45,9 +47,9 @@ require (
github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 // indirect
github.com/KyleBanks/depth v1.2.1 // indirect
github.com/Microsoft/go-winio v0.6.2 // indirect
- github.com/PuerkitoBio/goquery v1.5.1 // indirect
+ github.com/PuerkitoBio/goquery v1.9.1 // indirect
github.com/Workiva/go-datastructures v1.1.3 // indirect
- github.com/andybalholm/cascadia v1.2.0 // indirect
+ github.com/andybalholm/cascadia v1.3.2 // indirect
github.com/antchfx/htmlquery v1.2.3 // indirect
github.com/antchfx/xmlquery v1.3.1 // indirect
github.com/antchfx/xpath v1.1.10 // indirect
@@ -84,7 +86,10 @@ require (
github.com/gabriel-vasile/mimetype v1.4.3 // indirect
github.com/gdamore/encoding v1.0.0 // indirect
github.com/gin-contrib/sse v0.1.0 // indirect
- github.com/go-logr/logr v1.4.1 // indirect
+ github.com/go-faster/errors v0.7.1 // indirect
+ github.com/go-faster/jx v1.1.0 // indirect
+ github.com/go-faster/xor v1.0.0 // indirect
+ github.com/go-logr/logr v1.4.2 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-ole/go-ole v1.3.0 // indirect
github.com/go-openapi/jsonpointer v0.21.0 // indirect
@@ -106,6 +111,8 @@ require (
github.com/google/pprof v0.0.0-20240207164012-fb44976bdcd5 // indirect
github.com/gopherjs/gopherjs v0.0.0-20190812055157-5d271430af9f // indirect
github.com/gorilla/websocket v1.5.1 // indirect
+ github.com/gotd/ige v0.2.2 // indirect
+ github.com/gotd/neo v0.1.5 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/hashicorp/golang-lru v1.0.2 // indirect
@@ -124,7 +131,7 @@ require (
github.com/jpillora/backoff v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/kennygrant/sanitize v1.2.4 // indirect
- github.com/klauspost/compress v1.17.8 // indirect
+ github.com/klauspost/compress v1.17.9 // indirect
github.com/klauspost/cpuid/v2 v2.2.7 // indirect
github.com/koron/go-ssdp v0.0.4 // indirect
github.com/leodido/go-urn v1.4.0 // indirect
@@ -204,6 +211,7 @@ require (
github.com/sagikazarmark/locafero v0.4.0 // indirect
github.com/sagikazarmark/slog-shim v0.1.0 // indirect
github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect
+ github.com/segmentio/asm v1.2.0 // indirect
github.com/shirou/gopsutil v3.21.11+incompatible // indirect
github.com/smartystreets/assertions v1.13.0 // indirect
github.com/sourcegraph/conc v0.3.0 // indirect
@@ -224,27 +232,28 @@ require (
github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 // indirect
github.com/yusufpapurcu/wmi v1.2.4 // indirect
go.opencensus.io v0.24.0 // indirect
- go.opentelemetry.io/otel v1.24.0 // indirect
+ go.opentelemetry.io/otel v1.28.0 // indirect
go.opentelemetry.io/otel/exporters/prometheus v0.44.0 // indirect
- go.opentelemetry.io/otel/metric v1.24.0 // indirect
- go.opentelemetry.io/otel/sdk v1.24.0 // indirect
- go.opentelemetry.io/otel/sdk/metric v1.21.0 // indirect
- go.opentelemetry.io/otel/trace v1.24.0 // indirect
+ go.opentelemetry.io/otel/metric v1.28.0 // indirect
+ go.opentelemetry.io/otel/sdk v1.28.0 // indirect
+ go.opentelemetry.io/otel/sdk/metric v1.28.0 // indirect
+ go.opentelemetry.io/otel/trace v1.28.0 // indirect
+ go.uber.org/atomic v1.11.0 // indirect
go.uber.org/dig v1.17.1 // indirect
go.uber.org/fx v1.21.1 // indirect
go.uber.org/mock v0.4.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
go.uber.org/zap v1.27.0 // indirect
golang.org/x/arch v0.8.0 // indirect
- golang.org/x/crypto v0.23.0 // indirect
+ golang.org/x/crypto v0.24.0 // indirect
golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 // indirect
- golang.org/x/mod v0.17.0 // indirect
- golang.org/x/net v0.25.0 // indirect
+ golang.org/x/mod v0.18.0 // indirect
+ golang.org/x/net v0.26.0 // indirect
golang.org/x/sync v0.7.0 // indirect
- golang.org/x/sys v0.21.0 // indirect
- golang.org/x/term v0.20.0 // indirect
- golang.org/x/text v0.15.0 // indirect
- golang.org/x/tools v0.21.0 // indirect
+ golang.org/x/sys v0.22.0 // indirect
+ golang.org/x/term v0.21.0 // indirect
+ golang.org/x/text v0.16.0 // indirect
+ golang.org/x/tools v0.22.0 // indirect
gonum.org/v1/gonum v0.15.0 // indirect
google.golang.org/appengine v1.6.8 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240513163218-0867130af1f8 // indirect
@@ -252,5 +261,7 @@ require (
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
lukechampine.com/blake3 v1.2.1 // indirect
+ nhooyr.io/websocket v1.8.11 // indirect
+ rsc.io/qr v0.2.0 // indirect
rsc.io/tmplfunc v0.0.3 // indirect
)
diff --git a/go.sum b/go.sum
index 9c8f39e1..8c6f2b94 100644
--- a/go.sum
+++ b/go.sum
@@ -18,15 +18,17 @@ github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERo
github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
-github.com/PuerkitoBio/goquery v1.5.1 h1:PSPBGne8NIUWw+/7vFBV+kG2J/5MOjbzc7154OaKCSE=
github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc=
+github.com/PuerkitoBio/goquery v1.9.1 h1:mTL6XjbJTZdpfL+Gwl5U2h1l9yEkJjhmlTeV9VPW7UI=
+github.com/PuerkitoBio/goquery v1.9.1/go.mod h1:cW1n6TmIMDoORQU5IU/P1T3tGFunOeXEpGP2WHRwkbY=
github.com/VictoriaMetrics/fastcache v1.12.2 h1:N0y9ASrJ0F6h0QaC3o6uJb3NIZ9VKLjCM7NQbSmF7WI=
github.com/VictoriaMetrics/fastcache v1.12.2/go.mod h1:AmC+Nzz1+3G2eCPapF6UcsnkThDcMsQicp4xDukwJYI=
github.com/Workiva/go-datastructures v1.1.3 h1:LRdRrug9tEuKk7TGfz/sct5gjVj44G9pfqDt4qm7ghw=
github.com/Workiva/go-datastructures v1.1.3/go.mod h1:1yZL+zfsztete+ePzZz/Zb1/t5BnDuE2Ya2MMGhzP6A=
github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
-github.com/andybalholm/cascadia v1.2.0 h1:vuRCkM5Ozh/BfmsaTm26kbjm0mIOM3yS5Ek/F5h18aE=
github.com/andybalholm/cascadia v1.2.0/go.mod h1:YCyR8vOZT9aZ1CHEd8ap0gMVm2aFgxBp0T0eFw1RUQY=
+github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss=
+github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU=
github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
github.com/antchfx/htmlquery v1.2.3 h1:sP3NFDneHx2stfNXCKbhHFo8XgNjCACnU/4AO5gWz6M=
github.com/antchfx/htmlquery v1.2.3/go.mod h1:B0ABL+F5irhhMWg54ymEZinzMSi0Kt3I2if0BLYa3V0=
@@ -189,9 +191,16 @@ github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU=
github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y=
github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
+github.com/go-faster/errors v0.7.1 h1:MkJTnDoEdi9pDabt1dpWf7AA8/BaSYZqibYyhZ20AYg=
+github.com/go-faster/errors v0.7.1/go.mod h1:5ySTjWFiphBs07IKuiL69nxdfd5+fzh1u7FPGZP2quo=
+github.com/go-faster/jx v1.1.0 h1:ZsW3wD+snOdmTDy9eIVgQdjUpXRRV4rqW8NS3t+20bg=
+github.com/go-faster/jx v1.1.0/go.mod h1:vKDNikrKoyUmpzaJ0OkIkRQClNHFX/nF3dnTJZb3skg=
+github.com/go-faster/xor v0.3.0/go.mod h1:x5CaDY9UKErKzqfRfFZdfu+OSTfoZny3w5Ak7UxcipQ=
+github.com/go-faster/xor v1.0.0 h1:2o8vTOgErSGHP3/7XwA5ib1FTtUsNtwCoLLBjl31X38=
+github.com/go-faster/xor v1.0.0/go.mod h1:x5CaDY9UKErKzqfRfFZdfu+OSTfoZny3w5Ak7UxcipQ=
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
-github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ=
-github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
+github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
@@ -299,6 +308,12 @@ github.com/gopherjs/gopherjs v0.0.0-20190812055157-5d271430af9f h1:KMlcu9X58lhTA
github.com/gopherjs/gopherjs v0.0.0-20190812055157-5d271430af9f/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY=
github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY=
+github.com/gotd/ige v0.2.2 h1:XQ9dJZwBfDnOGSTxKXBGP4gMud3Qku2ekScRjDWWfEk=
+github.com/gotd/ige v0.2.2/go.mod h1:tuCRb+Y5Y3eNTo3ypIfNpQ4MFjrnONiL2jN2AKZXmb0=
+github.com/gotd/neo v0.1.5 h1:oj0iQfMbGClP8xI59x7fE/uHoTJD7NZH9oV1WNuPukQ=
+github.com/gotd/neo v0.1.5/go.mod h1:9A2a4bn9zL6FADufBdt7tZt+WMhvZoc5gWXihOPoiBQ=
+github.com/gotd/td v0.105.0 h1:FjU9pgmL5Qt10+cosPCz4agvQT/hMBz6QMi1fFH7ekY=
+github.com/gotd/td v0.105.0/go.mod h1:aVe5/LP/nNIyAqaW3CwB0Ckum+MkcfvazwMOLHV0bqQ=
github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw=
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
@@ -371,8 +386,8 @@ github.com/kennygrant/sanitize v1.2.4/go.mod h1:LGsjYYtgxbetdg5owWB2mpgUL6e2nfw2
github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU=
-github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
+github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
+github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM=
github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
@@ -630,8 +645,8 @@ github.com/rivo/uniseg v0.4.3/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUc
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
-github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M=
-github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA=
+github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
+github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik=
github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo=
@@ -647,6 +662,8 @@ github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca h1:NugYot0LIVPxT
github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU=
github.com/sashabaranov/go-openai v1.26.2 h1:cVlQa3gn3eYqNXRW03pPlpy6zLG52EU4g0FrWXc0EFI=
github.com/sashabaranov/go-openai v1.26.2/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg=
+github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys=
+github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs=
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI=
github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
@@ -727,6 +744,7 @@ github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSW
github.com/supranational/blst v0.3.11 h1:LyU6FolezeWAhvQk0k6O/d49jqgO52MSDDfYgbeoEm4=
github.com/supranational/blst v0.3.11/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw=
github.com/swaggo/files v1.0.1 h1:J1bVJ4XHZNq0I46UU90611i9/YzdrF7x92oX1ig5IdE=
+github.com/swaggo/files v1.0.1/go.mod h1:0qXmMNH6sXNf+73t65aKeB+ApmgxdnkQzVTAj2uaMUg=
github.com/swaggo/gin-swagger v1.6.0 h1:y8sxvQ3E20/RCyrXeFfg60r6H0Z+SwpTjMYsMm+zy8M=
github.com/swaggo/gin-swagger v1.6.0/go.mod h1:BG00cCEy294xtVpyIAHG6+e2Qzj/xKlRdOqDkvq0uzo=
github.com/swaggo/swag v1.16.3 h1:PnCYjPCah8FK4I26l2F/KQ4yz3sILcVUN3cTlBFA9Pg=
@@ -774,20 +792,22 @@ github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQ
go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA=
go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
-go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo=
-go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo=
+go.opentelemetry.io/otel v1.28.0 h1:/SqNcYk+idO0CxKEUOtKQClMK/MimZihKYMruSMViUo=
+go.opentelemetry.io/otel v1.28.0/go.mod h1:q68ijF8Fc8CnMHKyzqL6akLO46ePnjkgfIMIjUIX9z4=
go.opentelemetry.io/otel/exporters/prometheus v0.44.0 h1:08qeJgaPC0YEBu2PQMbqU3rogTlyzpjhCI2b58Yn00w=
go.opentelemetry.io/otel/exporters/prometheus v0.44.0/go.mod h1:ERL2uIeBtg4TxZdojHUwzZfIFlUIjZtxubT5p4h1Gjg=
-go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI=
-go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco=
-go.opentelemetry.io/otel/sdk v1.24.0 h1:YMPPDNymmQN3ZgczicBY3B6sf9n62Dlj9pWD3ucgoDw=
-go.opentelemetry.io/otel/sdk v1.24.0/go.mod h1:KVrIYw6tEubO9E96HQpcmpTKDVn9gdv35HoYiQWGDFg=
-go.opentelemetry.io/otel/sdk/metric v1.21.0 h1:smhI5oD714d6jHE6Tie36fPx4WDFIg+Y6RfAY4ICcR0=
-go.opentelemetry.io/otel/sdk/metric v1.21.0/go.mod h1:FJ8RAsoPGv/wYMgBdUJXOm+6pzFY3YdljnXtv1SBE8Q=
-go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI=
-go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU=
+go.opentelemetry.io/otel/metric v1.28.0 h1:f0HGvSl1KRAU1DLgLGFjrwVyismPlnuU6JD6bOeuA5Q=
+go.opentelemetry.io/otel/metric v1.28.0/go.mod h1:Fb1eVBFZmLVTMb6PPohq3TO9IIhUisDsbJoL/+uQW4s=
+go.opentelemetry.io/otel/sdk v1.28.0 h1:b9d7hIry8yZsgtbmM0DKyPWMMUMlK9NEKuIG4aBqWyE=
+go.opentelemetry.io/otel/sdk v1.28.0/go.mod h1:oYj7ClPUA7Iw3m+r7GeEjz0qckQRJK2B8zjcZEfu7Pg=
+go.opentelemetry.io/otel/sdk/metric v1.28.0 h1:OkuaKgKrgAbYrrY0t92c+cC+2F6hsFNnCQArXCKlg08=
+go.opentelemetry.io/otel/sdk/metric v1.28.0/go.mod h1:cWPjykihLAPvXKi4iZc1dpER3Jdq2Z0YLse3moQUCpg=
+go.opentelemetry.io/otel/trace v1.28.0 h1:GhQ9cUuQGmNDd5BTCP2dAvv75RdMxEfTmYejp+lkx9g=
+go.opentelemetry.io/otel/trace v1.28.0/go.mod h1:jPyXzNPg6da9+38HEwElrQiHlVMTnVfM3/yv2OlIHaI=
go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
+go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
go.uber.org/dig v1.17.1 h1:Tga8Lz8PcYNsWsyHMZ1Vm0OQOUaJNDyvPImgbAu9YSc=
go.uber.org/dig v1.17.1/go.mod h1:Us0rSJiThwCv2GteUN0Q7OKvU7n5J4dxZ9JKUXozFdE=
go.uber.org/fx v1.21.1 h1:RqBh3cYdzZS0uqwVeEjOX2p73dddLpym315myy/Bpb0=
@@ -830,8 +850,8 @@ golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf
golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg=
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
-golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI=
-golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
+golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI=
+golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM=
golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc=
@@ -848,8 +868,8 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
-golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA=
-golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0=
+golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -877,6 +897,7 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b
golang.org/x/net v0.0.0-20210423184538-5f58ad60dda6/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
golang.org/x/net v0.13.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA=
@@ -885,8 +906,8 @@ golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY=
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
-golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac=
-golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
+golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ=
+golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -899,6 +920,7 @@ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -948,8 +970,8 @@ golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws=
-golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
+golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
@@ -961,8 +983,8 @@ golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY=
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58=
-golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw=
-golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
+golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA=
+golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
@@ -976,8 +998,8 @@ golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
-golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk=
-golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
+golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
@@ -1003,8 +1025,8 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f
golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
-golang.org/x/tools v0.21.0 h1:qc0xYgIbsSDt9EyWz05J5wfa7LOVW0YTLOXrqdLAWIw=
-golang.org/x/tools v0.21.0/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
+golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA=
+golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -1087,8 +1109,12 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
lukechampine.com/blake3 v1.2.1 h1:YuqqRuaqsGV71BV/nm9xlI0MKUv4QC54jQnBChWbGnI=
lukechampine.com/blake3 v1.2.1/go.mod h1:0OFRp7fBtAylGVCO40o87sbupkyIGgbpv1+M1k1LM6k=
+nhooyr.io/websocket v1.8.11 h1:f/qXNc2/3DpoSZkHt1DQu6rj4zGC8JmkkLkWss0MgN0=
+nhooyr.io/websocket v1.8.11/go.mod h1:rN9OFWIUwuxg4fR5tELlYC04bXYowCP9GX47ivo2l+c=
nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50=
rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
+rsc.io/qr v0.2.0 h1:6vBLea5/NRMVTz8V66gipeLycZMl/+UlFmk8DvqQ6WY=
+rsc.io/qr v0.2.0/go.mod h1:IF+uZjkb9fqyeF/4tlBoynqmQxUoPfWEKh921coOuXs=
rsc.io/tmplfunc v0.0.3 h1:53XFQh69AfOa8Tw0Jm7t+GV7KZhOi6jzsCzTtKbMvzU=
rsc.io/tmplfunc v0.0.3/go.mod h1:AG3sTPzElb1Io3Yg4voV9AGZJuleGAwaVRxL9M49PhA=
sourcegraph.com/sourcegraph/go-diff v0.5.0/go.mod h1:kuch7UrkMzY0X+p9CRK03kfuPQ2zzQcaEFbx8wA8rck=
diff --git a/pkg/api/handlers_data.go b/pkg/api/handlers_data.go
index d3e02d20..5ed38579 100644
--- a/pkg/api/handlers_data.go
+++ b/pkg/api/handlers_data.go
@@ -10,6 +10,7 @@ import (
"os"
"sync"
+ "github.com/masa-finance/masa-oracle/pkg/scrapers/telegram"
"github.com/masa-finance/masa-oracle/pkg/workers"
"github.com/multiformats/go-multiaddr"
"github.com/sirupsen/logrus"
@@ -714,6 +715,67 @@ func (api *API) WebData() gin.HandlerFunc {
}
}
+// StartAuth starts the authentication process with Telegram.
+func (api *API) StartAuth() gin.HandlerFunc {
+ return func(c *gin.Context) {
+ var reqBody struct {
+ PhoneNumber string `json:"phone_number"`
+ }
+ if err := c.ShouldBindJSON(&reqBody); err != nil {
+ c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid request body"})
+ return
+ }
+
+ client, err := telegram.InitializeClient()
+ if err != nil {
+ c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to initialize Telegram client"})
+ return
+ }
+
+ phoneCodeHash, err := telegram.StartAuthentication(c.Request.Context(), client, reqBody.PhoneNumber)
+ if err != nil {
+ c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to start authentication"})
+ return
+ }
+
+ c.JSON(http.StatusOK, gin.H{"message": "Code sent to Telegram app", "phone_code_hash": phoneCodeHash})
+ }
+}
+
+// CompleteAuth completes the authentication process with Telegram.
+func (api *API) CompleteAuth() gin.HandlerFunc {
+ return func(c *gin.Context) {
+ var reqBody struct {
+ PhoneNumber string `json:"phone_number"`
+ Code string `json:"code"`
+ PhoneCodeHash string `json:"phone_code_hash"`
+ }
+ if err := c.ShouldBindJSON(&reqBody); err != nil {
+ c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid request body"})
+ return
+ }
+
+ client, err := telegram.InitializeClient()
+ if err != nil {
+ c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to initialize Telegram client"})
+ return
+ }
+
+ auth, err := telegram.CompleteAuthentication(c.Request.Context(), client, reqBody.PhoneNumber, reqBody.Code, reqBody.PhoneCodeHash)
+ if err != nil {
+ // Check if 2FA is required
+ if err.Error() == "2FA required" {
+ c.JSON(http.StatusUnauthorized, gin.H{"error": "Two-factor authentication is required"})
+ return
+ }
+ c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to complete authentication"})
+ return
+ }
+
+ c.JSON(http.StatusOK, gin.H{"message": "Authentication successful", "auth": auth})
+ }
+}
+
// LocalLlmChat handles requests for chatting with AI models hosted by ollama.
// It expects a JSON request body with a structure formatted for the model. For example for Ollama:
//
diff --git a/pkg/api/routes.go b/pkg/api/routes.go
index 13b7b772..70254334 100644
--- a/pkg/api/routes.go
+++ b/pkg/api/routes.go
@@ -217,6 +217,33 @@ func SetupRoutes(node *masa.OracleNode) *gin.Engine {
// @Router /discord/profile/{userID} [get]
v1.GET("/data/discord/profile/:userID", API.SearchDiscordProfile())
+ // @Summary Start Telegram Authentication
+ // @Description Initiates the authentication process with Telegram by sending a code to the provided phone number.
+ // @Tags Authentication
+ // @Accept json
+ // @Produce json
+ // @Param phone_number body string true "Phone Number"
+ // @Success 200 {object} map[string]interface{} "Successfully sent authentication code"
+ // @Failure 400 {object} ErrorResponse "Invalid request body"
+ // @Failure 500 {object} ErrorResponse "Failed to initialize Telegram client or to start authentication"
+ // @Router /auth/telegram/start [post]
+ v1.POST("/auth/telegram/start", API.StartAuth())
+
+ // @Summary Complete Telegram Authentication
+ // @Description Completes the authentication process with Telegram using the code sent to the phone number.
+ // @Tags Authentication
+ // @Accept json
+ // @Produce json
+ // @Param phone_number body string true "Phone Number"
+ // @Param code body string true "Authentication Code"
+ // @Param phone_code_hash body string true "Phone Code Hash"
+ // @Success 200 {object} map[string]interface{} "Successfully authenticated"
+ // @Failure 400 {object} ErrorResponse "Invalid request body"
+ // @Failure 401 {object} ErrorResponse "Two-factor authentication is required"
+ // @Failure 500 {object} ErrorResponse "Failed to initialize Telegram client or to complete authentication"
+ // @Router /auth/telegram/complete [post]
+ v1.POST("/auth/telegram/complete", API.CompleteAuth())
+
// oauth tests
// v1.GET("/data/discord/exchangetoken/:code", API.ExchangeDiscordTokenHandler())
diff --git a/pkg/scrapers/telegram/getchannelmessages.go b/pkg/scrapers/telegram/getchannelmessages.go
new file mode 100644
index 00000000..496252f8
--- /dev/null
+++ b/pkg/scrapers/telegram/getchannelmessages.go
@@ -0,0 +1,151 @@
+package telegram
+
+import (
+ "bufio"
+ "context"
+ "crypto/rand"
+ "fmt"
+ "os"
+ "path/filepath"
+ "strings"
+
+ "github.com/gotd/td/session"
+ "github.com/gotd/td/telegram"
+ "github.com/gotd/td/telegram/auth"
+ "github.com/gotd/td/tg"
+)
+
+// Fetch messages from a group
+func fetchChannelMessages(ctx context.Context, client *telegram.Client, channelID int64, accessHash int64) error {
+ inputPeer := &tg.InputPeerChannel{ // Use InputPeerChannel instead of InputChannel
+ ChannelID: channelID,
+ AccessHash: accessHash,
+ }
+ result, err := client.API().MessagesGetHistory(ctx, &tg.MessagesGetHistoryRequest{
+ Peer: inputPeer, // Pass inputPeer here
+ Limit: 100, // Adjust the number of messages to fetch
+ })
+ if err != nil {
+ return err
+ }
+
+ // Type assert the result to *tg.MessagesChannelMessages to access Messages field
+ messages, ok := result.(*tg.MessagesChannelMessages)
+ if !ok {
+ return fmt.Errorf("unexpected type %T", result)
+ }
+
+ // Process the messages
+ for _, m := range messages.Messages {
+ message, ok := m.(*tg.Message) // Type assert to *tg.Message
+ if !ok {
+ // Handle the case where the message is not a regular message (e.g., service message)
+ continue
+ }
+ fmt.Printf("Message ID: %d, Content: %s\n", message.ID, message.Message)
+ }
+
+ return nil
+}
+
+func resolveChannelUsername(ctx context.Context, client *telegram.Client, username string) (*tg.InputChannel, error) {
+ resolved, err := client.API().ContactsResolveUsername(ctx, username)
+ if err != nil {
+ return nil, err
+ }
+
+ channel := &tg.InputChannel{
+ ChannelID: resolved.Chats[0].GetID(),
+ AccessHash: resolved.Chats[0].(*tg.Channel).AccessHash,
+ }
+
+ fmt.Printf("Channel ID: %d, Access Hash: %d\n", channel.ChannelID, channel.AccessHash)
+ return channel, nil
+}
+
+func main() {
+ // Define your Telegram app credentials
+ appID := 28423325 // Your actual app ID
+ appHash := "c60c0a268973ea3f7d52e16e4ab2a0d3" // Your actual app hash
+
+ // Define the path to the directory where session data will be stored
+ sessionDir := filepath.Join(os.Getenv("HOME"), ".telegram-sessions")
+
+ // Create the session directory if it doesn't already exist
+ if err := os.MkdirAll(sessionDir, 0700); err != nil {
+ fmt.Fprintf(os.Stderr, "Failed to create session directory: %v\n", err)
+ os.Exit(1)
+ }
+
+ // Create a session storage
+ storage := &session.FileStorage{
+ Path: filepath.Join(sessionDir, "session.json"),
+ }
+
+ // Create a random seed for the client
+ seed := make([]byte, 32)
+ if _, err := rand.Read(seed); err != nil {
+ fmt.Fprintf(os.Stderr, "Failed to generate random seed: %v\n", err)
+ os.Exit(1)
+ }
+
+ client := telegram.NewClient(appID, appHash, telegram.Options{
+ SessionStorage: storage,
+ })
+
+ // Continue with the rest of your application logic
+ ctx := context.Background()
+
+ // Check if the session file already exists and has content
+ sessionExists := false
+ if info, err := os.Stat(filepath.Join(sessionDir, "session.json")); err == nil && info.Size() > 0 {
+ sessionExists = true
+ }
+
+ // Define the phone number and password (if 2FA is enabled)
+ phone := "+13053398321"
+ password := "" // Leave empty if 2FA is not enabled
+
+ // Define the code prompt function
+ codePrompt := func(ctx context.Context, sentCode *tg.AuthSentCode) (string, error) {
+ fmt.Print("Enter code: ")
+ code, err := bufio.NewReader(os.Stdin).ReadString('\n')
+ if err != nil {
+ return "", err
+ }
+ return strings.TrimSpace(code), nil
+ }
+
+ // Set up and perform the authentication flow only if no valid session exists
+ if !sessionExists {
+ authFlow := auth.NewFlow(
+ auth.Constant(phone, password, auth.CodeAuthenticatorFunc(codePrompt)),
+ auth.SendCodeOptions{},
+ )
+ if err := client.Run(ctx, func(ctx context.Context) error {
+ return authFlow.Run(ctx, client.Auth())
+ }); err != nil {
+ fmt.Fprintf(os.Stderr, "Authentication failed: %v\n", err)
+ os.Exit(1)
+ }
+ } else {
+ // If a session exists, simply start the client
+ if err := client.Run(ctx, func(ctx context.Context) error {
+ username := "coinlistofficialchannel" // Replace with the actual username of the channel
+ channel, err := resolveChannelUsername(ctx, client, username)
+ if err != nil {
+ fmt.Fprintf(os.Stderr, "Failed to resolve channel username: %v\n", err)
+ os.Exit(1)
+ }
+ if err := fetchChannelMessages(ctx, client, channel.ChannelID, channel.AccessHash); err != nil {
+ fmt.Fprintf(os.Stderr, "Failed to fetch channel messages: %v\n", err)
+ os.Exit(1)
+ }
+ return nil // No operation, just start the client
+ }); err != nil {
+ fmt.Fprintf(os.Stderr, "Failed to start client with existing session: %v\n", err)
+ os.Exit(1)
+ }
+ }
+
+}
diff --git a/pkg/scrapers/telegram/telegram_client.go b/pkg/scrapers/telegram/telegram_client.go
new file mode 100644
index 00000000..f0c061da
--- /dev/null
+++ b/pkg/scrapers/telegram/telegram_client.go
@@ -0,0 +1,128 @@
+package telegram
+
+import (
+ "context"
+ "crypto/rand"
+ "errors"
+ "os"
+ "path/filepath"
+ "sync"
+
+ "github.com/gotd/td/session"
+ "github.com/gotd/td/telegram"
+ "github.com/gotd/td/telegram/auth"
+ "github.com/gotd/td/tg"
+)
+
+var (
+ client *telegram.Client
+ once sync.Once
+ appID = 28423325 // Your actual app ID
+ appHash = "c60c0a268973ea3f7d52e16e4ab2a0d3" // Your actual app hash
+ sessionDir = filepath.Join(os.Getenv("HOME"), ".telegram-sessions")
+)
+
+func InitializeClient() (*telegram.Client, error) {
+ var err error
+ once.Do(func() {
+ // Ensure the session directory exists
+ if err = os.MkdirAll(sessionDir, 0700); err != nil {
+ return
+ }
+
+ // Create a session storage
+ storage := &session.FileStorage{
+ Path: filepath.Join(sessionDir, "session.json"),
+ }
+
+ // Create a random seed for the client
+ seed := make([]byte, 32)
+ if _, err = rand.Read(seed); err != nil {
+ return
+ }
+
+ // Initialize the Telegram client
+ client = telegram.NewClient(appID, appHash, telegram.Options{
+ SessionStorage: storage,
+ })
+ })
+
+ return client, err
+}
+
+// func AuthenticateUser(ctx context.Context, phoneNumber string) error {
+// // Check if the session file already exists and has content
+// sessionExists := false
+// if info, err := os.Stat(filepath.Join(sessionDir, "session.json")); err == nil && info.Size() > 0 {
+// sessionExists = true
+// }
+
+// if !sessionExists {
+// // Define the code prompt function
+// codePrompt := func(ctx context.Context, sentCode *tg.AuthSentCode) (string, error) {
+// fmt.Print("Enter code: ")
+// code, err := bufio.NewReader(os.Stdin).ReadString('\n')
+// if err != nil {
+// return "", err
+// }
+// return strings.TrimSpace(code), nil
+// }
+
+// // Set up and perform the authentication flow
+// authFlow := auth.NewFlow(
+// auth.Constant(phoneNumber, "", auth.CodeAuthenticatorFunc(codePrompt)), // Empty password for simplicity
+// auth.SendCodeOptions{},
+// )
+// if err := client.Run(ctx, func(ctx context.Context) error {
+// return authFlow.Run(ctx, client.Auth())
+// }); err != nil {
+// return fmt.Errorf("authentication failed: %v", err)
+// }
+// }
+
+// return nil
+// }
+
+// StartAuthentication sends the phone number to Telegram and requests a code.
+func StartAuthentication(ctx context.Context, client *telegram.Client, phoneNumber string) (string, error) {
+ // Call the SendCode method of the client to send the code to the user's Telegram app
+ sentCode, err := client.Auth().SendCode(ctx, phoneNumber, auth.SendCodeOptions{})
+ if err != nil {
+ return "", err
+ }
+
+ // Extract the phoneCodeHash from the sentCode object
+ var phoneCodeHash string
+ switch code := sentCode.(type) {
+ case *tg.AuthSentCode:
+ phoneCodeHash = code.PhoneCodeHash
+ default:
+ return "", errors.New("unexpected type of AuthSentCode")
+ }
+
+ // Return the phoneCodeHash to be used in the next step
+ return phoneCodeHash, nil
+}
+
+// CompleteAuthentication uses the provided code to authenticate with Telegram.
+func CompleteAuthentication(ctx context.Context, client *telegram.Client, phoneNumber, code, phoneCodeHash string) (*tg.AuthAuthorization, error) {
+ // Use the provided code and phoneCodeHash to authenticate
+ auth, err := client.Auth().SignIn(ctx, phoneNumber, code, phoneCodeHash)
+ if err != nil {
+ // Handle the specific error if password authentication is needed
+ if err == errors.New("2FA required") {
+ // Here you would handle the second factor authentication (2FA)
+ // This usually involves prompting the user for their password.
+ }
+ return nil, err
+ }
+
+ // At this point, authentication was successful, and you have the user's Telegram auth data.
+ // You can now create a session for the user or perform other post-authentication tasks.
+
+ return auth, nil
+}
+
+func GetClient() *telegram.Client {
+ return client
+}
diff --git a/tg_public_keys.pem b/tg_public_keys.pem
new file mode 100644
index 00000000..1de207e8
--- /dev/null
+++ b/tg_public_keys.pem
@@ -0,0 +1,16 @@
+-----BEGIN RSA PUBLIC KEY-----
+MIIBCgKCAQEAyMEdY1aR+sCR3ZSJrtztKTKqigvO/vBfqACJLZtS7QMgCGXJ6XIR
+yy7mx66W0/sOFa7/1mAZtEoIokDP3ShoqF4fVNb6XeqgQfaUHd8wJpDWHcR2OFwv
+plUUI1PLTktZ9uW2WE23b+ixNwJjJGwBDJPQEQFBE+vfmH0JP503wr5INS1poWg/
+j25sIWeYPHYeOrFp/eXaqhISP6G+q2IeTaWTXpwZj4LzXq5YOpk4bYEQ6mvRq7D1
+aHWfYmlEGepfaYR8Q0YqvvhYtMte3ITnuSJs171+GDqpdKcSwHnd6FudwGO4pcCO
+j4WcDuXc2CTHgH8gFTNhp/Y8/SpDOhvn9QIDAQAB
+-----END RSA PUBLIC KEY-----
+-----BEGIN RSA PUBLIC KEY-----
+MIIBCgKCAQEA6LszBcC1LGzyr992NzE0ieY+BSaOW622Aa9Bd4ZHLl+TuFQ4lo4g
+5nKaMBwK/BIb9xUfg0Q29/2mgIR6Zr9krM7HjuIcCzFvDtr+L0GQjae9H0pRB2OO
+62cECs5HKhT5DZ98K33vmWiLowc621dQuwKWSQKjWf50XYFw42h21P2KXUGyp2y/
++aEyZ+uVgLLQbRA1dEjSDZ2iGRy12Mk5gpYc397aYp438fsJoHIgJ2lgMv5h7WY9
+t6N/byY9Nw9p21Og3AoXSL2q/2IJ1WRUhebgAdGVMlV1fkuOQoEzR7EdpqtQD9Cs
+5+bfo3Nhmcyvk5ftB0WkJ9z6bNZ7yxrP8wIDAQAB
+-----END RSA PUBLIC KEY-----
\ No newline at end of file
From 571de59891a23898db7e1d2a4eb99b996dbbf843 Mon Sep 17 00:00:00 2001
From: Nolan Jacobson <50815660+nolanjacobson@users.noreply.github.com>
Date: Fri, 12 Jul 2024 16:48:50 -0400
Subject: [PATCH 02/18] chore: latest
---
pkg/api/handlers_data.go | 20 ++----
pkg/scrapers/telegram/telegram_client.go | 86 ++++++++++++++++++------
2 files changed, 74 insertions(+), 32 deletions(-)
diff --git a/pkg/api/handlers_data.go b/pkg/api/handlers_data.go
index 5ed38579..9b1976f2 100644
--- a/pkg/api/handlers_data.go
+++ b/pkg/api/handlers_data.go
@@ -726,13 +726,13 @@ func (api *API) StartAuth() gin.HandlerFunc {
return
}
- client, err := telegram.InitializeClient()
- if err != nil {
- c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to initialize Telegram client"})
- return
- }
+ // client, err := telegram.InitializeClient()
+ // if err != nil {
+ // c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to initialize Telegram client"})
+ // return
+ // }
- phoneCodeHash, err := telegram.StartAuthentication(c.Request.Context(), client, reqBody.PhoneNumber)
+ phoneCodeHash, err := telegram.StartAuthentication(c.Request.Context(), reqBody.PhoneNumber)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to start authentication"})
return
@@ -755,13 +755,7 @@ func (api *API) CompleteAuth() gin.HandlerFunc {
return
}
- client, err := telegram.InitializeClient()
- if err != nil {
- c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to initialize Telegram client"})
- return
- }
-
- auth, err := telegram.CompleteAuthentication(c.Request.Context(), client, reqBody.PhoneNumber, reqBody.Code, reqBody.PhoneCodeHash)
+ auth, err := telegram.CompleteAuthentication(c.Request.Context(), reqBody.PhoneNumber, reqBody.Code, reqBody.PhoneCodeHash)
if err != nil {
// Check if 2FA is required
if err.Error() == "2FA required" {
diff --git a/pkg/scrapers/telegram/telegram_client.go b/pkg/scrapers/telegram/telegram_client.go
index f0c061da..ee6c5f36 100644
--- a/pkg/scrapers/telegram/telegram_client.go
+++ b/pkg/scrapers/telegram/telegram_client.go
@@ -4,6 +4,7 @@ import (
"context"
"crypto/rand"
"errors"
+ "log"
"os"
"path/filepath"
"sync"
@@ -84,43 +85,90 @@ func InitializeClient() (*telegram.Client, error) {
// }
// StartAuthentication sends the phone number to Telegram and requests a code.
-func StartAuthentication(ctx context.Context, client *telegram.Client, phoneNumber string) (string, error) {
- // Call the SendCode method of the client to send the code to the user's Telegram app
- sentCode, err := client.Auth().SendCode(ctx, phoneNumber, auth.SendCodeOptions{})
+func StartAuthentication(ctx context.Context, phoneNumber string) (string, error) {
+ // Initialize the Telegram client (if not already initialized)
+ client, err := InitializeClient()
if err != nil {
+ log.Printf("Failed to initialize Telegram client: %v", err)
return "", err
}
- // Extract the phoneCodeHash from the sentCode object
+ // Define a variable to hold the phoneCodeHash
var phoneCodeHash string
- switch code := sentCode.(type) {
- case *tg.AuthSentCode:
- phoneCodeHash = code.PhoneCodeHash
- default:
- return "", errors.New("unexpected type of AuthSentCode")
+
+ // Use client.Run to start the client and execute the SendCode method
+ err = client.Run(ctx, func(ctx context.Context) error {
+ // Call the SendCode method of the client to send the code to the user's Telegram app
+ sentCode, err := client.Auth().SendCode(ctx, phoneNumber, auth.SendCodeOptions{
+ AllowFlashCall: true,
+ CurrentNumber: true,
+ })
+ if err != nil {
+ log.Printf("Error sending code: %v", err)
+ return err
+ }
+
+ log.Printf("Code sent successfully to: %s", phoneNumber)
+
+ // Extract the phoneCodeHash from the sentCode object
+ switch code := sentCode.(type) {
+ case *tg.AuthSentCode:
+ phoneCodeHash = code.PhoneCodeHash
+ default:
+ return errors.New("unexpected type of AuthSentCode")
+ }
+
+ return nil
+ })
+
+ if err != nil {
+ log.Printf("Failed to run client or send code: %v", err)
+ return "", err
}
// Return the phoneCodeHash to be used in the next step
+ log.Printf("Authentication process started successfully for: %s", phoneNumber)
return phoneCodeHash, nil
}
// CompleteAuthentication uses the provided code to authenticate with Telegram.
-func CompleteAuthentication(ctx context.Context, client *telegram.Client, phoneNumber, code, phoneCodeHash string) (*tg.AuthAuthorization, error) {
- // Use the provided code and phoneCodeHash to authenticate
- auth, err := client.Auth().SignIn(ctx, phoneNumber, code, phoneCodeHash)
+func CompleteAuthentication(ctx context.Context, phoneNumber, code, phoneCodeHash string) (*tg.AuthAuthorization, error) {
+ // Initialize the Telegram client (if not already initialized)
+ client, err := InitializeClient()
if err != nil {
- // Handle the specific error if password authentication is needed
- if err == errors.New("2FA required") {
- // Here you would handle the second factor authentication (2FA)
- // This usually involves prompting the user for their password.
+ log.Printf("Failed to initialize Telegram client: %v", err)
+ return nil, err
+ }
+
+ // Define a variable to hold the authentication result
+ var authResult *tg.AuthAuthorization
+
+ // Use client.Run to start the client and execute the SignIn method
+ err = client.Run(ctx, func(ctx context.Context) error {
+ // Use the provided code and phoneCodeHash to authenticate
+ auth, err := client.Auth().SignIn(ctx, phoneNumber, code, phoneCodeHash)
+ if err != nil {
+ // Handle the specific error if password authentication is needed
+ if err == errors.New("2FA required") {
+ // Here you would handle the second factor authentication (2FA)
+ // This usually involves prompting the user for their password.
+ }
+ return err
}
+
+ // At this point, authentication was successful, and you have the user's Telegram auth data.
+ authResult = auth
+ return nil
+ })
+
+ if err != nil {
+ log.Printf("Failed to run client or sign in: %v", err)
return nil, err
}
- // At this point, authentication was successful, and you have the user's Telegram auth data.
// You can now create a session for the user or perform other post-authentication tasks.
-
- return auth, nil
+ log.Printf("Authentication successful for: %s", phoneNumber)
+ return authResult, nil
}
func GetClient() *telegram.Client {
From 75c7f8fd4f1d7b4c17a820c5f4aaaa492df4715f Mon Sep 17 00:00:00 2001
From: Nolan Jacobson <50815660+nolanjacobson@users.noreply.github.com>
Date: Fri, 12 Jul 2024 17:09:25 -0400
Subject: [PATCH 03/18] authentication is working
---
pkg/scrapers/telegram/telegram_client.go | 6 +-----
1 file changed, 1 insertion(+), 5 deletions(-)
diff --git a/pkg/scrapers/telegram/telegram_client.go b/pkg/scrapers/telegram/telegram_client.go
index ee6c5f36..573bd248 100644
--- a/pkg/scrapers/telegram/telegram_client.go
+++ b/pkg/scrapers/telegram/telegram_client.go
@@ -148,11 +148,7 @@ func CompleteAuthentication(ctx context.Context, phoneNumber, code, phoneCodeHas
// Use the provided code and phoneCodeHash to authenticate
auth, err := client.Auth().SignIn(ctx, phoneNumber, code, phoneCodeHash)
if err != nil {
- // Handle the specific error if password authentication is needed
- if err == errors.New("2FA required") {
- // Here you would handle the second factor authentication (2FA)
- // This usually involves prompting the user for their password.
- }
+ log.Printf("Error during SignIn: %v", err)
return err
}
From 818ce1f9be06b46b910dadfefda01056ba6d033d Mon Sep 17 00:00:00 2001
From: Nolan Jacobson <50815660+nolanjacobson@users.noreply.github.com>
Date: Tue, 16 Jul 2024 10:28:28 -0400
Subject: [PATCH 04/18] chore: telegram fetchChannelMessages
---
pkg/api/handlers_data.go | 18 ++
pkg/scrapers/telegram/getchannelmessages.go | 174 ++++++--------------
pkg/scrapers/telegram/telegram_client.go | 33 ----
3 files changed, 71 insertions(+), 154 deletions(-)
diff --git a/pkg/api/handlers_data.go b/pkg/api/handlers_data.go
index 9b1976f2..c18b7120 100644
--- a/pkg/api/handlers_data.go
+++ b/pkg/api/handlers_data.go
@@ -770,6 +770,24 @@ func (api *API) CompleteAuth() gin.HandlerFunc {
}
}
+func GetChannelMessagesHandler() gin.HandlerFunc {
+ return func(c *gin.Context) {
+ username := c.Param("username") // Assuming "username" is a URL parameter
+ if username == "" {
+ c.JSON(http.StatusBadRequest, gin.H{"error": "Username must be provided"})
+ return
+ }
+
+ messages, err := telegram.fetchChannelMessages(c.Request.Context(), username)
+ if err != nil {
+ c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+ return
+ }
+
+ c.JSON(http.StatusOK, gin.H{"messages": messages})
+ }
+}
+
// LocalLlmChat handles requests for chatting with AI models hosted by ollama.
// It expects a JSON request body with a structure formatted for the model. For example for Ollama:
//
diff --git a/pkg/scrapers/telegram/getchannelmessages.go b/pkg/scrapers/telegram/getchannelmessages.go
index 496252f8..436c1fec 100644
--- a/pkg/scrapers/telegram/getchannelmessages.go
+++ b/pkg/scrapers/telegram/getchannelmessages.go
@@ -1,151 +1,83 @@
package telegram
import (
- "bufio"
"context"
- "crypto/rand"
"fmt"
- "os"
- "path/filepath"
- "strings"
+ "log"
- "github.com/gotd/td/session"
- "github.com/gotd/td/telegram"
- "github.com/gotd/td/telegram/auth"
"github.com/gotd/td/tg"
)
// Fetch messages from a group
-func fetchChannelMessages(ctx context.Context, client *telegram.Client, channelID int64, accessHash int64) error {
- inputPeer := &tg.InputPeerChannel{ // Use InputPeerChannel instead of InputChannel
- ChannelID: channelID,
- AccessHash: accessHash,
- }
- result, err := client.API().MessagesGetHistory(ctx, &tg.MessagesGetHistoryRequest{
- Peer: inputPeer, // Pass inputPeer here
- Limit: 100, // Adjust the number of messages to fetch
- })
- if err != nil {
- return err
- }
-
- // Type assert the result to *tg.MessagesChannelMessages to access Messages field
- messages, ok := result.(*tg.MessagesChannelMessages)
- if !ok {
- return fmt.Errorf("unexpected type %T", result)
- }
-
- // Process the messages
- for _, m := range messages.Messages {
- message, ok := m.(*tg.Message) // Type assert to *tg.Message
- if !ok {
- // Handle the case where the message is not a regular message (e.g., service message)
- continue
- }
- fmt.Printf("Message ID: %d, Content: %s\n", message.ID, message.Message)
- }
-
- return nil
-}
-
-func resolveChannelUsername(ctx context.Context, client *telegram.Client, username string) (*tg.InputChannel, error) {
- resolved, err := client.API().ContactsResolveUsername(ctx, username)
+func fetchChannelMessages(ctx context.Context, username string) ([]*tg.Message, error) {
+ client, err := InitializeClient()
if err != nil {
+ log.Printf("Failed to initialize Telegram client: %v", err)
return nil, err
}
- channel := &tg.InputChannel{
- ChannelID: resolved.Chats[0].GetID(),
- AccessHash: resolved.Chats[0].(*tg.Channel).AccessHash,
+ channel, err := resolveChannelUsername(ctx, username) // Edit: Assign the second value to err
+ if err != nil {
+ return nil, err // Handle the error if resolveChannelUsername fails
}
+ var messagesSlice []*tg.Message // Define a slice to hold the messages
- fmt.Printf("Channel ID: %d, Access Hash: %d\n", channel.ChannelID, channel.AccessHash)
- return channel, nil
-}
-
-func main() {
- // Define your Telegram app credentials
- appID := 28423325 // Your actual app ID
- appHash := "c60c0a268973ea3f7d52e16e4ab2a0d3" // Your actual app hash
-
- // Define the path to the directory where session data will be stored
- sessionDir := filepath.Join(os.Getenv("HOME"), ".telegram-sessions")
-
- // Create the session directory if it doesn't already exist
- if err := os.MkdirAll(sessionDir, 0700); err != nil {
- fmt.Fprintf(os.Stderr, "Failed to create session directory: %v\n", err)
- os.Exit(1)
- }
+ err = client.Run(ctx, func(ctx context.Context) error {
+ inputPeer := &tg.InputPeerChannel{ // Use InputPeerChannel instead of InputChannel
+ ChannelID: channel.ChannelID,
+ AccessHash: channel.AccessHash,
+ }
+ result, err := client.API().MessagesGetHistory(ctx, &tg.MessagesGetHistoryRequest{
+ Peer: inputPeer, // Pass inputPeer here
+ Limit: 100, // Adjust the number of messages to fetch
+ })
+ if err != nil {
+ return err
+ }
- // Create a session storage
- storage := &session.FileStorage{
- Path: filepath.Join(sessionDir, "session.json"),
- }
+ // Type assert the result to *tg.MessagesChannelMessages to access Messages field
+ messages, ok := result.(*tg.MessagesChannelMessages)
+ if !ok {
+ return fmt.Errorf("unexpected type %T", result)
+ }
- // Create a random seed for the client
- seed := make([]byte, 32)
- if _, err := rand.Read(seed); err != nil {
- fmt.Fprintf(os.Stderr, "Failed to generate random seed: %v\n", err)
- os.Exit(1)
- }
+ // Process the messages
+ for _, m := range messages.Messages {
+ message, ok := m.(*tg.Message) // Type assert to *tg.Message
+ if !ok {
+ // Handle the case where the message is not a regular message (e.g., service message)
+ continue
+ }
+ messagesSlice = append(messagesSlice, message) // Append the message to the slice
+ }
- client := telegram.NewClient(appID, appHash, telegram.Options{
- SessionStorage: storage,
+ return nil
})
- // Continue with the rest of your application logic
- ctx := context.Background()
+ return messagesSlice, err // Return the slice of messages and any error
+}
- // Check if the session file already exists and has content
- sessionExists := false
- if info, err := os.Stat(filepath.Join(sessionDir, "session.json")); err == nil && info.Size() > 0 {
- sessionExists = true
+func resolveChannelUsername(ctx context.Context, username string) (*tg.InputChannel, error) {
+ client, err := InitializeClient()
+ if err != nil {
+ log.Printf("Failed to initialize Telegram client: %v", err)
+ return nil, err
}
- // Define the phone number and password (if 2FA is enabled)
- phone := "+13053398321"
- password := "" // Leave empty if 2FA is not enabled
-
- // Define the code prompt function
- codePrompt := func(ctx context.Context, sentCode *tg.AuthSentCode) (string, error) {
- fmt.Print("Enter code: ")
- code, err := bufio.NewReader(os.Stdin).ReadString('\n')
+ var channel *tg.InputChannel
+ err = client.Run(ctx, func(ctx context.Context) error {
+ resolved, err := client.API().ContactsResolveUsername(ctx, username)
if err != nil {
- return "", err
+ return err
}
- return strings.TrimSpace(code), nil
- }
- // Set up and perform the authentication flow only if no valid session exists
- if !sessionExists {
- authFlow := auth.NewFlow(
- auth.Constant(phone, password, auth.CodeAuthenticatorFunc(codePrompt)),
- auth.SendCodeOptions{},
- )
- if err := client.Run(ctx, func(ctx context.Context) error {
- return authFlow.Run(ctx, client.Auth())
- }); err != nil {
- fmt.Fprintf(os.Stderr, "Authentication failed: %v\n", err)
- os.Exit(1)
+ channel = &tg.InputChannel{
+ ChannelID: resolved.Chats[0].GetID(),
+ AccessHash: resolved.Chats[0].(*tg.Channel).AccessHash,
}
- } else {
- // If a session exists, simply start the client
- if err := client.Run(ctx, func(ctx context.Context) error {
- username := "coinlistofficialchannel" // Replace with the actual username of the channel
- channel, err := resolveChannelUsername(ctx, client, username)
- if err != nil {
- fmt.Fprintf(os.Stderr, "Failed to resolve channel username: %v\n", err)
- os.Exit(1)
- }
- if err := fetchChannelMessages(ctx, client, channel.ChannelID, channel.AccessHash); err != nil {
- fmt.Fprintf(os.Stderr, "Failed to fetch channel messages: %v\n", err)
- os.Exit(1)
- }
- return nil // No operation, just start the client
- }); err != nil {
- fmt.Fprintf(os.Stderr, "Failed to start client with existing session: %v\n", err)
- os.Exit(1)
- }
- }
+ fmt.Printf("Channel ID: %d, Access Hash: %d\n", channel.ChannelID, channel.AccessHash)
+ return nil
+ })
+ return channel, err
}
diff --git a/pkg/scrapers/telegram/telegram_client.go b/pkg/scrapers/telegram/telegram_client.go
index 573bd248..8f60a1e8 100644
--- a/pkg/scrapers/telegram/telegram_client.go
+++ b/pkg/scrapers/telegram/telegram_client.go
@@ -51,39 +51,6 @@ func InitializeClient() (*telegram.Client, error) {
return client, err
}
-// func AuthenticateUser(ctx context.Context, phoneNumber string) error {
-// // Check if the session file already exists and has content
-// sessionExists := false
-// if info, err := os.Stat(filepath.Join(sessionDir, "session.json")); err == nil && info.Size() > 0 {
-// sessionExists = true
-// }
-
-// if !sessionExists {
-// // Define the code prompt function
-// codePrompt := func(ctx context.Context, sentCode *tg.AuthSentCode) (string, error) {
-// fmt.Print("Enter code: ")
-// code, err := bufio.NewReader(os.Stdin).ReadString('\n')
-// if err != nil {
-// return "", err
-// }
-// return strings.TrimSpace(code), nil
-// }
-
-// // Set up and perform the authentication flow
-// authFlow := auth.NewFlow(
-// auth.Constant(phoneNumber, "", auth.CodeAuthenticatorFunc(codePrompt)), // Empty password for simplicity
-// auth.SendCodeOptions{},
-// )
-// if err := client.Run(ctx, func(ctx context.Context) error {
-// return authFlow.Run(ctx, client.Auth())
-// }); err != nil {
-// return fmt.Errorf("authentication failed: %v", err)
-// }
-// }
-
-// return nil
-// }
-
// StartAuthentication sends the phone number to Telegram and requests a code.
func StartAuthentication(ctx context.Context, phoneNumber string) (string, error) {
// Initialize the Telegram client (if not already initialized)
From 132b09ac366f99ea163ee802ae85eef8a88e99fc Mon Sep 17 00:00:00 2001
From: Nolan Jacobson <50815660+nolanjacobson@users.noreply.github.com>
Date: Tue, 16 Jul 2024 13:42:55 -0400
Subject: [PATCH 05/18] feat: sentiment analysis for telegram is working
---
pkg/api/handlers_data.go | 62 ++++++++++++++-
pkg/api/routes.go | 23 ++++++
pkg/llmbridge/sentiment.go | 85 +++++++++++++++++++++
pkg/scrapers/telegram/getchannelmessages.go | 74 ++++++++++--------
pkg/scrapers/telegram/telegram_client.go | 16 ++--
pkg/workers/handler.go | 8 ++
pkg/workers/workers.go | 4 +-
7 files changed, 231 insertions(+), 41 deletions(-)
diff --git a/pkg/api/handlers_data.go b/pkg/api/handlers_data.go
index c18b7120..9772c51f 100644
--- a/pkg/api/handlers_data.go
+++ b/pkg/api/handlers_data.go
@@ -6,6 +6,7 @@ import (
"encoding/json"
"fmt"
"io"
+ "log"
"net/http"
"os"
"sync"
@@ -247,6 +248,56 @@ func (api *API) SearchDiscordMessagesAndAnalyzeSentiment() gin.HandlerFunc {
}
}
+// SearchTelegramMessagesAndAnalyzeSentiment processes a request to search Telegram messages and analyze sentiment.
+func (api *API) SearchTelegramMessagesAndAnalyzeSentiment() gin.HandlerFunc {
+ return func(c *gin.Context) {
+ if !api.Node.IsStaked {
+ c.JSON(http.StatusBadRequest, gin.H{"error": "Node has not staked and cannot participate"})
+ return
+ }
+
+ var reqBody struct {
+ Username string `json:"username"` // Telegram usernames are used instead of channel IDs
+ Prompt string `json:"prompt"`
+ Model string `json:"model"`
+ }
+ if err := c.ShouldBindJSON(&reqBody); err != nil {
+ c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid request body"})
+ return
+ }
+
+ if reqBody.Username == "" {
+ c.JSON(http.StatusBadRequest, gin.H{"error": "Username parameter is missing"})
+ return
+ }
+
+ if reqBody.Prompt == "" {
+ c.JSON(http.StatusBadRequest, gin.H{"error": "Prompt parameter is missing"})
+ return
+ }
+
+ if reqBody.Model == "" {
+ c.JSON(http.StatusBadRequest, gin.H{"error": "Model parameter is missing"})
+ return
+ }
+
+ bodyBytes, wErr := json.Marshal(reqBody)
+ if wErr != nil {
+ c.JSON(http.StatusBadRequest, gin.H{"error": wErr.Error()})
+ return
+ }
+ requestID := uuid.New().String()
+ responseCh := pubsub2.GetResponseChannelMap().CreateChannel(requestID)
+ defer pubsub2.GetResponseChannelMap().Delete(requestID)
+ wErr = publishWorkRequest(api, requestID, workers.WORKER.TelegramSentiment, bodyBytes)
+ if wErr != nil {
+ c.JSON(http.StatusBadRequest, gin.H{"error": wErr.Error()})
+ return
+ }
+ handleWorkResponse(c, responseCh)
+ }
+}
+
// SearchWebAndAnalyzeSentiment returns a gin.HandlerFunc that processes web search requests and performs sentiment analysis.
// It first validates the request body for required fields such as URL, Depth, and Model. If the Model is set to "all",
// it iterates through all available models to perform sentiment analysis on the web content fetched from the specified URL.
@@ -762,7 +813,7 @@ func (api *API) CompleteAuth() gin.HandlerFunc {
c.JSON(http.StatusUnauthorized, gin.H{"error": "Two-factor authentication is required"})
return
}
- c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to complete authentication"})
+ c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to complete authentication", "details": err.Error()})
return
}
@@ -770,19 +821,22 @@ func (api *API) CompleteAuth() gin.HandlerFunc {
}
}
-func GetChannelMessagesHandler() gin.HandlerFunc {
+func (api *API) GetChannelMessagesHandler() gin.HandlerFunc {
return func(c *gin.Context) {
- username := c.Param("username") // Assuming "username" is a URL parameter
+ username := c.Param("username")
if username == "" {
c.JSON(http.StatusBadRequest, gin.H{"error": "Username must be provided"})
return
}
- messages, err := telegram.fetchChannelMessages(c.Request.Context(), username)
+ log.Printf("Starting to fetch messages for username: %s", username)
+ messages, err := telegram.FetchChannelMessages(c.Request.Context(), username)
if err != nil {
+ log.Printf("Error fetching messages for username: %s, error: %v", username, err)
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
+ log.Printf("Finished fetching messages for username: %s", username)
c.JSON(http.StatusOK, gin.H{"messages": messages})
}
diff --git a/pkg/api/routes.go b/pkg/api/routes.go
index 70254334..120e764c 100644
--- a/pkg/api/routes.go
+++ b/pkg/api/routes.go
@@ -244,6 +244,18 @@ func SetupRoutes(node *masa.OracleNode) *gin.Engine {
// @Router /auth/telegram/complete [post]
v1.POST("/auth/telegram/complete", API.CompleteAuth())
+ // @Summary Get Telegram Channel Messages
+ // @Description Retrieves messages from a specified Telegram channel.
+ // @Tags Telegram
+ // @Accept json
+ // @Produce json
+ // @Param username path string true "Telegram Username"
+ // @Success 200 {object} map[string][]Message "Successfully retrieved messages"
+ // @Failure 400 {object} ErrorResponse "Username must be provided"
+ // @Failure 500 {object} ErrorResponse "Failed to fetch channel messages"
+ // @Router /telegram/channel/{username}/messages [get]
+ v1.GET("/telegram/channel/:username/messages", API.GetChannelMessagesHandler())
+
// oauth tests
// v1.GET("/data/discord/exchangetoken/:code", API.ExchangeDiscordTokenHandler())
@@ -406,6 +418,17 @@ func SetupRoutes(node *masa.OracleNode) *gin.Engine {
// @Router /sentiment/tweets [post]
v1.POST("/sentiment/discord", API.SearchDiscordMessagesAndAnalyzeSentiment())
+ // @Summary Analyze Sentiment of Telegram Messages
+ // @Description Searches for Telegram messages and analyzes their sentiment
+ // @Tags Sentiment
+ // @Accept json
+ // @Produce json
+ // @Param query body string true "Search Query"
+ // @Success 200 {object} SentimentAnalysisResponse "Successfully analyzed sentiment of Telegram messages"
+ // @Failure 400 {object} ErrorResponse "Error analyzing sentiment of Telegram messages"
+ // @Router /sentiment/telegram [post]
+ v1.POST("/sentiment/telegram", API.SearchTelegramMessagesAndAnalyzeSentiment())
+
// @Summary Analyze Sentiment of Web Content
// @Description Searches for web content and analyzes its sentiment
// @Tags Sentiment
diff --git a/pkg/llmbridge/sentiment.go b/pkg/llmbridge/sentiment.go
index 985fe109..9eb277a2 100644
--- a/pkg/llmbridge/sentiment.go
+++ b/pkg/llmbridge/sentiment.go
@@ -9,6 +9,7 @@ import (
"net/http"
"strings"
+ "github.com/gotd/td/tg"
"github.com/masa-finance/masa-oracle/pkg/config"
twitterscraper "github.com/masa-finance/masa-twitter-scraper"
"github.com/ollama/ollama/api"
@@ -306,3 +307,87 @@ func AnalyzeSentimentDiscord(messages []string, model string, prompt string) (st
return messagesContent, SanitizeResponse(sentimentSummary), nil
}
}
+
+// AnalyzeSentimentTelegram analyzes the sentiment of the provided Telegram messages by sending them to the sentiment analysis API.
+func AnalyzeSentimentTelegram(messages []*tg.Message, model string, prompt string) (string, string, error) {
+ // Concatenate messages with a newline character
+ var messageTexts []string
+ for _, msg := range messages {
+ if msg != nil {
+ messageTexts = append(messageTexts, msg.Message)
+ }
+ }
+ messagesContent := strings.Join(messageTexts, "\n")
+
+ // The rest of the code follows the same pattern as AnalyzeSentimentDiscord
+ if strings.Contains(model, "claude-") {
+ client := NewClaudeClient() // Adjusted to call without arguments
+ payloadBytes, err := CreatePayload(messagesContent, model, prompt)
+ if err != nil {
+ logrus.Errorf("Error creating payload: %v", err)
+ return "", "", err
+ }
+ resp, err := client.SendRequest(payloadBytes)
+ if err != nil {
+ logrus.Errorf("Error sending request to Claude API: %v", err)
+ return "", "", err
+ }
+ defer resp.Body.Close()
+ sentimentSummary, err := ParseResponse(resp)
+ if err != nil {
+ logrus.Errorf("Error parsing response from Claude: %v", err)
+ return "", "", err
+ }
+ return messagesContent, sentimentSummary, nil
+
+ } else {
+ stream := false
+
+ genReq := api.ChatRequest{
+ Model: model,
+ Messages: []api.Message{
+ {Role: "user", Content: messagesContent},
+ {Role: "assistant", Content: prompt},
+ },
+ Stream: &stream,
+ Options: map[string]interface{}{
+ "temperature": 0.0,
+ "seed": 42,
+ "num_ctx": 4096,
+ },
+ }
+
+ requestJSON, err := json.Marshal(genReq)
+ if err != nil {
+ logrus.Errorf("Error marshaling request JSON: %v", err)
+ return "", "", err
+ }
+ uri := config.GetInstance().LLMChatUrl
+ if uri == "" {
+ errMsg := "ollama api url not set"
+ logrus.Errorf(errMsg)
+ return "", "", errors.New(errMsg)
+ }
+ resp, err := http.Post(uri, "application/json", bytes.NewReader(requestJSON))
+ if err != nil {
+ logrus.Errorf("Error sending request to API: %v", err)
+ return "", "", err
+ }
+ defer resp.Body.Close()
+ body, err := io.ReadAll(resp.Body)
+ if err != nil {
+ logrus.Errorf("Error reading response body: %v", err)
+ return "", "", err
+ }
+
+ var payload api.ChatResponse
+ err = json.Unmarshal(body, &payload)
+ if err != nil {
+ logrus.Errorf("Error unmarshaling response JSON: %v", err)
+ return "", "", err
+ }
+
+ sentimentSummary := payload.Message.Content
+ return messagesContent, SanitizeResponse(sentimentSummary), nil
+ }
+}
diff --git a/pkg/scrapers/telegram/getchannelmessages.go b/pkg/scrapers/telegram/getchannelmessages.go
index 436c1fec..bcc21ea8 100644
--- a/pkg/scrapers/telegram/getchannelmessages.go
+++ b/pkg/scrapers/telegram/getchannelmessages.go
@@ -4,25 +4,44 @@ import (
"context"
"fmt"
"log"
+ "time"
"github.com/gotd/td/tg"
+ "github.com/masa-finance/masa-oracle/pkg/llmbridge"
)
// Fetch messages from a group
-func fetchChannelMessages(ctx context.Context, username string) ([]*tg.Message, error) {
- client, err := InitializeClient()
- if err != nil {
- log.Printf("Failed to initialize Telegram client: %v", err)
- return nil, err
- }
+func FetchChannelMessages(ctx context.Context, username string) ([]*tg.Message, error) {
+ // Initialize the Telegram client (if not already initialized)
+ client = GetClient()
- channel, err := resolveChannelUsername(ctx, username) // Edit: Assign the second value to err
- if err != nil {
- return nil, err // Handle the error if resolveChannelUsername fails
+ if client == nil {
+ var err error
+ client, err = InitializeClient()
+ if err != nil {
+ log.Printf("Failed to initialize Telegram client: %v", err)
+ return nil, err
+ }
}
+
var messagesSlice []*tg.Message // Define a slice to hold the messages
- err = client.Run(ctx, func(ctx context.Context) error {
+ // ctx, cancel := context.WithTimeout(context.Background(), time.Minute*5)
+ // defer cancel() // Make sure to cancel when you are done to free resources.
+
+ err := client.Run(ctx, func(ctx context.Context) error {
+ resolved, err := client.API().ContactsResolveUsername(ctx, username)
+ if err != nil {
+ return err
+ }
+
+ channel := &tg.InputChannel{
+ ChannelID: resolved.Chats[0].GetID(),
+ AccessHash: resolved.Chats[0].(*tg.Channel).AccessHash,
+ }
+
+ fmt.Printf("Channel ID: %d, Access Hash: %d\n", channel.ChannelID, channel.AccessHash)
+
inputPeer := &tg.InputPeerChannel{ // Use InputPeerChannel instead of InputChannel
ChannelID: channel.ChannelID,
AccessHash: channel.AccessHash,
@@ -57,27 +76,22 @@ func fetchChannelMessages(ctx context.Context, username string) ([]*tg.Message,
return messagesSlice, err // Return the slice of messages and any error
}
-func resolveChannelUsername(ctx context.Context, username string) (*tg.InputChannel, error) {
- client, err := InitializeClient()
+// ScrapeTelegramMessagesForSentiment scrapes messages from a Telegram channel and analyzes their sentiment.
+func ScrapeTelegramMessagesForSentiment(ctx context.Context, username string, model string, prompt string) (string, string, error) {
+ // Create a context with a timeout if needed
+ ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
+ defer cancel()
+ // Fetch messages from the Telegram channel
+ messages, err := FetchChannelMessages(ctx, username)
if err != nil {
- log.Printf("Failed to initialize Telegram client: %v", err)
- return nil, err
+ return "", "", fmt.Errorf("error fetching messages from Telegram channel: %v", err)
}
- var channel *tg.InputChannel
- err = client.Run(ctx, func(ctx context.Context) error {
- resolved, err := client.API().ContactsResolveUsername(ctx, username)
- if err != nil {
- return err
- }
-
- channel = &tg.InputChannel{
- ChannelID: resolved.Chats[0].GetID(),
- AccessHash: resolved.Chats[0].(*tg.Channel).AccessHash,
- }
-
- fmt.Printf("Channel ID: %d, Access Hash: %d\n", channel.ChannelID, channel.AccessHash)
- return nil
- })
- return channel, err
+ // Analyze the sentiment of the fetched messages
+ // Note: Ensure that llmbridge.AnalyzeSentimentTelegram is implemented and can handle the analysis
+ analysisPrompt, sentiment, err := llmbridge.AnalyzeSentimentTelegram(messages, model, prompt)
+ if err != nil {
+ return "", "", fmt.Errorf("error analyzing sentiment of Telegram messages: %v", err)
+ }
+ return analysisPrompt, sentiment, nil
}
diff --git a/pkg/scrapers/telegram/telegram_client.go b/pkg/scrapers/telegram/telegram_client.go
index 8f60a1e8..c0a7699c 100644
--- a/pkg/scrapers/telegram/telegram_client.go
+++ b/pkg/scrapers/telegram/telegram_client.go
@@ -101,17 +101,21 @@ func StartAuthentication(ctx context.Context, phoneNumber string) (string, error
// CompleteAuthentication uses the provided code to authenticate with Telegram.
func CompleteAuthentication(ctx context.Context, phoneNumber, code, phoneCodeHash string) (*tg.AuthAuthorization, error) {
// Initialize the Telegram client (if not already initialized)
- client, err := InitializeClient()
- if err != nil {
- log.Printf("Failed to initialize Telegram client: %v", err)
- return nil, err
+ client = GetClient()
+
+ if client == nil {
+ var err error
+ client, err = InitializeClient()
+ if err != nil {
+ log.Printf("Failed to initialize Telegram client: %v", err)
+ return nil, err
+ }
}
// Define a variable to hold the authentication result
var authResult *tg.AuthAuthorization
-
// Use client.Run to start the client and execute the SignIn method
- err = client.Run(ctx, func(ctx context.Context) error {
+ err := client.Run(ctx, func(ctx context.Context) error {
// Use the provided code and phoneCodeHash to authenticate
auth, err := client.Auth().SignIn(ctx, phoneNumber, code, phoneCodeHash)
if err != nil {
diff --git a/pkg/workers/handler.go b/pkg/workers/handler.go
index be8f100f..25d47236 100644
--- a/pkg/workers/handler.go
+++ b/pkg/workers/handler.go
@@ -1,6 +1,7 @@
package workers
import (
+ "context"
"encoding/json"
"fmt"
@@ -9,6 +10,7 @@ import (
masa "github.com/masa-finance/masa-oracle/pkg"
"github.com/masa-finance/masa-oracle/pkg/config"
"github.com/masa-finance/masa-oracle/pkg/scrapers/discord"
+ "github.com/masa-finance/masa-oracle/pkg/scrapers/telegram"
"github.com/masa-finance/masa-oracle/pkg/scrapers/twitter"
"github.com/masa-finance/masa-oracle/pkg/scrapers/web"
"github.com/masa-finance/masa-oracle/pkg/workers/messages"
@@ -82,6 +84,8 @@ func (a *Worker) HandleWork(ctx actor.Context, m *messages.Work, node *masa.Orac
}
}
+ opCtx := context.Background() // or ctx.Context if actor.Context provides it
+
switch workData["request"] {
case string(WORKER.DiscordProfile):
userID := bodyData["userID"].(string)
@@ -93,6 +97,10 @@ func (a *Worker) HandleWork(ctx actor.Context, m *messages.Work, node *masa.Orac
logrus.Infof("[+] Discord Channel Messages %s %s", m.Data, m.Sender)
channelID := bodyData["channelID"].(string)
_, resp, err = discord.ScrapeDiscordMessagesForSentiment(channelID, bodyData["model"].(string), bodyData["prompt"].(string))
+ case string(WORKER.TelegramSentiment):
+ logrus.Infof("[+] Telegram Channel Messages %s %s", m.Data, m.Sender)
+ username := bodyData["username"].(string)
+ _, resp, err = telegram.ScrapeTelegramMessagesForSentiment(opCtx, username, bodyData["model"].(string), bodyData["prompt"].(string))
case string(WORKER.DiscordGuildChannels):
guildID := bodyData["guildID"].(string)
resp, err = discord.GetGuildChannels(guildID)
diff --git a/pkg/workers/workers.go b/pkg/workers/workers.go
index 7a18a631..72fab32f 100644
--- a/pkg/workers/workers.go
+++ b/pkg/workers/workers.go
@@ -34,6 +34,7 @@ const (
DiscordProfile WorkerType = "discord-profile"
DiscordChannelMessages WorkerType = "discord-channel-messages"
DiscordSentiment WorkerType = "discord-sentiment"
+ TelegramSentiment WorkerType = "telegram-sentiment"
DiscordGuildChannels WorkerType = "discord-guild-channels"
DiscordUserGuilds WorkerType = "discord-user-guilds"
LLMChat WorkerType = "llm-chat"
@@ -48,12 +49,13 @@ const (
)
var WORKER = struct {
- Discord, DiscordProfile, DiscordChannelMessages, DiscordSentiment, DiscordGuildChannels, DiscordUserGuilds, LLMChat, Twitter, TwitterFollowers, TwitterProfile, TwitterSentiment, TwitterTrends, Web, WebSentiment, Test WorkerType
+ Discord, DiscordProfile, DiscordChannelMessages, DiscordSentiment, TelegramSentiment, DiscordGuildChannels, DiscordUserGuilds, LLMChat, Twitter, TwitterFollowers, TwitterProfile, TwitterSentiment, TwitterTrends, Web, WebSentiment, Test WorkerType
}{
Discord: Discord,
DiscordProfile: DiscordProfile,
DiscordChannelMessages: DiscordChannelMessages,
DiscordSentiment: DiscordSentiment,
+ TelegramSentiment: TelegramSentiment,
DiscordGuildChannels: DiscordGuildChannels,
DiscordUserGuilds: DiscordUserGuilds,
LLMChat: LLMChat,
From 534d916ac440153a50e023cb0fb67aba100dc8e4 Mon Sep 17 00:00:00 2001
From: Nolan Jacobson <50815660+nolanjacobson@users.noreply.github.com>
Date: Wed, 17 Jul 2024 11:47:23 -0400
Subject: [PATCH 06/18] chore: fix discord docs
---
docs/worker-node/discord-worker.md | 18 +++++++++---------
1 file changed, 9 insertions(+), 9 deletions(-)
diff --git a/docs/worker-node/discord-worker.md b/docs/worker-node/discord-worker.md
index 45e355d1..e294a720 100644
--- a/docs/worker-node/discord-worker.md
+++ b/docs/worker-node/discord-worker.md
@@ -13,11 +13,11 @@ As a worker in the Masa Oracle Node network, your primary function is to process
### Worker's Workflow
-1. **Initialization**: Your node, acting as a Worker, is part of a pool managed by a Manager actor. This setup ensures efficient distribution and handling of incoming Discord data requests.
+1 . **Initialization**: Your node, acting as a Worker, is part of a pool managed by a Manager actor. This setup ensures efficient distribution and handling of incoming Discord data requests.
-2. **Receiving Requests**: When a request for Discord data is received, the Manager actor delegates the task to you, the Worker, based on availability and capability.
+2 . **Receiving Requests**: When a request for Discord data is received, the Manager actor delegates the task to you, the Worker, based on availability and capability.
-3. **Processing Requests**: You then fetch the requested data from Discord using the specified criteria and prepare the data for return to the network.
+3 . **Processing Requests**: You then fetch the requested data from Discord using the specified criteria and prepare the data for return to the network.
## Prerequisites for Workers
@@ -31,12 +31,12 @@ To become a worker focused on Discord data requests, you need to:
To start processing Discord data requests, you need to retrieve your Discord bot token, which is essential for authenticating with the Discord API. This token will allow your bot to collect data on your guild. Here's how to get your token:
-1. Go to the [Discord Developer Portal](https://discord.com/developers/applications).
-2. Log in with your Discord account credentials.
-3. Click on the "New Application" button. Give your application a name and confirm the creation.
-4. Navigate to the "Bot" tab on the left-hand side and click on "Add Bot".
-5. Confirm the creation of the bot user.
-6. Under the "TOKEN" section, click on "Copy" to get your bot token.
+1 . Go to the [Discord Developer Portal](https://discord.com/developers/applications).
+2 . Log in with your Discord account credentials.
+3 . Click on the "New Application" button. Give your application a name and confirm the creation.
+4 . Navigate to the "Bot" tab on the left-hand side and click on "Add Bot".
+5 . Confirm the creation of the bot user.
+6 . Under the "TOKEN" section, click on "Copy" to get your bot token.
## 2) Setting Up Your Node for Discord Requests
From e470b511f305b16604dc85a8c83bcc3df594f0c7 Mon Sep 17 00:00:00 2001
From: Nolan Jacobson <50815660+nolanjacobson@users.noreply.github.com>
Date: Thu, 18 Jul 2024 19:43:09 -0400
Subject: [PATCH 07/18] updates to docs
---
docs/oracle-node/twitter-data.md | 198 +++++++++++++++++++++++++++++++
1 file changed, 198 insertions(+)
diff --git a/docs/oracle-node/twitter-data.md b/docs/oracle-node/twitter-data.md
index 603b38dd..c818bed4 100644
--- a/docs/oracle-node/twitter-data.md
+++ b/docs/oracle-node/twitter-data.md
@@ -207,6 +207,204 @@ Example response:
}
```
+## Advanced Search
+
+The Advanced Search feature allows users to perform more complex queries to filter tweets according to various criteria such as date ranges, specific users, hashtags, and more. Below you will find detailed information on how to construct advanced search queries.
+
+
+### Hashtag Search
+Search for tweets containing specific hashtags.
+
+**Syntax:** `#hashtag`
+
+**Example:**
+
+bash
+curl -X POST http://localhost:8080/api/v1/data/twitter/tweets/recent \
+-H "Content-Type: application/json" \
+-d '{"query": "#MasaNode", "count": 10}'
+
+
+### Mention Search
+Search for tweets mentioning a specific user.
+
+**Syntax:** `@username`
+
+**Example:**
+
+bash
+curl -X POST http://localhost:8080/api/v1/data/twitter/tweets/recent \
+-H "Content-Type: application/json" \
+-d '{"query": "@getmasafi", "count": 10}'
+
+### From User Search
+Search for tweets posted by a specific user.
+
+**Syntax:** `from:username`
+
+**Example:**
+
+bash
+curl -X POST http://localhost:8080/api/v1/data/twitter/tweets/recent \
+-H "Content-Type: application/json" \
+-d '{"query": "from:getmasafi", "count": 10}'
+
+### To User Search
+Search for tweets directed to a specific user.
+
+**Syntax:** `to:username`
+
+**Example:**
+
+bash
+curl -X POST http://localhost:8080/api/v1/data/twitter/tweets/recent \
+-H "Content-Type: application/json" \
+-d '{"query": "to:getmasafi", "count": 10}'
+
+### Language-Specific Search
+Search for tweets in a specific language.
+
+**Syntax:** `lang:language_code`
+
+**Example:**
+
+bash
+curl -X POST http://localhost:8080/api/v1/data/twitter/tweets/recent \
+-H "Content-Type: application/json" \
+-d '{"query": "Masa lang:en", "count": 10}'
+
+### Date Range Search
+Search for tweets within a specific date range.
+
+**Syntax:** `since:yyyy-mm-dd until:yyyy-mm-dd`
+
+**Example:**
+
+bash
+curl -X POST http://localhost:8080/api/v1/data/twitter/tweets/recent \
+-H "Content-Type: application/json" \
+-d '{"query": "Masa since:2021-01-01 until:2021-12-31", "count": 10}'
+
+### Retweets Filtering
+Exclude retweets from your search results.
+
+**Syntax:** `-filter:retweets`
+
+**Example:**
+
+bash
+curl -X POST http://localhost:8080/api/v1/data/twitter/tweets/recent \
+-H "Content-Type: application/json" \
+-d '{"query": "Masa -filter:retweets", "count": 10}'
+
+
+### Minimum Likes Filter
+Search for tweets with a minimum number of likes.
+
+**Syntax:** `min_faves:number`
+
+**Example:**
+
+bash
+curl -X POST http://localhost:8080/api/v1/data/twitter/tweets/recent \
+-H "Content-Type: application/json" \
+-d '{"query": "Masa min_faves:100", "count": 10}'
+
+### Minimum Retweets Filter
+Search for tweets with a minimum number of retweets.
+
+**Syntax:** `min_retweets:number`
+
+**Example:**
+
+bash
+curl -X POST http://localhost:8080/api/v1/data/twitter/tweets/recent \
+-H "Content-Type: application/json" \
+-d '{"query": "Masa min_retweets:50", "count": 10}'
+
+
+### Keyword Exclusion
+Exclude tweets with certain keywords.
+
+**Syntax:** `-keyword`
+
+**Example:**
+
+bash
+curl -X POST http://localhost:8080/api/v1/data/twitter/tweets/recent \
+-H "Content-Type: application/json" \
+-d '{"query": "Masa -moon", "count": 10}'
+
+### OR Operator
+Combine multiple terms, where at least one must be present.
+
+**Syntax:** `term1 OR term2`
+
+**Example:**
+
+bash
+curl -X POST http://localhost:8080/api/v1/data/twitter/tweets/recent \
+-H "Content-Type: application/json" \
+-d '{"query": "Masa OR Oracle", "count": 10}'
+
+### Geo-location Based Search
+Search for tweets by users located within a certain radius of a given latitude and longitude.
+
+**Syntax:** `geocode:latitude,longitude,radius`
+
+**Example:**
+
+bash
+curl -X POST http://localhost:8080/api/v1/data/twitter/tweets/recent \
+-H "Content-Type: application/json" \
+-d '{"query": "Masa geocode:37.781157,-122.398720,1mi", "count": 10}'
+
+### URL Inclusion Search
+Search for tweets containing a specific URL.
+
+**Syntax:** `url:"http://example.com"`
+
+**Example:**
+
+bash
+curl -X POST http://localhost:8080/api/v1/data/twitter/tweets/recent \
+-H "Content-Type: application/json" \
+-d '{"query": "url:\"http://example.com\"", "count": 10}'
+
+### Question Tweets Filter
+Search for tweets asking a question.
+
+**Syntax:** `?`
+
+**Example:**
+
+bash
+curl -X POST http://localhost:8080/api/v1/data/twitter/tweets/recent \
+-H "Content-Type: application/json" \
+-d '{"query": "Masa ?", "count": 10}'
+
+### Safe Search Mode
+Exclude adult content from your search results.
+
+**Syntax:** `filter:safe`
+
+**Example:**
+
+### Safe Search Mode
+Exclude adult content from your search results.
+
+**Syntax:** `filter:safe`
+
+**Example:**
+
+bash
+curl -X POST http://localhost:8080/api/v1/data/twitter/tweets/recent \
+-H "Content-Type: application/json" \
+-d '{"query": "Masa filter:safe", "count": 10}'
+
+
+Each of these search types can be combined to create more complex queries, allowing for highly targeted searches. Remember to test each example to ensure they work as expected and to provide users with accurate and helpful documentation.
+
## Use Case: Decentralized AI Agent
Imagine a decentralized AI agent designed to monitor and analyze public sentiment on social media platforms, specifically Twitter, regarding various cryptocurrencies. This agent, let's call it "CryptoSentimentAI," uses the Masa Oracle Node's Twitter data endpoints to gather real-time data on what people are saying about different cryptocurrencies.
From aed9bb93775994bc74c2435ca12edfb85b974841 Mon Sep 17 00:00:00 2001
From: Nolan Jacobson <50815660+nolanjacobson@users.noreply.github.com>
Date: Thu, 18 Jul 2024 20:37:09 -0400
Subject: [PATCH 08/18] chore: updates to swagger docs
---
cmd/telegram-cli/main.go | 168 -----------------------------------
docs/docs.go | 186 +++++++++++++++++++++++++++++++++++++++
pkg/api/routes.go | 2 +-
3 files changed, 187 insertions(+), 169 deletions(-)
delete mode 100644 cmd/telegram-cli/main.go
diff --git a/cmd/telegram-cli/main.go b/cmd/telegram-cli/main.go
deleted file mode 100644
index dd4af330..00000000
--- a/cmd/telegram-cli/main.go
+++ /dev/null
@@ -1,168 +0,0 @@
-package main
-
-import (
- "bufio"
- "context"
- "crypto/rand"
- "fmt"
- "os"
- "path/filepath"
- "strings"
-
- "github.com/gotd/td/session"
- "github.com/gotd/td/telegram"
- "github.com/gotd/td/telegram/auth"
- "github.com/gotd/td/tg"
-)
-
-// Fetch messages from a group
-func fetchChannelMessages(ctx context.Context, client *telegram.Client, channelID int64, accessHash int64) error {
- inputPeer := &tg.InputPeerChannel{ // Use InputPeerChannel instead of InputChannel
- ChannelID: channelID,
- AccessHash: accessHash,
- }
- result, err := client.API().MessagesGetHistory(ctx, &tg.MessagesGetHistoryRequest{
- Peer: inputPeer, // Pass inputPeer here
- Limit: 100, // Adjust the number of messages to fetch
- })
- if err != nil {
- return err
- }
-
- // Type assert the result to *tg.MessagesChannelMessages to access Messages field
- messages, ok := result.(*tg.MessagesChannelMessages)
- if !ok {
- return fmt.Errorf("unexpected type %T", result)
- }
-
- // Process the messages
- for _, m := range messages.Messages {
- message, ok := m.(*tg.Message) // Type assert to *tg.Message
- if !ok {
- // Handle the case where the message is not a regular message (e.g., service message)
- continue
- }
- fmt.Printf("Message ID: %d, Content: %s\n", message.ID, message.Message)
- }
-
- return nil
-}
-
-func resolveChannelUsername(ctx context.Context, client *telegram.Client, username string) (*tg.InputChannel, error) {
- resolved, err := client.API().ContactsResolveUsername(ctx, username)
- if err != nil {
- return nil, err
- }
-
- channel := &tg.InputChannel{
- ChannelID: resolved.Chats[0].GetID(),
- AccessHash: resolved.Chats[0].(*tg.Channel).AccessHash,
- }
-
- fmt.Printf("Channel ID: %d, Access Hash: %d\n", channel.ChannelID, channel.AccessHash)
- return channel, nil
-}
-
-func main() {
- // Define your Telegram app credentials
- appID := 28423325 // Your actual app ID
- appHash := "c60c0a268973ea3f7d52e16e4ab2a0d3" // Your actual app hash
-
- // Define the path to the directory where session data will be stored
- sessionDir := filepath.Join(os.Getenv("HOME"), ".telegram-sessions")
-
- // Create the session directory if it doesn't already exist
- if err := os.MkdirAll(sessionDir, 0700); err != nil {
- fmt.Fprintf(os.Stderr, "Failed to create session directory: %v\n", err)
- os.Exit(1)
- }
-
- // Create a session storage
- storage := &session.FileStorage{
- Path: filepath.Join(sessionDir, "session.json"),
- }
-
- // Create a random seed for the client
- seed := make([]byte, 32)
- if _, err := rand.Read(seed); err != nil {
- fmt.Fprintf(os.Stderr, "Failed to generate random seed: %v\n", err)
- os.Exit(1)
- }
-
- client := telegram.NewClient(appID, appHash, telegram.Options{
- SessionStorage: storage,
- })
-
- // Continue with the rest of your application logic
- ctx := context.Background()
-
- // Check if the session file already exists and has content
- sessionExists := false
- if info, err := os.Stat(filepath.Join(sessionDir, "session.json")); err == nil && info.Size() > 0 {
- sessionExists = true
- }
-
- // Define the phone number and password (if 2FA is enabled)
- phone := "+13053398321"
- password := "" // Leave empty if 2FA is not enabled
-
- // Define the code prompt function
- codePrompt := func(ctx context.Context, sentCode *tg.AuthSentCode) (string, error) {
- fmt.Print("Enter code: ")
- code, err := bufio.NewReader(os.Stdin).ReadString('\n')
- if err != nil {
- return "", err
- }
- return strings.TrimSpace(code), nil
- }
-
- // Set up and perform the authentication flow only if no valid session exists
- if !sessionExists {
- authFlow := auth.NewFlow(
- auth.Constant(phone, password, auth.CodeAuthenticatorFunc(codePrompt)),
- auth.SendCodeOptions{},
- )
- if err := client.Run(ctx, func(ctx context.Context) error {
- return authFlow.Run(ctx, client.Auth())
- }); err != nil {
- fmt.Fprintf(os.Stderr, "Authentication failed: %v\n", err)
- os.Exit(1)
- }
- } else {
- // If a session exists, simply start the client
- if err := client.Run(ctx, func(ctx context.Context) error {
- username := "coinlistofficialchannel" // Replace with the actual username of the channel
- channel, err := resolveChannelUsername(ctx, client, username)
- if err != nil {
- fmt.Fprintf(os.Stderr, "Failed to resolve channel username: %v\n", err)
- os.Exit(1)
- }
- if err := fetchChannelMessages(ctx, client, channel.ChannelID, channel.AccessHash); err != nil {
- fmt.Fprintf(os.Stderr, "Failed to fetch channel messages: %v\n", err)
- os.Exit(1)
- }
- return nil // No operation, just start the client
- }); err != nil {
- fmt.Fprintf(os.Stderr, "Failed to start client with existing session: %v\n", err)
- os.Exit(1)
- }
- }
-
- // The user is now authenticated, or a valid session is loaded
- // You can use the client to interact with the Telegram API
-
- // username := "coinlistofficialchannel" // Replace with the actual username of the channel
- // channel, err := resolveChannelUsername(ctx, client, username)
- // if err != nil {
- // fmt.Fprintf(os.Stderr, "Failed to resolve channel username: %v\n", err)
- // os.Exit(1)
- // }
-
- // // return channel
-
- // if err := fetchChannelMessages(ctx, client, channel.ChannelID, channel.AccessHash); err != nil {
- // fmt.Fprintf(os.Stderr, "Failed to fetch channel messages: %v\n", err)
- // os.Exit(1)
- // }
-
-}
diff --git a/docs/docs.go b/docs/docs.go
index 24d49473..e2dcff04 100644
--- a/docs/docs.go
+++ b/docs/docs.go
@@ -474,6 +474,44 @@ const docTemplate = `{
]
}
},
+ "/data/telegram/channel/{username}/messages": {
+ "get": {
+ "description": "Retrieves messages from a specified Telegram channel.",
+ "tags": ["Telegram"],
+ "summary": "Get Telegram Channel Messages",
+ "parameters": [
+ {
+ "name": "username",
+ "in": "path",
+ "description": "Telegram Username",
+ "required": true,
+ "type": "string"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "Successfully retrieved messages",
+ "schema": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/Message"
+ }
+ }
+ },
+ "400": {
+ "description": "Invalid username or error fetching messages",
+ "schema": {
+ "$ref": "#/definitions/ErrorResponse"
+ }
+ }
+ },
+ "security": [
+ {
+ "Bearer": []
+ }
+ ]
+ }
+ },
"/data/web": {
"post": {
"description": "Retrieves data from the web",
@@ -848,6 +886,45 @@ const docTemplate = `{
}
}
},
+ "/sentiment/telegram": {
+ "post": {
+ "description": "Searches for Telegram messages and analyzes their sentiment",
+ "tags": ["Sentiment"],
+ "summary": "Analyze Sentiment of Telegram Messages",
+ "consumes": ["application/json"],
+ "produces": ["application/json"],
+ "parameters": [
+ {
+ "name": "query",
+ "in": "body",
+ "description": "Search Query",
+ "required": true,
+ "schema": {
+ "type": "object",
+ "properties": {
+ "query": {
+ "type": "string"
+ }
+ }
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "Successfully analyzed sentiment of Telegram messages",
+ "schema": {
+ "$ref": "#/definitions/SentimentAnalysisResponse"
+ }
+ },
+ "400": {
+ "description": "Error analyzing sentiment of Telegram messages",
+ "schema": {
+ "$ref": "#/definitions/ErrorResponse"
+ }
+ }
+ }
+ }
+ },
"/sentiment/discord": {
"post": {
"description": "Searches for Discord messages and analyzes their sentiment",
@@ -932,6 +1009,115 @@ const docTemplate = `{
}
}
},
+ "/auth/telegram/start": {
+ "post": {
+ "description": "Initiates the authentication process with Telegram by sending a code to the provided phone number.",
+ "tags": ["Authentication"],
+ "summary": "Start Telegram Authentication",
+ "consumes": ["application/json"],
+ "produces": ["application/json"],
+ "parameters": [
+ {
+ "name": "phone_number",
+ "in": "body",
+ "description": "Phone Number",
+ "required": true,
+ "schema": {
+ "type": "object",
+ "properties": {
+ "phone_number": {
+ "type": "string"
+ }
+ }
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "Successfully sent authentication code",
+ "schema": {
+ "type": "object",
+ "additionalProperties": {
+ "type": "string"
+ }
+ }
+ },
+ "400": {
+ "description": "Invalid request body",
+ "schema": {
+ "$ref": "#/definitions/ErrorResponse"
+ }
+ },
+ "500": {
+ "description": "Failed to initialize Telegram client or to start authentication",
+ "schema": {
+ "$ref": "#/definitions/ErrorResponse"
+ }
+ }
+ }
+ }
+ },
+ "/auth/telegram/complete": {
+ "post": {
+ "description": "Completes the authentication process with Telegram using the code sent to the phone number.",
+ "tags": ["Authentication"],
+ "summary": "Complete Telegram Authentication",
+ "consumes": ["application/json"],
+ "produces": ["application/json"],
+ "parameters": [
+ {
+ "name": "phone_number",
+ "in": "body",
+ "description": "Phone Number",
+ "required": true,
+ "schema": {
+ "type": "object",
+ "properties": {
+ "phone_number": {
+ "type": "string"
+ },
+ "code": {
+ "type": "string"
+ },
+ "phone_code_hash": {
+ "type": "string"
+ }
+ },
+ "required": ["phone_number", "code", "phone_code_hash"]
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "Successfully authenticated",
+ "schema": {
+ "type": "object",
+ "additionalProperties": {
+ "type": "string"
+ }
+ }
+ },
+ "400": {
+ "description": "Invalid request body",
+ "schema": {
+ "$ref": "#/definitions/ErrorResponse"
+ }
+ },
+ "401": {
+ "description": "Two-factor authentication is required",
+ "schema": {
+ "$ref": "#/definitions/ErrorResponse"
+ }
+ },
+ "500": {
+ "description": "Failed to initialize Telegram client or to complete authentication",
+ "schema": {
+ "$ref": "#/definitions/ErrorResponse"
+ }
+ }
+ }
+ }
+ },
"/sentiment/web": {
"post": {
"description": "Searches for web content and analyzes its sentiment",
diff --git a/pkg/api/routes.go b/pkg/api/routes.go
index 120e764c..bc441a23 100644
--- a/pkg/api/routes.go
+++ b/pkg/api/routes.go
@@ -254,7 +254,7 @@ func SetupRoutes(node *masa.OracleNode) *gin.Engine {
// @Failure 400 {object} ErrorResponse "Username must be provided"
// @Failure 500 {object} ErrorResponse "Failed to fetch channel messages"
// @Router /telegram/channel/{username}/messages [get]
- v1.GET("/telegram/channel/:username/messages", API.GetChannelMessagesHandler())
+ v1.GET("/data/telegram/channel/:username/messages", API.GetChannelMessagesHandler())
// oauth tests
// v1.GET("/data/discord/exchangetoken/:code", API.ExchangeDiscordTokenHandler())
From eb38cce4ef9d49c1d4ee3fb3e8f97f74fdbe37e6 Mon Sep 17 00:00:00 2001
From: Nolan Jacobson <50815660+nolanjacobson@users.noreply.github.com>
Date: Mon, 22 Jul 2024 13:35:23 -0400
Subject: [PATCH 09/18] stash
---
tg_public_keys.pem | 16 ----------------
1 file changed, 16 deletions(-)
delete mode 100644 tg_public_keys.pem
diff --git a/tg_public_keys.pem b/tg_public_keys.pem
deleted file mode 100644
index 1de207e8..00000000
--- a/tg_public_keys.pem
+++ /dev/null
@@ -1,16 +0,0 @@
------BEGIN RSA PUBLIC KEY-----
-MIIBCgKCAQEAyMEdY1aR+sCR3ZSJrtztKTKqigvO/vBfqACJLZtS7QMgCGXJ6XIR
-yy7mx66W0/sOFa7/1mAZtEoIokDP3ShoqF4fVNb6XeqgQfaUHd8wJpDWHcR2OFwv
-plUUI1PLTktZ9uW2WE23b+ixNwJjJGwBDJPQEQFBE+vfmH0JP503wr5INS1poWg/
-j25sIWeYPHYeOrFp/eXaqhISP6G+q2IeTaWTXpwZj4LzXq5YOpk4bYEQ6mvRq7D1
-aHWfYmlEGepfaYR8Q0YqvvhYtMte3ITnuSJs171+GDqpdKcSwHnd6FudwGO4pcCO
-j4WcDuXc2CTHgH8gFTNhp/Y8/SpDOhvn9QIDAQAB
------END RSA PUBLIC KEY-----
------BEGIN RSA PUBLIC KEY-----
-MIIBCgKCAQEA6LszBcC1LGzyr992NzE0ieY+BSaOW622Aa9Bd4ZHLl+TuFQ4lo4g
-5nKaMBwK/BIb9xUfg0Q29/2mgIR6Zr9krM7HjuIcCzFvDtr+L0GQjae9H0pRB2OO
-62cECs5HKhT5DZ98K33vmWiLowc621dQuwKWSQKjWf50XYFw42h21P2KXUGyp2y/
-+aEyZ+uVgLLQbRA1dEjSDZ2iGRy12Mk5gpYc397aYp438fsJoHIgJ2lgMv5h7WY9
-t6N/byY9Nw9p21Og3AoXSL2q/2IJ1WRUhebgAdGVMlV1fkuOQoEzR7EdpqtQD9Cs
-5+bfo3Nhmcyvk5ftB0WkJ9z6bNZ7yxrP8wIDAQAB
------END RSA PUBLIC KEY-----
\ No newline at end of file
From a20391835f6cd02d914422858fae98a8d39afbd7 Mon Sep 17 00:00:00 2001
From: Nolan Jacobson <50815660+nolanjacobson@users.noreply.github.com>
Date: Wed, 24 Jul 2024 20:28:31 -0400
Subject: [PATCH 10/18] chore: final touches to Telegram PR
---
cmd/masa-node/main.go | 2 +-
go.sum | 4 +-
pkg/api/handlers_data.go | 31 +++++++-----
pkg/api/handlers_node.go | 28 ++++++-----
pkg/api/templates/index.html | 16 ++++++
pkg/config/app.go | 1 +
pkg/config/constants.go | 1 +
pkg/config/welcome.go | 3 +-
pkg/oracle_node.go | 85 +++++++++++++++++---------------
pkg/pubsub/node_data.go | 10 ++++
pkg/pubsub/node_event_tracker.go | 1 +
pkg/workers/handler.go | 2 +-
pkg/workers/workers.go | 70 +++++++++++++-------------
13 files changed, 151 insertions(+), 103 deletions(-)
diff --git a/cmd/masa-node/main.go b/cmd/masa-node/main.go
index fd0e19aa..b7d4b650 100644
--- a/cmd/masa-node/main.go
+++ b/cmd/masa-node/main.go
@@ -125,7 +125,7 @@ func main() {
multiAddr := node.GetMultiAddrs().String() // Get the multiaddress
ipAddr := node.Host.Addrs()[0].String() // Get the IP address
// Display the welcome message with the multiaddress and IP address
- config.DisplayWelcomeMessage(multiAddr, ipAddr, keyManager.EthAddress, isStaked, isValidator, cfg.TwitterScraper, cfg.DiscordScraper, cfg.WebScraper, config.Version)
+ config.DisplayWelcomeMessage(multiAddr, ipAddr, keyManager.EthAddress, isStaked, isValidator, cfg.TwitterScraper, cfg.TelegramScraper, cfg.DiscordScraper, cfg.WebScraper, config.Version)
<-ctx.Done()
}
diff --git a/go.sum b/go.sum
index c37af138..354b2e31 100644
--- a/go.sum
+++ b/go.sum
@@ -669,8 +669,8 @@ github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6g
github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ=
github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca h1:NugYot0LIVPxTvN8n+Kvkn6TrbMyxQiuvKdEwFdR9vI=
github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU=
-github.com/sashabaranov/go-openai v1.26.2 h1:cVlQa3gn3eYqNXRW03pPlpy6zLG52EU4g0FrWXc0EFI=
-github.com/sashabaranov/go-openai v1.26.2/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg=
+github.com/sashabaranov/go-openai v1.26.3 h1:Tjnh4rcvsSU68f66r05mys+Zou4vo4qyvkne6AIRJPI=
+github.com/sashabaranov/go-openai v1.26.3/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg=
github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys=
github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs=
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
diff --git a/pkg/api/handlers_data.go b/pkg/api/handlers_data.go
index 9772c51f..6470ce99 100644
--- a/pkg/api/handlers_data.go
+++ b/pkg/api/handlers_data.go
@@ -6,7 +6,6 @@ import (
"encoding/json"
"fmt"
"io"
- "log"
"net/http"
"os"
"sync"
@@ -823,22 +822,32 @@ func (api *API) CompleteAuth() gin.HandlerFunc {
func (api *API) GetChannelMessagesHandler() gin.HandlerFunc {
return func(c *gin.Context) {
- username := c.Param("username")
- if username == "" {
- c.JSON(http.StatusBadRequest, gin.H{"error": "Username must be provided"})
+ var reqBody struct {
+ Username string `json:"username"` // Telegram usernames are used instead of channel IDs
+ }
+ if err := c.ShouldBindJSON(&reqBody); err != nil {
+ c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid request body"})
return
}
- log.Printf("Starting to fetch messages for username: %s", username)
- messages, err := telegram.FetchChannelMessages(c.Request.Context(), username)
- if err != nil {
- log.Printf("Error fetching messages for username: %s, error: %v", username, err)
- c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+ if reqBody.Username == "" {
+ c.JSON(http.StatusBadRequest, gin.H{"error": "Username parameter is missing"})
return
}
- log.Printf("Finished fetching messages for username: %s", username)
- c.JSON(http.StatusOK, gin.H{"messages": messages})
+ // worker handler implementation
+ bodyBytes, err := json.Marshal(reqBody)
+ if err != nil {
+ c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+ }
+ requestID := uuid.New().String()
+ responseCh := pubsub2.GetResponseChannelMap().CreateChannel(requestID)
+ defer pubsub2.GetResponseChannelMap().Delete(requestID)
+ err = publishWorkRequest(api, requestID, workers.WORKER.TelegramChannelMessages, bodyBytes)
+ if err != nil {
+ c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+ }
+ handleWorkResponse(c, responseCh)
}
}
diff --git a/pkg/api/handlers_node.go b/pkg/api/handlers_node.go
index 1d4ae801..7372017c 100644
--- a/pkg/api/handlers_node.go
+++ b/pkg/api/handlers_node.go
@@ -390,19 +390,20 @@ func (api *API) NodeStatusPageHandler() gin.HandlerFunc {
return func(c *gin.Context) {
// Initialize default values for the template data
templateData := gin.H{
- "TotalPeers": 0,
- "Name": "Masa Status Page",
- "PeerID": api.Node.Host.ID().String(),
- "IsStaked": false,
- "IsTwitterScraper": false,
- "IsDiscordScraper": false,
- "IsWebScraper": false,
- "FirstJoined": api.Node.FromUnixTime(time.Now().Unix()),
- "LastJoined": api.Node.FromUnixTime(time.Now().Unix()),
- "CurrentUptime": "0",
- "TotalUptime": "0",
- "Rewards": "Coming Soon!",
- "BytesScraped": "0 MB",
+ "TotalPeers": 0,
+ "Name": "Masa Status Page",
+ "PeerID": api.Node.Host.ID().String(),
+ "IsStaked": false,
+ "IsTwitterScraper": false,
+ "IsDiscordScraper": false,
+ "IsTelegramScraper": false,
+ "IsWebScraper": false,
+ "FirstJoined": api.Node.FromUnixTime(time.Now().Unix()),
+ "LastJoined": api.Node.FromUnixTime(time.Now().Unix()),
+ "CurrentUptime": "0",
+ "TotalUptime": "0",
+ "Rewards": "Coming Soon!",
+ "BytesScraped": "0 MB",
}
if api.Node != nil && api.Node.Host != nil {
@@ -414,6 +415,7 @@ func (api *API) NodeStatusPageHandler() gin.HandlerFunc {
templateData["IsStaked"] = nd.IsStaked
templateData["IsTwitterScraper"] = nd.IsTwitterScraper
templateData["IsDiscordScraper"] = nd.IsDiscordScraper
+ templateData["IsTelegramScraper"] = nd.IsTelegramScraper
templateData["IsWebScraper"] = nd.IsWebScraper
templateData["FirstJoined"] = api.Node.FromUnixTime(nd.FirstJoinedUnix)
templateData["LastJoined"] = api.Node.FromUnixTime(nd.LastJoinedUnix)
diff --git a/pkg/api/templates/index.html b/pkg/api/templates/index.html
index 24030ed6..d689fce1 100644
--- a/pkg/api/templates/index.html
+++ b/pkg/api/templates/index.html
@@ -70,6 +70,22 @@ Node Information
{{end}}
+
+ | Telegram Scraper |
+ {{if .IsTelegramScraper}}
+
+ {{.IsTelegramScraper}}
+ |
+ {{else}}
+
+ {{.IsTelegramScraper}}
+ |
+ {{end}}
+
| Web Scraper |
{{if .IsWebScraper}}
diff --git a/pkg/config/app.go b/pkg/config/app.go
index 5c4cf99a..a3f458bf 100644
--- a/pkg/config/app.go
+++ b/pkg/config/app.go
@@ -90,6 +90,7 @@ type AppConfig struct {
GPTApiKey string `mapstructure:"gptApiKey"`
TwitterScraper bool `mapstructure:"twitterScraper"`
DiscordScraper bool `mapstructure:"discordScraper"`
+ TelegramScraper bool `mapstructure:"telegramScraper"`
WebScraper bool `mapstructure:"webScraper"`
LlmServer bool `mapstructure:"llmServer"`
LLMChatUrl string `mapstructure:"llmChatUrl"`
diff --git a/pkg/config/constants.go b/pkg/config/constants.go
index 9ddbbf5f..508053c1 100644
--- a/pkg/config/constants.go
+++ b/pkg/config/constants.go
@@ -120,6 +120,7 @@ const (
GPTApiKey = "OPENAI_API_KEY"
TwitterScraper = "TWITTER_SCRAPER"
DiscordScraper = "DISCORD_SCRAPER"
+ TelegramScraper = "TELEGRAM_SCRAPER"
WebScraper = "WEB_SCRAPER"
LlmServer = "LLM_SERVER"
LlmChatUrl = "LLM_CHAT_URL"
diff --git a/pkg/config/welcome.go b/pkg/config/welcome.go
index b26d3010..968edd0e 100644
--- a/pkg/config/welcome.go
+++ b/pkg/config/welcome.go
@@ -4,7 +4,7 @@ import (
"fmt"
)
-func DisplayWelcomeMessage(multiAddr, ipAddr, publicKeyHex string, isStaked bool, isValidator bool, isTwitterScraper bool, isDiscordScraper bool, isWebScraper bool, version string) {
+func DisplayWelcomeMessage(multiAddr, ipAddr, publicKeyHex string, isStaked bool, isValidator bool, isTwitterScraper bool, isTelegramScraper bool, isDiscordScraper bool, isWebScraper bool, version string) {
// ANSI escape code for yellow text
yellow := "\033[33m"
blue := "\033[34m"
@@ -31,6 +31,7 @@ func DisplayWelcomeMessage(multiAddr, ipAddr, publicKeyHex string, isStaked bool
fmt.Printf(blue+"%-20s %t\n"+reset, "Is Validator:", isValidator)
fmt.Printf(blue+"%-20s %t\n"+reset, "Is TwitterScraper:", isTwitterScraper)
fmt.Printf(blue+"%-20s %t\n"+reset, "Is DiscordScraper:", isDiscordScraper)
+ fmt.Printf(blue+"%-20s %t\n"+reset, "Is TelegramScraper:", isTelegramScraper)
fmt.Printf(blue+"%-20s %t\n"+reset, "Is WebScraper:", isWebScraper)
fmt.Println("")
}
diff --git a/pkg/oracle_node.go b/pkg/oracle_node.go
index d8f6c302..a0fdd080 100644
--- a/pkg/oracle_node.go
+++ b/pkg/oracle_node.go
@@ -42,29 +42,30 @@ import (
)
type OracleNode struct {
- Host host.Host
- PrivKey *ecdsa.PrivateKey
- Protocol protocol.ID
- priorityAddrs multiaddr.Multiaddr
- multiAddrs []multiaddr.Multiaddr
- DHT *dht.IpfsDHT
- Context context.Context
- PeerChan chan myNetwork.PeerEvent
- NodeTracker *pubsub2.NodeEventTracker
- PubSubManager *pubsub2.Manager
- Signature string
- IsStaked bool
- IsValidator bool
- IsTwitterScraper bool
- IsDiscordScraper bool
- IsWebScraper bool
- IsLlmServer bool
- StartTime time.Time
- WorkerTracker *pubsub2.WorkerEventTracker
- BlockTracker *pubsub2.BlockEventTracker
- ActorEngine *actor.RootContext
- ActorRemote *remote.Remote
- Blockchain *chain.Chain
+ Host host.Host
+ PrivKey *ecdsa.PrivateKey
+ Protocol protocol.ID
+ priorityAddrs multiaddr.Multiaddr
+ multiAddrs []multiaddr.Multiaddr
+ DHT *dht.IpfsDHT
+ Context context.Context
+ PeerChan chan myNetwork.PeerEvent
+ NodeTracker *pubsub2.NodeEventTracker
+ PubSubManager *pubsub2.Manager
+ Signature string
+ IsStaked bool
+ IsValidator bool
+ IsTwitterScraper bool
+ IsDiscordScraper bool
+ IsTelegramScraper bool
+ IsWebScraper bool
+ IsLlmServer bool
+ StartTime time.Time
+ WorkerTracker *pubsub2.WorkerEventTracker
+ BlockTracker *pubsub2.BlockEventTracker
+ ActorEngine *actor.RootContext
+ ActorRemote *remote.Remote
+ Blockchain *chain.Chain
}
// GetMultiAddrs returns the priority multiaddr for this node.
@@ -148,6 +149,8 @@ func NewOracleNode(ctx context.Context, isStaked bool) (*OracleNode, error) {
isValidator, _ := strconv.ParseBool(cfg.Validator)
isTwitterScraper := cfg.TwitterScraper
isDiscordScraper := cfg.DiscordScraper
+ isTelegramScraper := cfg.TelegramScraper
+
isWebScraper := cfg.WebScraper
system := actor.NewActorSystemWithConfig(actor.Configure(
@@ -172,23 +175,24 @@ func NewOracleNode(ctx context.Context, isStaked bool) (*OracleNode, error) {
go r.Start()
return &OracleNode{
- Host: hst,
- PrivKey: masacrypto.KeyManagerInstance().EcdsaPrivKey,
- Protocol: config.ProtocolWithVersion(config.OracleProtocol),
- multiAddrs: myNetwork.GetMultiAddressesForHostQuiet(hst),
- Context: ctx,
- PeerChan: make(chan myNetwork.PeerEvent),
- NodeTracker: pubsub2.NewNodeEventTracker(config.Version, cfg.Environment),
- PubSubManager: subscriptionManager,
- IsStaked: isStaked,
- IsValidator: isValidator,
- IsTwitterScraper: isTwitterScraper,
- IsDiscordScraper: isDiscordScraper,
- IsWebScraper: isWebScraper,
- IsLlmServer: cfg.LlmServer,
- ActorEngine: engine,
- ActorRemote: r,
- Blockchain: &chain.Chain{},
+ Host: hst,
+ PrivKey: masacrypto.KeyManagerInstance().EcdsaPrivKey,
+ Protocol: config.ProtocolWithVersion(config.OracleProtocol),
+ multiAddrs: myNetwork.GetMultiAddressesForHostQuiet(hst),
+ Context: ctx,
+ PeerChan: make(chan myNetwork.PeerEvent),
+ NodeTracker: pubsub2.NewNodeEventTracker(config.Version, cfg.Environment),
+ PubSubManager: subscriptionManager,
+ IsStaked: isStaked,
+ IsValidator: isValidator,
+ IsTwitterScraper: isTwitterScraper,
+ IsDiscordScraper: isDiscordScraper,
+ IsTelegramScraper: isTelegramScraper,
+ IsWebScraper: isWebScraper,
+ IsLlmServer: cfg.LlmServer,
+ ActorEngine: engine,
+ ActorRemote: r,
+ Blockchain: &chain.Chain{},
}, nil
}
@@ -236,6 +240,7 @@ func (node *OracleNode) Start() (err error) {
cfg := config.GetInstance()
nodeData.IsDiscordScraper = cfg.DiscordScraper
+ nodeData.IsTelegramScraper = cfg.TelegramScraper
nodeData.IsTwitterScraper = cfg.TwitterScraper
nodeData.IsWebScraper = cfg.WebScraper
nodeData.IsValidator = cfg.Validator == "true"
diff --git a/pkg/pubsub/node_data.go b/pkg/pubsub/node_data.go
index b7a57ee8..114da65e 100644
--- a/pkg/pubsub/node_data.go
+++ b/pkg/pubsub/node_data.go
@@ -63,6 +63,7 @@ type NodeData struct {
IsValidator bool `json:"isValidator"`
IsTwitterScraper bool `json:"isTwitterScraper"`
IsDiscordScraper bool `json:"isDiscordScraper"`
+ IsTelegramScraper bool `json:"isTelegramScraper"`
IsWebScraper bool `json:"isWebScraper"`
BytesScraped int `json:"bytesScraped"`
Records any `json:"records,omitempty"`
@@ -78,6 +79,7 @@ func NewNodeData(addr multiaddr.Multiaddr, peerId peer.ID, publicKey string, act
wn, _ := strconv.ParseBool(cfg.Validator)
ts := cfg.TwitterScraper
ds := cfg.DiscordScraper
+ tgs := cfg.TelegramScraper
ws := cfg.WebScraper
ver := cfg.Version
@@ -93,6 +95,7 @@ func NewNodeData(addr multiaddr.Multiaddr, peerId peer.ID, publicKey string, act
IsValidator: wn,
IsTwitterScraper: ts,
IsDiscordScraper: ds,
+ IsTelegramScraper: tgs,
IsWebScraper: ws,
BytesScraped: 0,
Version: ver,
@@ -140,6 +143,13 @@ func (n *NodeData) DiscordScraper() bool {
return cfg.DiscordScraper
}
+// TelegramScraper checks if the current node is configured as a Telegram scraper.
+// It retrieves the configuration instance and returns the value of the TelegramScraper field.
+func (n *NodeData) TelegramScraper() bool {
+ cfg := config.GetInstance()
+ return cfg.TelegramScraper
+}
+
// WebScraper checks if the current node is configured as a Web scraper.
// It retrieves the configuration instance and returns the value of the WebScraper field.
func (n *NodeData) WebScraper() bool {
diff --git a/pkg/pubsub/node_event_tracker.go b/pkg/pubsub/node_event_tracker.go
index 20eecaca..05bc55e2 100644
--- a/pkg/pubsub/node_event_tracker.go
+++ b/pkg/pubsub/node_event_tracker.go
@@ -303,6 +303,7 @@ func (net *NodeEventTracker) AddOrUpdateNodeData(nodeData *NodeData, forceGossip
nd.BytesScraped = nodeData.BytesScraped
nd.IsStaked = nodeData.IsStaked
nd.IsDiscordScraper = nodeData.IsDiscordScraper
+ nd.IsTelegramScraper = nodeData.IsTelegramScraper
nd.IsTwitterScraper = nodeData.IsTwitterScraper
nd.IsWebScraper = nodeData.IsWebScraper
nd.Records = nodeData.Records
diff --git a/pkg/workers/handler.go b/pkg/workers/handler.go
index 25d47236..7b2280ff 100644
--- a/pkg/workers/handler.go
+++ b/pkg/workers/handler.go
@@ -180,7 +180,7 @@ func (a *Worker) HandleWork(ctx actor.Context, m *messages.Work, node *masa.Orac
return
}
cfg := config.GetInstance()
- if cfg.TwitterScraper || cfg.DiscordScraper || cfg.WebScraper {
+ if cfg.TwitterScraper || cfg.DiscordScraper || cfg.TwitterScraper || cfg.WebScraper {
ctx.Respond(&messages.Response{RequestId: workData["request_id"], Value: string(jsn)})
}
for _, pid := range getPeers(node) {
diff --git a/pkg/workers/workers.go b/pkg/workers/workers.go
index 62c9733f..b3e1e977 100644
--- a/pkg/workers/workers.go
+++ b/pkg/workers/workers.go
@@ -30,43 +30,45 @@ import (
type WorkerType string
const (
- Discord WorkerType = "discord"
- DiscordProfile WorkerType = "discord-profile"
- DiscordChannelMessages WorkerType = "discord-channel-messages"
- DiscordSentiment WorkerType = "discord-sentiment"
- TelegramSentiment WorkerType = "telegram-sentiment"
- DiscordGuildChannels WorkerType = "discord-guild-channels"
- DiscordUserGuilds WorkerType = "discord-user-guilds"
- LLMChat WorkerType = "llm-chat"
- Twitter WorkerType = "twitter"
- TwitterFollowers WorkerType = "twitter-followers"
- TwitterProfile WorkerType = "twitter-profile"
- TwitterSentiment WorkerType = "twitter-sentiment"
- TwitterTrends WorkerType = "twitter-trends"
- Web WorkerType = "web"
- WebSentiment WorkerType = "web-sentiment"
- Test WorkerType = "test"
+ Discord WorkerType = "discord"
+ DiscordProfile WorkerType = "discord-profile"
+ DiscordChannelMessages WorkerType = "discord-channel-messages"
+ DiscordSentiment WorkerType = "discord-sentiment"
+ TelegramSentiment WorkerType = "telegram-sentiment"
+ TelegramChannelMessages WorkerType = "telegram-channel-messages"
+ DiscordGuildChannels WorkerType = "discord-guild-channels"
+ DiscordUserGuilds WorkerType = "discord-user-guilds"
+ LLMChat WorkerType = "llm-chat"
+ Twitter WorkerType = "twitter"
+ TwitterFollowers WorkerType = "twitter-followers"
+ TwitterProfile WorkerType = "twitter-profile"
+ TwitterSentiment WorkerType = "twitter-sentiment"
+ TwitterTrends WorkerType = "twitter-trends"
+ Web WorkerType = "web"
+ WebSentiment WorkerType = "web-sentiment"
+ Test WorkerType = "test"
)
var WORKER = struct {
- Discord, DiscordProfile, DiscordChannelMessages, DiscordSentiment, TelegramSentiment, DiscordGuildChannels, DiscordUserGuilds, LLMChat, Twitter, TwitterFollowers, TwitterProfile, TwitterSentiment, TwitterTrends, Web, WebSentiment, Test WorkerType
+ Discord, DiscordProfile, DiscordChannelMessages, DiscordSentiment, TelegramSentiment, TelegramChannelMessages, DiscordGuildChannels, DiscordUserGuilds, LLMChat, Twitter, TwitterFollowers, TwitterProfile, TwitterSentiment, TwitterTrends, Web, WebSentiment, Test WorkerType
}{
- Discord: Discord,
- DiscordProfile: DiscordProfile,
- DiscordChannelMessages: DiscordChannelMessages,
- DiscordSentiment: DiscordSentiment,
- TelegramSentiment: TelegramSentiment,
- DiscordGuildChannels: DiscordGuildChannels,
- DiscordUserGuilds: DiscordUserGuilds,
- LLMChat: LLMChat,
- Twitter: Twitter,
- TwitterFollowers: TwitterFollowers,
- TwitterProfile: TwitterProfile,
- TwitterSentiment: TwitterSentiment,
- TwitterTrends: TwitterTrends,
- Web: Web,
- WebSentiment: WebSentiment,
- Test: Test,
+ Discord: Discord,
+ DiscordProfile: DiscordProfile,
+ DiscordChannelMessages: DiscordChannelMessages,
+ DiscordSentiment: DiscordSentiment,
+ TelegramSentiment: TelegramSentiment,
+ TelegramChannelMessages: TelegramChannelMessages,
+ DiscordGuildChannels: DiscordGuildChannels,
+ DiscordUserGuilds: DiscordUserGuilds,
+ LLMChat: LLMChat,
+ Twitter: Twitter,
+ TwitterFollowers: TwitterFollowers,
+ TwitterProfile: TwitterProfile,
+ TwitterSentiment: TwitterSentiment,
+ TwitterTrends: TwitterTrends,
+ Web: Web,
+ WebSentiment: WebSentiment,
+ Test: Test,
}
var (
@@ -329,7 +331,7 @@ func SendWork(node *masa.OracleNode, m *pubsub2.Message) {
for _, p := range peers {
for _, addr := range p.Multiaddrs {
ipAddr, _ := addr.ValueForProtocol(multiaddr.P_IP4)
- if !isBootnode(ipAddr) && (p.IsTwitterScraper || p.IsWebScraper || p.IsDiscordScraper) {
+ if !isBootnode(ipAddr) && (p.IsTwitterScraper || p.IsWebScraper || p.IsDiscordScraper || p.IsTelegramScraper) {
logrus.Infof("[+] Worker Address: %s", ipAddr)
wg.Add(1)
go func(p pubsub.NodeData) {
From 300f82ca19f16de8cfe647997dae324e87a597d5 Mon Sep 17 00:00:00 2001
From: Nolan Jacobson <50815660+nolanjacobson@users.noreply.github.com>
Date: Wed, 24 Jul 2024 20:56:38 -0400
Subject: [PATCH 11/18] chore: saved file
---
pkg/oracle_node.go | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pkg/oracle_node.go b/pkg/oracle_node.go
index a0fdd080..50dde539 100644
--- a/pkg/oracle_node.go
+++ b/pkg/oracle_node.go
@@ -319,7 +319,7 @@ func (node *OracleNode) handleStream(stream network.Stream) {
func (node *OracleNode) IsWorker() bool {
// need to get this by node data
cfg := config.GetInstance()
- if cfg.TwitterScraper || cfg.DiscordScraper || cfg.WebScraper {
+ if cfg.TwitterScraper || cfg.DiscordScraper || cfg.TelegramScraper || cfg.WebScraper {
return true
}
return false
From 22a6bb2190ac3964d5a2501095ec19bba859dd13 Mon Sep 17 00:00:00 2001
From: Nolan Jacobson <50815660+nolanjacobson@users.noreply.github.com>
Date: Wed, 24 Jul 2024 21:00:55 -0400
Subject: [PATCH 12/18] chore: update vyper
---
pkg/config/app.go | 3 +++
pkg/oracle_node.go | 2 +-
2 files changed, 4 insertions(+), 1 deletion(-)
diff --git a/pkg/config/app.go b/pkg/config/app.go
index a3f458bf..b193e9f6 100644
--- a/pkg/config/app.go
+++ b/pkg/config/app.go
@@ -209,6 +209,7 @@ func (c *AppConfig) setDefaultConfig() {
viper.SetDefault(Validator, "false")
viper.SetDefault(TwitterScraper, "false")
viper.SetDefault(DiscordScraper, "false")
+ viper.SetDefault(TelegramScraper, "false")
viper.SetDefault(WebScraper, "false")
viper.SetDefault(CachePath, "CACHE")
viper.SetDefault(ClaudeApiURL, "https://api.anthropic.com/v1/messages")
@@ -229,6 +230,7 @@ func (c *AppConfig) setDefaultConfig() {
viper.SetDefault(PrivKeyFile, filepath.Join(viper.GetString(MasaDir), "masa_oracle_key"))
viper.SetDefault(TwitterScraper, false)
viper.SetDefault(DiscordScraper, false)
+ viper.SetDefault(TelegramScraper, false)
viper.SetDefault(WebScraper, false)
viper.SetDefault(LlmServer, false)
}
@@ -292,6 +294,7 @@ func (c *AppConfig) setCommandLineConfig() error {
pflag.StringVar(&c.GPTApiKey, "gptApiKey", viper.GetString(GPTApiKey), "OpenAI API Key")
pflag.BoolVar(&c.TwitterScraper, "twitterScraper", viper.GetBool(TwitterScraper), "TwitterScraper")
pflag.BoolVar(&c.DiscordScraper, "discordScraper", viper.GetBool(DiscordScraper), "DiscordScraper")
+ pflag.BoolVar(&c.TelegramScraper, "telegramScraper", viper.GetBool(TelegramScraper), "TelegramScraper")
pflag.BoolVar(&c.WebScraper, "webScraper", viper.GetBool(WebScraper), "WebScraper")
pflag.StringVar(&c.LLMChatUrl, "llmChatUrl", viper.GetString(LlmChatUrl), "URL for support LLM Chat calls")
pflag.StringVar(&c.LLMCfUrl, "llmCfUrl", viper.GetString(LlmCfUrl), "URL for support LLM Cloudflare calls")
diff --git a/pkg/oracle_node.go b/pkg/oracle_node.go
index 50dde539..d988158e 100644
--- a/pkg/oracle_node.go
+++ b/pkg/oracle_node.go
@@ -314,7 +314,7 @@ func (node *OracleNode) handleStream(stream network.Stream) {
// IsWorker determines if the OracleNode is configured to act as an actor.
// An actor node is one that has at least one of the following scrapers enabled:
-// TwitterScraper, DiscordScraper, or WebScraper.
+// TwitterScraper, DiscordScraper, TelegramScraper, or WebScraper.
// It returns true if any of these scrapers are enabled, otherwise false.
func (node *OracleNode) IsWorker() bool {
// need to get this by node data
From 332fff757b2118ad105f0146408a5220ddcc47e2 Mon Sep 17 00:00:00 2001
From: Nolan Jacobson <50815660+nolanjacobson@users.noreply.github.com>
Date: Wed, 24 Jul 2024 21:14:49 -0400
Subject: [PATCH 13/18] chore: fixed merge conflicts with test
---
go.sum | 2 ++
pkg/config/app.go | 78 -----------------------------------------
pkg/oracle_node.go | 29 +++++----------
pkg/pubsub/node_data.go | 39 +++++++--------------
4 files changed, 23 insertions(+), 125 deletions(-)
diff --git a/go.sum b/go.sum
index b183f44c..43892d4e 100644
--- a/go.sum
+++ b/go.sum
@@ -671,6 +671,8 @@ github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca h1:NugYot0LIVPxT
github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU=
github.com/sashabaranov/go-openai v1.27.0 h1:L3hO6650YUbKrbGUC6yCjsUluhKZ9h1/jcgbTItI8Mo=
github.com/sashabaranov/go-openai v1.27.0/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg=
+github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys=
+github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs=
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI=
github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
diff --git a/pkg/config/app.go b/pkg/config/app.go
index 43f3593b..ff9fe7b5 100644
--- a/pkg/config/app.go
+++ b/pkg/config/app.go
@@ -140,79 +140,6 @@ func (c *AppConfig) setDefaultConfig() {
// Set defaults
viper.SetDefault(MasaDir, filepath.Join(usr.HomeDir, ".masa"))
- // Set values from .env
- _, b, _, _ := runtime.Caller(0)
- rootDir := filepath.Join(filepath.Dir(b), "../..")
- if _, _ = os.Stat(rootDir + "/.env"); !os.IsNotExist(err) {
- _ = godotenv.Load()
-
- // Fetch bootnodes from s3
- if os.Getenv("BOOTNODES") != "" {
- var url string
- if os.Getenv("ENV") == "dev" {
- url = "https://masa-oracle-init-dev.s3.amazonaws.com/node_init.json"
- } else if os.Getenv("ENV") == "test" {
- url = "https://masa-oracle-init-test.s3.amazonaws.com/node_init.json"
- } else if os.Getenv("ENV") == "main" {
- url = "https://masa-oracle-init-main.s3.amazonaws.com/node_init.json"
- }
- if url != "" {
- resp, err := http.Get(url)
- if err != nil {
- logrus.Errorf("Failed to fetch %s: %v", url, err)
- } else {
- defer func(Body io.ReadCloser) {
- err := Body.Close()
- if err != nil {
- logrus.Errorf("Failed to close body: %v", err)
- }
- }(resp.Body)
- var nodeInitData struct {
- Name string `json:"name"`
- Id string `json:"id"`
- NodeType string `json:"nodeType"`
- BootNodes []string `json:"bootNodes"`
- }
- if err = json.NewDecoder(resp.Body).Decode(&nodeInitData); err != nil {
- logrus.Errorf("Failed to parse: %v", err)
- } else {
- viper.SetDefault("Bootnodes", strings.Join(nodeInitData.BootNodes, ","))
- }
- }
- } else {
- viper.SetDefault("Bootnodes", os.Getenv("BOOTNODES"))
- }
- }
- viper.SetDefault(RpcUrl, os.Getenv("RPC_URL"))
- viper.SetDefault(Environment, os.Getenv("ENV"))
- viper.SetDefault(FilePath, os.Getenv("FILE_PATH"))
- viper.SetDefault(Validator, os.Getenv("VALIDATOR"))
- viper.SetDefault(CachePath, os.Getenv("CACHE_PATH"))
- viper.SetDefault(TwitterUsername, os.Getenv("TWITTER_USER"))
- viper.SetDefault(TwitterPassword, os.Getenv("TWITTER_PASS"))
- viper.SetDefault(DiscordBotToken, os.Getenv("DISCORD_BOT_TOKEN"))
- viper.SetDefault(ClaudeApiKey, os.Getenv("CLAUDE_API_KEY"))
- viper.SetDefault(ClaudeApiURL, os.Getenv("CLAUDE_API_URL"))
- viper.SetDefault(ClaudeApiVersion, os.Getenv("CLAUDE_API_VERSION"))
- viper.SetDefault(GPTApiKey, os.Getenv("OPENAI_API_KEY"))
- viper.SetDefault(LlmChatUrl, os.Getenv(LlmChatUrl))
- viper.SetDefault(LlmCfUrl, os.Getenv(LlmCfUrl))
-
- } else {
- viper.SetDefault(FilePath, ".")
- viper.SetDefault(RpcUrl, "https://ethereum-sepolia.publicnode.com")
- viper.SetDefault(Validator, "false")
- viper.SetDefault(TwitterScraper, "false")
- viper.SetDefault(DiscordScraper, "false")
- viper.SetDefault(TelegramScraper, "false")
- viper.SetDefault(WebScraper, "false")
- viper.SetDefault(CachePath, "CACHE")
- viper.SetDefault(ClaudeApiURL, "https://api.anthropic.com/v1/messages")
- viper.SetDefault(ClaudeApiVersion, "2023-06-01")
- viper.SetDefault(LlmChatUrl, "http://localhost:11434/api/chat")
- viper.SetDefault(LlmCfUrl, "https://gateway.ai.cloudflare.com/v1/a72433aa3bb83aecaca1bc8acecdb166/masa/workers-ai/")
- }
-
// Set defaults
viper.SetDefault(PortNbr, "4001")
viper.SetDefault(UDP, true)
@@ -223,11 +150,6 @@ func (c *AppConfig) setDefaultConfig() {
viper.SetDefault(LogLevel, "info")
viper.SetDefault(LogFilePath, "masa_node.log")
viper.SetDefault(PrivKeyFile, filepath.Join(viper.GetString(MasaDir), "masa_oracle_key"))
- viper.SetDefault(TwitterScraper, false)
- viper.SetDefault(DiscordScraper, false)
- viper.SetDefault(TelegramScraper, false)
- viper.SetDefault(WebScraper, false)
- viper.SetDefault(LlmServer, false)
}
// setFileConfig loads configuration from a YAML file.
diff --git a/pkg/oracle_node.go b/pkg/oracle_node.go
index d4d23f52..bedd7219 100644
--- a/pkg/oracle_node.go
+++ b/pkg/oracle_node.go
@@ -64,7 +64,7 @@ type OracleNode struct {
IsLlmServer bool
StartTime time.Time
WorkerTracker *pubsub2.WorkerEventTracker
- BlockTracker *pubsub2.BlockEventTracker
+ BlockTracker *BlockEventTracker
ActorEngine *actor.RootContext
ActorRemote *remote.Remote
Blockchain *chain.Chain
@@ -149,13 +149,6 @@ func NewOracleNode(ctx context.Context, isStaked bool) (*OracleNode, error) {
return nil, err
}
- isValidator, _ := strconv.ParseBool(cfg.Validator)
- isTwitterScraper := cfg.TwitterScraper
- isDiscordScraper := cfg.DiscordScraper
- isTelegramScraper := cfg.TelegramScraper
-
- isWebScraper := cfg.WebScraper
-
system := actor.NewActorSystemWithConfig(actor.Configure(
actor.ConfigOption(func(config *actor.Config) {
config.LoggerFactory = func(system *actor.ActorSystem) *slog.Logger {
@@ -187,11 +180,11 @@ func NewOracleNode(ctx context.Context, isStaked bool) (*OracleNode, error) {
NodeTracker: pubsub2.NewNodeEventTracker(config.Version, cfg.Environment),
PubSubManager: subscriptionManager,
IsStaked: isStaked,
- IsValidator: isValidator,
- IsTwitterScraper: isTwitterScraper,
- IsDiscordScraper: isDiscordScraper,
- IsTelegramScraper: isTelegramScraper,
- IsWebScraper: isWebScraper,
+ IsValidator: cfg.Validator,
+ IsTwitterScraper: cfg.TwitterScraper,
+ IsDiscordScraper: cfg.DiscordScraper,
+ IsTelegramScraper: cfg.TelegramScraper,
+ IsWebScraper: cfg.WebScraper,
IsLlmServer: cfg.LlmServer,
ActorEngine: engine,
ActorRemote: r,
@@ -240,18 +233,12 @@ func (node *OracleNode) Start() (err error) {
nodeData.IsStaked = node.IsStaked
nodeData.SelfIdentified = true
nodeData.IsDiscordScraper = node.IsDiscordScraper
+ nodeData.IsTelegramScraper = node.IsTelegramScraper
nodeData.IsTwitterScraper = node.IsTwitterScraper
nodeData.IsWebScraper = node.IsWebScraper
nodeData.IsValidator = node.IsValidator
}
- cfg := config.GetInstance()
- nodeData.IsDiscordScraper = cfg.DiscordScraper
- nodeData.IsTelegramScraper = cfg.TelegramScraper
- nodeData.IsTwitterScraper = cfg.TwitterScraper
- nodeData.IsWebScraper = cfg.WebScraper
- nodeData.IsValidator = cfg.Validator == "true"
-
nodeData.Joined()
node.NodeTracker.HandleNodeData(*nodeData)
@@ -324,7 +311,7 @@ func (node *OracleNode) handleStream(stream network.Stream) {
// IsWorker determines if the OracleNode is configured to act as an actor.
// An actor node is one that has at least one of the following scrapers enabled:
-// TwitterScraper, DiscordScraper, TelegramScraper, or WebScraper.
+// TwitterScraper, DiscordScraper, or WebScraper.
// It returns true if any of these scrapers are enabled, otherwise false.
func (node *OracleNode) IsWorker() bool {
// need to get this by node data
diff --git a/pkg/pubsub/node_data.go b/pkg/pubsub/node_data.go
index 4e25b888..cfd9c8a9 100644
--- a/pkg/pubsub/node_data.go
+++ b/pkg/pubsub/node_data.go
@@ -73,13 +73,7 @@ type NodeData struct {
func NewNodeData(addr multiaddr.Multiaddr, peerId peer.ID, publicKey string, activity int) *NodeData {
multiaddrs := make([]JSONMultiaddr, 0)
multiaddrs = append(multiaddrs, JSONMultiaddr{addr})
- cfg := config.GetInstance()
- wn, _ := strconv.ParseBool(cfg.Validator)
- ts := cfg.TwitterScraper
- ds := cfg.DiscordScraper
- tgs := cfg.TelegramScraper
- ws := cfg.WebScraper
- ver := cfg.Version
+ // cfg := config.GetInstance()
return &NodeData{
PeerId: peerId,
@@ -90,13 +84,6 @@ func NewNodeData(addr multiaddr.Multiaddr, peerId peer.ID, publicKey string, act
EthAddress: publicKey,
Activity: activity,
SelfIdentified: false,
- IsValidator: wn,
- IsTwitterScraper: ts,
- IsDiscordScraper: ds,
- IsTelegramScraper: tgs,
- IsWebScraper: ws,
- BytesScraped: 0,
- Version: ver,
}
}
@@ -119,11 +106,10 @@ func (n *NodeData) DiscordScraper() bool {
return n.IsDiscordScraper
}
-// TelegramScraper checks if the current node is configured as a Telegram scraper.
+// TelegramScraper checks if the current node is configured as a Discord scraper.
// It retrieves the configuration instance and returns the value of the TelegramScraper field.
func (n *NodeData) TelegramScraper() bool {
- cfg := config.GetInstance()
- return cfg.TelegramScraper
+ return n.IsTelegramScraper
}
// WebScraper checks if the current node is configured as a Web scraper.
@@ -229,15 +215,16 @@ func (n *NodeData) UpdateAccumulatedUptime() {
func GetSelfNodeDataJson(host host.Host, isStaked bool) []byte {
// Create and populate NodeData
nodeData := NodeData{
- PeerId: host.ID(),
- IsStaked: isStaked,
- EthAddress: masacrypto.KeyManagerInstance().EthAddress,
- IsTwitterScraper: config.GetInstance().TwitterScraper,
- IsDiscordScraper: config.GetInstance().DiscordScraper,
- IsWebScraper: config.GetInstance().WebScraper,
- IsValidator: config.GetInstance().Validator,
- IsActive: true,
- Version: config.Version,
+ PeerId: host.ID(),
+ IsStaked: isStaked,
+ EthAddress: masacrypto.KeyManagerInstance().EthAddress,
+ IsTwitterScraper: config.GetInstance().TwitterScraper,
+ IsDiscordScraper: config.GetInstance().DiscordScraper,
+ IsTelegramScraper: config.GetInstance().TelegramScraper,
+ IsWebScraper: config.GetInstance().WebScraper,
+ IsValidator: config.GetInstance().Validator,
+ IsActive: true,
+ Version: config.Version,
}
// Convert NodeData to JSON
From d8b1101ce5943ed431bdcc30110ff142cb3891f4 Mon Sep 17 00:00:00 2001
From: Nolan Jacobson <50815660+nolanjacobson@users.noreply.github.com>
Date: Wed, 24 Jul 2024 21:53:48 -0400
Subject: [PATCH 14/18] chore: update teelgram messages handler
---
pkg/api/handlers_data.go | 6 +-----
pkg/scrapers/telegram/telegram_client.go | 4 ++++
2 files changed, 5 insertions(+), 5 deletions(-)
diff --git a/pkg/api/handlers_data.go b/pkg/api/handlers_data.go
index fe3da8b1..60255448 100644
--- a/pkg/api/handlers_data.go
+++ b/pkg/api/handlers_data.go
@@ -11,8 +11,8 @@ import (
"os"
"sync"
- "github.com/masa-finance/masa-oracle/pkg/scrapers/telegram"
"github.com/masa-finance/masa-oracle/pkg/chain"
+ "github.com/masa-finance/masa-oracle/pkg/scrapers/telegram"
"github.com/masa-finance/masa-oracle/pkg/workers"
"github.com/multiformats/go-multiaddr"
"github.com/sirupsen/logrus"
@@ -827,10 +827,6 @@ func (api *API) GetChannelMessagesHandler() gin.HandlerFunc {
var reqBody struct {
Username string `json:"username"` // Telegram usernames are used instead of channel IDs
}
- if err := c.ShouldBindJSON(&reqBody); err != nil {
- c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid request body"})
- return
- }
if reqBody.Username == "" {
c.JSON(http.StatusBadRequest, gin.H{"error": "Username parameter is missing"})
diff --git a/pkg/scrapers/telegram/telegram_client.go b/pkg/scrapers/telegram/telegram_client.go
index c0a7699c..28040790 100644
--- a/pkg/scrapers/telegram/telegram_client.go
+++ b/pkg/scrapers/telegram/telegram_client.go
@@ -8,6 +8,7 @@ import (
"os"
"path/filepath"
"sync"
+ "time"
"github.com/gotd/td/session"
"github.com/gotd/td/telegram"
@@ -112,6 +113,9 @@ func CompleteAuthentication(ctx context.Context, phoneNumber, code, phoneCodeHas
}
}
+ ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+ defer cancel() // It's important to call cancel to release resources if the operation completes before the timeout
+
// Define a variable to hold the authentication result
var authResult *tg.AuthAuthorization
// Use client.Run to start the client and execute the SignIn method
From 8572ebc54052fe0243eed6e3b7fa90d4fa34473e Mon Sep 17 00:00:00 2001
From: Nolan Jacobson <50815660+nolanjacobson@users.noreply.github.com>
Date: Thu, 25 Jul 2024 11:10:41 -0400
Subject: [PATCH 15/18] chore: fix context issues
---
go.mod | 1 +
go.sum | 2 +
pkg/api/handlers_data.go | 10 +----
pkg/scrapers/telegram/getchannelmessages.go | 17 +++------
pkg/scrapers/telegram/telegram_client.go | 42 ++++++++++++++-------
pkg/workers/handler.go | 9 +++--
6 files changed, 43 insertions(+), 38 deletions(-)
diff --git a/go.mod b/go.mod
index b458e5f6..12280415 100644
--- a/go.mod
+++ b/go.mod
@@ -18,6 +18,7 @@ require (
github.com/gocolly/colly/v2 v2.1.0
github.com/golang-jwt/jwt/v4 v4.5.0
github.com/google/uuid v1.6.0
+ github.com/gotd/contrib v0.20.0
github.com/gotd/td v0.105.0
github.com/ipfs/go-cid v0.4.1
github.com/ipfs/go-datastore v0.6.0
diff --git a/go.sum b/go.sum
index 43892d4e..a35134b1 100644
--- a/go.sum
+++ b/go.sum
@@ -314,6 +314,8 @@ github.com/gopherjs/gopherjs v0.0.0-20190812055157-5d271430af9f h1:KMlcu9X58lhTA
github.com/gopherjs/gopherjs v0.0.0-20190812055157-5d271430af9f/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY=
github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY=
+github.com/gotd/contrib v0.20.0 h1:1Wc4+HMQiIKYQuGHVwVksIx152HFTP6B5n88dDe0ZYw=
+github.com/gotd/contrib v0.20.0/go.mod h1:P6o8W4niqhDPHLA0U+SA/L7l3BQHYLULpeHfRSePn9o=
github.com/gotd/ige v0.2.2 h1:XQ9dJZwBfDnOGSTxKXBGP4gMud3Qku2ekScRjDWWfEk=
github.com/gotd/ige v0.2.2/go.mod h1:tuCRb+Y5Y3eNTo3ypIfNpQ4MFjrnONiL2jN2AKZXmb0=
github.com/gotd/neo v0.1.5 h1:oj0iQfMbGClP8xI59x7fE/uHoTJD7NZH9oV1WNuPukQ=
diff --git a/pkg/api/handlers_data.go b/pkg/api/handlers_data.go
index 60255448..0a9208c2 100644
--- a/pkg/api/handlers_data.go
+++ b/pkg/api/handlers_data.go
@@ -778,13 +778,7 @@ func (api *API) StartAuth() gin.HandlerFunc {
return
}
- // client, err := telegram.InitializeClient()
- // if err != nil {
- // c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to initialize Telegram client"})
- // return
- // }
-
- phoneCodeHash, err := telegram.StartAuthentication(c.Request.Context(), reqBody.PhoneNumber)
+ phoneCodeHash, err := telegram.StartAuthentication(reqBody.PhoneNumber)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to start authentication"})
return
@@ -807,7 +801,7 @@ func (api *API) CompleteAuth() gin.HandlerFunc {
return
}
- auth, err := telegram.CompleteAuthentication(c.Request.Context(), reqBody.PhoneNumber, reqBody.Code, reqBody.PhoneCodeHash)
+ auth, err := telegram.CompleteAuthentication(reqBody.PhoneNumber, reqBody.Code, reqBody.PhoneCodeHash)
if err != nil {
// Check if 2FA is required
if err.Error() == "2FA required" {
diff --git a/pkg/scrapers/telegram/getchannelmessages.go b/pkg/scrapers/telegram/getchannelmessages.go
index bcc21ea8..0558b3ee 100644
--- a/pkg/scrapers/telegram/getchannelmessages.go
+++ b/pkg/scrapers/telegram/getchannelmessages.go
@@ -4,16 +4,15 @@ import (
"context"
"fmt"
"log"
- "time"
"github.com/gotd/td/tg"
"github.com/masa-finance/masa-oracle/pkg/llmbridge"
)
// Fetch messages from a group
-func FetchChannelMessages(ctx context.Context, username string) ([]*tg.Message, error) {
+func FetchChannelMessages(username string) ([]*tg.Message, error) {
// Initialize the Telegram client (if not already initialized)
- client = GetClient()
+ // client = GetClient()
if client == nil {
var err error
@@ -26,10 +25,7 @@ func FetchChannelMessages(ctx context.Context, username string) ([]*tg.Message,
var messagesSlice []*tg.Message // Define a slice to hold the messages
- // ctx, cancel := context.WithTimeout(context.Background(), time.Minute*5)
- // defer cancel() // Make sure to cancel when you are done to free resources.
-
- err := client.Run(ctx, func(ctx context.Context) error {
+ err := client.Run(clientCtx, func(ctx context.Context) error {
resolved, err := client.API().ContactsResolveUsername(ctx, username)
if err != nil {
return err
@@ -77,12 +73,9 @@ func FetchChannelMessages(ctx context.Context, username string) ([]*tg.Message,
}
// ScrapeTelegramMessagesForSentiment scrapes messages from a Telegram channel and analyzes their sentiment.
-func ScrapeTelegramMessagesForSentiment(ctx context.Context, username string, model string, prompt string) (string, string, error) {
- // Create a context with a timeout if needed
- ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
- defer cancel()
+func ScrapeTelegramMessagesForSentiment(username string, model string, prompt string) (string, string, error) {
// Fetch messages from the Telegram channel
- messages, err := FetchChannelMessages(ctx, username)
+ messages, err := FetchChannelMessages(username)
if err != nil {
return "", "", fmt.Errorf("error fetching messages from Telegram channel: %v", err)
}
diff --git a/pkg/scrapers/telegram/telegram_client.go b/pkg/scrapers/telegram/telegram_client.go
index 28040790..852e5aed 100644
--- a/pkg/scrapers/telegram/telegram_client.go
+++ b/pkg/scrapers/telegram/telegram_client.go
@@ -8,8 +8,8 @@ import (
"os"
"path/filepath"
"sync"
- "time"
+ "github.com/gotd/contrib/bg"
"github.com/gotd/td/session"
"github.com/gotd/td/telegram"
"github.com/gotd/td/telegram/auth"
@@ -17,11 +17,12 @@ import (
)
var (
- client *telegram.Client
- once sync.Once
- appID = 28423325 // Your actual app ID
- appHash = "c60c0a268973ea3f7d52e16e4ab2a0d3" // Your actual app hash
- sessionDir = filepath.Join(os.Getenv("HOME"), ".telegram-sessions")
+ client *telegram.Client
+ once sync.Once
+ appID = 28423325 // Your actual app ID
+ appHash = "c60c0a268973ea3f7d52e16e4ab2a0d3" // Your actual app hash
+ sessionDir = filepath.Join(os.Getenv("HOME"), ".telegram-sessions")
+ clientCtx, cancelClient = context.WithCancel(context.Background())
)
func InitializeClient() (*telegram.Client, error) {
@@ -53,7 +54,7 @@ func InitializeClient() (*telegram.Client, error) {
}
// StartAuthentication sends the phone number to Telegram and requests a code.
-func StartAuthentication(ctx context.Context, phoneNumber string) (string, error) {
+func StartAuthentication(phoneNumber string) (string, error) {
// Initialize the Telegram client (if not already initialized)
client, err := InitializeClient()
if err != nil {
@@ -65,7 +66,7 @@ func StartAuthentication(ctx context.Context, phoneNumber string) (string, error
var phoneCodeHash string
// Use client.Run to start the client and execute the SendCode method
- err = client.Run(ctx, func(ctx context.Context) error {
+ err = client.Run(clientCtx, func(ctx context.Context) error {
// Call the SendCode method of the client to send the code to the user's Telegram app
sentCode, err := client.Auth().SendCode(ctx, phoneNumber, auth.SendCodeOptions{
AllowFlashCall: true,
@@ -100,26 +101,23 @@ func StartAuthentication(ctx context.Context, phoneNumber string) (string, error
}
// CompleteAuthentication uses the provided code to authenticate with Telegram.
-func CompleteAuthentication(ctx context.Context, phoneNumber, code, phoneCodeHash string) (*tg.AuthAuthorization, error) {
+func CompleteAuthentication(phoneNumber, code, phoneCodeHash string) (*tg.AuthAuthorization, error) {
// Initialize the Telegram client (if not already initialized)
client = GetClient()
if client == nil {
var err error
client, err = InitializeClient()
+ log.Printf("Initializing client again")
if err != nil {
log.Printf("Failed to initialize Telegram client: %v", err)
return nil, err
}
}
-
- ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
- defer cancel() // It's important to call cancel to release resources if the operation completes before the timeout
-
// Define a variable to hold the authentication result
var authResult *tg.AuthAuthorization
// Use client.Run to start the client and execute the SignIn method
- err := client.Run(ctx, func(ctx context.Context) error {
+ err := client.Run(clientCtx, func(ctx context.Context) error {
// Use the provided code and phoneCodeHash to authenticate
auth, err := client.Auth().SignIn(ctx, phoneNumber, code, phoneCodeHash)
if err != nil {
@@ -129,6 +127,17 @@ func CompleteAuthentication(ctx context.Context, phoneNumber, code, phoneCodeHas
// At this point, authentication was successful, and you have the user's Telegram auth data.
authResult = auth
+ stop, err := bg.Connect(client)
+ if err != nil {
+ return err
+ }
+ defer func() { _ = stop() }()
+
+ // Now you can use client.
+ if _, err := client.Auth().Status(ctx); err != nil {
+ return err
+ }
+
return nil
})
@@ -142,6 +151,11 @@ func CompleteAuthentication(ctx context.Context, phoneNumber, code, phoneCodeHas
return authResult, nil
}
+// Call this function when your application is shutting down
+func shutdownClient() {
+ cancelClient()
+}
+
func GetClient() *telegram.Client {
return client
}
diff --git a/pkg/workers/handler.go b/pkg/workers/handler.go
index 02b4ca3c..ce61d169 100644
--- a/pkg/workers/handler.go
+++ b/pkg/workers/handler.go
@@ -1,7 +1,6 @@
package workers
import (
- "context"
"encoding/json"
"fmt"
"strings"
@@ -89,8 +88,6 @@ func (a *Worker) HandleWork(ctx actor.Context, m *messages.Work, node *masa.Orac
}
}
- opCtx := context.Background() // or ctx.Context if actor.Context provides it
-
switch workData["request"] {
case string(WORKER.DiscordProfile):
userID := bodyData["userID"].(string)
@@ -102,10 +99,14 @@ func (a *Worker) HandleWork(ctx actor.Context, m *messages.Work, node *masa.Orac
logrus.Infof("[+] Discord Channel Messages %s %s", m.Data, m.Sender)
channelID := bodyData["channelID"].(string)
_, resp, err = discord.ScrapeDiscordMessagesForSentiment(channelID, bodyData["model"].(string), bodyData["prompt"].(string))
+ case string(WORKER.TelegramChannelMessages):
+ logrus.Infof("[+] Telegram Channel Messages %s %s", m.Data, m.Sender)
+ username := bodyData["username"].(string)
+ resp, err = telegram.FetchChannelMessages(username) // Removed the underscore placeholder
case string(WORKER.TelegramSentiment):
logrus.Infof("[+] Telegram Channel Messages %s %s", m.Data, m.Sender)
username := bodyData["username"].(string)
- _, resp, err = telegram.ScrapeTelegramMessagesForSentiment(opCtx, username, bodyData["model"].(string), bodyData["prompt"].(string))
+ _, resp, err = telegram.ScrapeTelegramMessagesForSentiment(username, bodyData["model"].(string), bodyData["prompt"].(string))
case string(WORKER.DiscordGuildChannels):
guildID := bodyData["guildID"].(string)
resp, err = discord.GetGuildChannels(guildID)
From ac84b845fde227fac5794b178ba242186e65ba1a Mon Sep 17 00:00:00 2001
From: Nolan Jacobson <50815660+nolanjacobson@users.noreply.github.com>
Date: Thu, 25 Jul 2024 11:20:17 -0400
Subject: [PATCH 16/18] chore: minor fixes
---
docs/docs.go | 21 +++++++++++++++------
pkg/api/routes.go | 3 +--
pkg/scrapers/telegram/getchannelmessages.go | 4 ++--
pkg/scrapers/telegram/telegram_client.go | 4 ++--
4 files changed, 20 insertions(+), 12 deletions(-)
diff --git a/docs/docs.go b/docs/docs.go
index e2dcff04..67b189bc 100644
--- a/docs/docs.go
+++ b/docs/docs.go
@@ -474,18 +474,27 @@ const docTemplate = `{
]
}
},
- "/data/telegram/channel/{username}/messages": {
- "get": {
+ "/data/telegram/channel/messages": {
+ "post": { // Changed from "get" to "post" to allow body parameters
"description": "Retrieves messages from a specified Telegram channel.",
"tags": ["Telegram"],
"summary": "Get Telegram Channel Messages",
"parameters": [
{
- "name": "username",
- "in": "path",
- "description": "Telegram Username",
+ "in": "body",
+ "name": "body",
+ "description": "Request body",
"required": true,
- "type": "string"
+ "schema": {
+ "type": "object",
+ "properties": {
+ "username": {
+ "type": "string",
+ "description": "Telegram Username"
+ }
+ },
+ "required": ["username"]
+ }
}
],
"responses": {
diff --git a/pkg/api/routes.go b/pkg/api/routes.go
index 1b8cf55c..7e6d18f1 100644
--- a/pkg/api/routes.go
+++ b/pkg/api/routes.go
@@ -247,12 +247,11 @@ func SetupRoutes(node *masa.OracleNode) *gin.Engine {
// @Tags Telegram
// @Accept json
// @Produce json
- // @Param username path string true "Telegram Username"
// @Success 200 {object} map[string][]Message "Successfully retrieved messages"
// @Failure 400 {object} ErrorResponse "Username must be provided"
// @Failure 500 {object} ErrorResponse "Failed to fetch channel messages"
// @Router /telegram/channel/{username}/messages [get]
- v1.GET("/data/telegram/channel/:username/messages", API.GetChannelMessagesHandler())
+ v1.GET("/data/telegram/channel/messages", API.GetChannelMessagesHandler())
// oauth tests
// v1.GET("/data/discord/exchangetoken/:code", API.ExchangeDiscordTokenHandler())
diff --git a/pkg/scrapers/telegram/getchannelmessages.go b/pkg/scrapers/telegram/getchannelmessages.go
index 0558b3ee..c7bce7b2 100644
--- a/pkg/scrapers/telegram/getchannelmessages.go
+++ b/pkg/scrapers/telegram/getchannelmessages.go
@@ -26,7 +26,7 @@ func FetchChannelMessages(username string) ([]*tg.Message, error) {
var messagesSlice []*tg.Message // Define a slice to hold the messages
err := client.Run(clientCtx, func(ctx context.Context) error {
- resolved, err := client.API().ContactsResolveUsername(ctx, username)
+ resolved, err := client.API().ContactsResolveUsername(clientCtx, username)
if err != nil {
return err
}
@@ -42,7 +42,7 @@ func FetchChannelMessages(username string) ([]*tg.Message, error) {
ChannelID: channel.ChannelID,
AccessHash: channel.AccessHash,
}
- result, err := client.API().MessagesGetHistory(ctx, &tg.MessagesGetHistoryRequest{
+ result, err := client.API().MessagesGetHistory(clientCtx, &tg.MessagesGetHistoryRequest{
Peer: inputPeer, // Pass inputPeer here
Limit: 100, // Adjust the number of messages to fetch
})
diff --git a/pkg/scrapers/telegram/telegram_client.go b/pkg/scrapers/telegram/telegram_client.go
index 852e5aed..adc37c56 100644
--- a/pkg/scrapers/telegram/telegram_client.go
+++ b/pkg/scrapers/telegram/telegram_client.go
@@ -68,7 +68,7 @@ func StartAuthentication(phoneNumber string) (string, error) {
// Use client.Run to start the client and execute the SendCode method
err = client.Run(clientCtx, func(ctx context.Context) error {
// Call the SendCode method of the client to send the code to the user's Telegram app
- sentCode, err := client.Auth().SendCode(ctx, phoneNumber, auth.SendCodeOptions{
+ sentCode, err := client.Auth().SendCode(clientCtx, phoneNumber, auth.SendCodeOptions{
AllowFlashCall: true,
CurrentNumber: true,
})
@@ -119,7 +119,7 @@ func CompleteAuthentication(phoneNumber, code, phoneCodeHash string) (*tg.AuthAu
// Use client.Run to start the client and execute the SignIn method
err := client.Run(clientCtx, func(ctx context.Context) error {
// Use the provided code and phoneCodeHash to authenticate
- auth, err := client.Auth().SignIn(ctx, phoneNumber, code, phoneCodeHash)
+ auth, err := client.Auth().SignIn(clientCtx, phoneNumber, code, phoneCodeHash)
if err != nil {
log.Printf("Error during SignIn: %v", err)
return err
From 3f8e77149288796c74ff7b94bdcbc4a0bc0a0cde Mon Sep 17 00:00:00 2001
From: Nolan Jacobson <50815660+nolanjacobson@users.noreply.github.com>
Date: Thu, 25 Jul 2024 11:25:11 -0400
Subject: [PATCH 17/18] chore: fixes
---
pkg/api/routes.go | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pkg/api/routes.go b/pkg/api/routes.go
index 7e6d18f1..141b9eaa 100644
--- a/pkg/api/routes.go
+++ b/pkg/api/routes.go
@@ -251,7 +251,7 @@ func SetupRoutes(node *masa.OracleNode) *gin.Engine {
// @Failure 400 {object} ErrorResponse "Username must be provided"
// @Failure 500 {object} ErrorResponse "Failed to fetch channel messages"
// @Router /telegram/channel/{username}/messages [get]
- v1.GET("/data/telegram/channel/messages", API.GetChannelMessagesHandler())
+ v1.POST("/data/telegram/channel/messages", API.GetChannelMessagesHandler())
// oauth tests
// v1.GET("/data/discord/exchangetoken/:code", API.ExchangeDiscordTokenHandler())
From a231baeac86f47543523ed0afd757c76c00a8f5b Mon Sep 17 00:00:00 2001
From: Nolan Jacobson <50815660+nolanjacobson@users.noreply.github.com>
Date: Thu, 25 Jul 2024 11:56:18 -0400
Subject: [PATCH 18/18] chore: latest with ctx still closing
---
docs/docs.go | 2 +-
pkg/api/handlers_data.go | 6 ++++++
pkg/scrapers/telegram/telegram_client.go | 18 ++++++++++++------
3 files changed, 19 insertions(+), 7 deletions(-)
diff --git a/docs/docs.go b/docs/docs.go
index 67b189bc..f4f8fde6 100644
--- a/docs/docs.go
+++ b/docs/docs.go
@@ -475,7 +475,7 @@ const docTemplate = `{
}
},
"/data/telegram/channel/messages": {
- "post": { // Changed from "get" to "post" to allow body parameters
+ "post": {
"description": "Retrieves messages from a specified Telegram channel.",
"tags": ["Telegram"],
"summary": "Get Telegram Channel Messages",
diff --git a/pkg/api/handlers_data.go b/pkg/api/handlers_data.go
index 0a9208c2..68ed982e 100644
--- a/pkg/api/handlers_data.go
+++ b/pkg/api/handlers_data.go
@@ -822,6 +822,12 @@ func (api *API) GetChannelMessagesHandler() gin.HandlerFunc {
Username string `json:"username"` // Telegram usernames are used instead of channel IDs
}
+ // Bind the JSON body to the struct
+ if err := c.ShouldBindJSON(&reqBody); err != nil {
+ c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid request body"})
+ return
+ }
+
if reqBody.Username == "" {
c.JSON(http.StatusBadRequest, gin.H{"error": "Username parameter is missing"})
return
diff --git a/pkg/scrapers/telegram/telegram_client.go b/pkg/scrapers/telegram/telegram_client.go
index adc37c56..bc225770 100644
--- a/pkg/scrapers/telegram/telegram_client.go
+++ b/pkg/scrapers/telegram/telegram_client.go
@@ -6,8 +6,10 @@ import (
"errors"
"log"
"os"
+ "os/signal"
"path/filepath"
"sync"
+ "syscall"
"github.com/gotd/contrib/bg"
"github.com/gotd/td/session"
@@ -17,12 +19,13 @@ import (
)
var (
- client *telegram.Client
- once sync.Once
- appID = 28423325 // Your actual app ID
- appHash = "c60c0a268973ea3f7d52e16e4ab2a0d3" // Your actual app hash
- sessionDir = filepath.Join(os.Getenv("HOME"), ".telegram-sessions")
- clientCtx, cancelClient = context.WithCancel(context.Background())
+ client *telegram.Client
+ once sync.Once
+ appID = 28423325 // Your actual app ID
+ appHash = "c60c0a268973ea3f7d52e16e4ab2a0d3" // Your actual app hash
+ sessionDir = filepath.Join(os.Getenv("HOME"), ".telegram-sessions")
+ clientCtx context.Context
+ cancelClient context.CancelFunc
)
func InitializeClient() (*telegram.Client, error) {
@@ -44,6 +47,9 @@ func InitializeClient() (*telegram.Client, error) {
return
}
+ clientCtx, cancelClient = signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+ defer cancelClient()
+
// Initialize the Telegram client
client = telegram.NewClient(appID, appHash, telegram.Options{
SessionStorage: storage,