feat: ledger

.env.example

@@ -1,40 +1,50 @@
-# This env is a helper for when testing the masa oracle node locally 
+## note: all nodes need to be staked to participate
 
-# Node Configuration
-BOOTNODES=
+## PEER configuration
+BOOTNODES=/ip4/35.223.224.220/udp/4001/quic-v1/p2p/16Uiu2HAmPxXXjR1XJEwckh6q1UStheMmGaGe8fyXdeRs3SejadSa
+ENV=test
+FILE_PATH=.
+PORT=8080
+RPC_URL=https://ethereum-sepolia.publicnode.com
+
+## VALIDATOR configuration
+BOOTNODES=/ip4/35.223.224.220/udp/4001/quic-v1/p2p/16Uiu2HAmPxXXjR1XJEwckh6q1UStheMmGaGe8fyXdeRs3SejadSa
+ENV=test
+FILE_PATH=.
+PORT=8080
+RPC_URL=https://ethereum-sepolia.publicnode.com
+VALIDATOR=true
 
-API_KEY=
-ENV=
+## ORACLE configuration
+BOOTNODES=/ip4/35.223.224.220/udp/4001/quic-v1/p2p/16Uiu2HAmPxXXjR1XJEwckh6q1UStheMmGaGe8fyXdeRs3SejadSa
+ENV=test
 FILE_PATH=.
 PORT=8080
 RPC_URL=https://ethereum-sepolia.publicnode.com
-VALIDATOR=false
+TWITTER_USER= your twitter username
+TWITTER_PASS= your twitter password
+TWITTER_2FA_CODE= your twitter 2fa code
+DISCORD_SCRAPER=true
+DISCORD_BOT_TOKEN= your discord bot token
+TWITTER_SCRAPER=true
+WEB_SCRAPER=true
+
+## optional added features for ORACLE or VALIDATOR nodes only
 
-# AI LLM
+# Claude API and Elevenlabs API
 CLAUDE_API_KEY=
 CLAUDE_API_URL=
 CLAUDE_API_VERSION=
 ELAB_KEY=
 
 # For use with ollama https://hub.docker.com/r/ollama/ollama
-LLM_SERVER=false
+LLM_SERVER=true
 LLM_CHAT_URL=http://localhost:11434/api/chat
 
-# For use with bring your own Cloudflare worker token
+# Bring your own Cloudflare worker token
 LLM_CF_URL=
 LLM_CF_TOKEN=
 
-# For use with bring your own OpenAI api key
+# Bring your own OpenAI api key
 OPENAI_API_KEY=
 PROMPT="You are a helpful assistant."
-
-# For use with bring your own Twitter credentials
-# TWITTER_USER=
-# TWITTER_PASS=
-# TWITTER_2FA_CODE=
-
-# Worker participation
-# DISCORD_SCRAPER=true
-# DISCORD_BOT_TOKEN=
-# TWITTER_SCRAPER=false
-# WEB_SCRAPER=false

.github/workflows/tests.yaml

@@ -0,0 +1,27 @@
+name: Go
+
+on:
+  push:
+    branches:
+      - '**'
+  pull_request:
+    branches:
+      - '**'
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v2
+
+    - name: Set up Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: '^1.22'
+
+    - name: Install dependencies
+      run: go mod tidy
+
+    - name: Run tests
+      run: go test ./...

.gitignore

@@ -66,4 +66,5 @@ cmd/masa-node-cli/log.txt
 output.mp3
 docs/api-reference.md
 ./masa-node
-transcription.txt
+transcription.txt
+notes.txt

Makefile

@@ -24,7 +24,8 @@ test:
 clean:
 	@rm -rf bin
 	@rm masa_node.log
-
+	@rm -rf ~/.masa/cache
+
 proto:
 	sh pkg/workers/messages/build.sh
 

cmd/masa-node/main.go

@@ -87,9 +87,7 @@ func main() {
 	}
 
 	// Init cache resolver
-	go db.InitResolverCache(node, keyManager)
-	// Subscribe to blocks -- moving to new ticket
-	// go masa.SubscribeToBlocks(ctx, node)
+	db.InitResolverCache(node, keyManager)
 
 	// Subscribe and if actor start monitoring actor workers
 	// considering all that matters is if the node is staked
@@ -98,6 +96,7 @@ func main() {
 	if node.IsStaked {
 		go workers.SubscribeToWorkers(node)
 		go workers.MonitorWorkers(ctx, node)
+		go masa.SubscribeToBlocks(ctx, node)
 	}
 
 	// Listen for SIGINT (CTRL+C)
@@ -111,7 +110,6 @@ func main() {
 		if nodeData != nil {
 			nodeData.Left()
 		}
-		// node.NodeTracker.DumpNodeData()
 		cancel()
 	}()
 

docs/oracle-node/quickstart.md

@@ -86,7 +86,7 @@ Is Validator:              false
 Is TwitterScraper:      false
 Is WebScraper:          false
 INFO[0001] Peer added to DHT: 16Uiu2HAmHpx13GPKZAP3WpgpYkZ39M5cwuvmXS5gGvrsa5ofLNoq 
-INFO[0005] Successfully advertised protocol /masa/oracle_protocol/v0.0.2-beta-dev
+INFO[0005] Successfully advertised protocol /masa/oracle_protocol/v0.0.9-beta-dev
 ```
 
 ### 6. Stake the node with 1000 Sepolia MASA minimum
@@ -139,7 +139,7 @@ Is Validator:              false
 Is TwitterScraper:      false
 Is WebScraper:          false
 INFO[0001] Peer added to DHT: 16Uiu2HAmHpx13GPKZAP3WpgpYkZ39M5cwuvmXS5gGvrsa5ofLNoq 
-INFO[0005] Successfully advertised protocol /masa/oracle_protocol/v0.0.2-beta-dev
+INFO[0005] Successfully advertised protocol /masa/oracle_protocol/v0.0.9-beta-dev
 ```
 
 ### 7. View swagger API

docs/worker-node/discord-worker.md

@@ -38,7 +38,6 @@ To start processing Discord data requests, you need to retrieve your Discord bot
 5 . Confirm the creation of the bot user.
 6 . Under the "TOKEN" section, click on "Copy" to get your bot token.
 
-
 ### Adding Discord Credentials
 
 1 . Locate your `.env` file in your Masa Oracle Node's directory.

docs/worker-node/quickstart.md

@@ -86,7 +86,7 @@ Is Validator:              false
 Is TwitterScraper:      false
 Is WebScraper:          false
 INFO[0001] Peer added to DHT: 16Uiu2HAmHpx13GPKZAP3WpgpYkZ39M5cwuvmXS5gGvrsa5ofLNoq 
-INFO[0005] Successfully advertised protocol /masa/oracle_protocol/v0.0.2-beta-dev
+INFO[0005] Successfully advertised protocol /masa/oracle_protocol/v0.0.9-beta-dev
 ```
 
 ### 6. Stake the node with 1000 Sepolia MASA minimum
@@ -139,7 +139,7 @@ Is Validator:              false
 Is TwitterScraper:      false
 Is WebScraper:          false
 INFO[0001] Peer added to DHT: 16Uiu2HAmHpx13GPKZAP3WpgpYkZ39M5cwuvmXS5gGvrsa5ofLNoq 
-INFO[0005] Successfully advertised protocol /masa/oracle_protocol/v0.0.2-beta-dev
+INFO[0005] Successfully advertised protocol /masa/oracle_protocol/v0.0.9-beta-dev
 ```
 
 ### 7. View swagger API

go.mod

@@ -21,6 +21,7 @@ require (
 	github.com/ipfs/go-cid v0.4.1
 	github.com/ipfs/go-datastore v0.6.0
 	github.com/ipfs/go-ds-leveldb v0.5.0
+	github.com/ipfs/go-ipfs-api v0.7.0
 	github.com/joho/godotenv v1.5.1
 	github.com/lib/pq v1.10.9
 	github.com/libp2p/go-libp2p v0.34.0
@@ -56,6 +57,7 @@ require (
 	github.com/benbjohnson/clock v1.3.5 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bits-and-blooms/bitset v1.13.0 // indirect
+	github.com/blang/semver/v4 v4.0.0 // indirect
 	github.com/btcsuite/btcd/btcec/v2 v2.3.2 // indirect
 	github.com/btcsuite/btcd/chaincfg/chainhash v1.1.0 // indirect
 	github.com/bytedance/sonic v1.11.6 // indirect
@@ -69,6 +71,7 @@ require (
 	github.com/containerd/cgroups v1.1.0 // indirect
 	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
+	github.com/crackcomm/go-gitignore v0.0.0-20231225121904-e25f5bc08668 // indirect
 	github.com/crate-crypto/go-kzg-4844 v1.0.0 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect
@@ -155,6 +158,7 @@ require (
 	github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect
 	github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect
 	github.com/minio/sha256-simd v1.0.1 // indirect
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/mmcloughlin/addchain v0.4.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect

go.sum

@@ -51,6 +51,8 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bits-and-blooms/bitset v1.13.0 h1:bAQ9OPNFYbGHV6Nez0tmNI0RiEu7/hxlYJRUA0wFAVE=
 github.com/bits-and-blooms/bitset v1.13.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
+github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
+github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=
 github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g=
 github.com/btcsuite/btcd/btcec/v2 v2.3.2 h1:5n0X6hX0Zk+6omWcihdYvdAlGf2DfasC0GMf7DClJ3U=
 github.com/btcsuite/btcd/btcec/v2 v2.3.2/go.mod h1:zYzJ8etWJQIv1Ogk7OzpWjowwOdXY1W/17j2MW85J04=
@@ -70,6 +72,8 @@ github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
 github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 h1:SKI1/fuSdodxmNNyVBR8d7X/HuLnRpvvFO0AgyQk764=
+github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927/go.mod h1:h/aW8ynjgkuj+NQRlZcDbAbM1ORAbXjXX77sX7T289U=
 github.com/chyeh/pubip v0.0.0-20170203095919-b7e679cf541c h1:++BhWlmSX+n8m3O4gPfy3S4PTZ0TMzH6nelerBLPUng=
 github.com/chyeh/pubip v0.0.0-20170203095919-b7e679cf541c/go.mod h1:C7ma6h458jTWT65mXC58L1Q6hnEtr0unur8cMc0UEXM=
 github.com/cilium/ebpf v0.2.0/go.mod h1:To2CFviqOWL/M0gIMsvSMlqe7em/l1ALkX1PyjrX2Qs=
@@ -110,6 +114,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:ma
 github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4=
 github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/crackcomm/go-gitignore v0.0.0-20231225121904-e25f5bc08668 h1:ZFUue+PNxmHlu7pYv+IYMtqlaO/0VwaGEqKepZf9JpA=
+github.com/crackcomm/go-gitignore v0.0.0-20231225121904-e25f5bc08668/go.mod h1:p1d6YEZWvFzEh4KLyvBcVSnrfNDDvK2zfK/4x2v/4pE=
 github.com/crate-crypto/go-ipa v0.0.0-20240223125850-b1e8a79f509c h1:uQYC5Z1mdLRPrZhHjHxufI8+2UG/i25QG92j0Er9p6I=
 github.com/crate-crypto/go-ipa v0.0.0-20240223125850-b1e8a79f509c/go.mod h1:geZJZH3SzKCqnz5VT0q/DyIG/tvu/dZk+VIfXicupJs=
 github.com/crate-crypto/go-kzg-4844 v1.0.0 h1:TsSgHwrkTKecKJ4kadtHi4b3xHW5dCFUDFnUp1TsawI=
@@ -335,6 +341,8 @@ github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ
 github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46UU0LZ723meps=
 github.com/ipfs/go-ds-leveldb v0.5.0 h1:s++MEBbD3ZKc9/8/njrn4flZLnCuY9I79v94gBUNumo=
 github.com/ipfs/go-ds-leveldb v0.5.0/go.mod h1:d3XG9RUDzQ6V4SHi8+Xgj9j1XuEk1z82lquxrVbml/Q=
+github.com/ipfs/go-ipfs-api v0.7.0 h1:CMBNCUl0b45coC+lQCXEVpMhwoqjiaCwUIrM+coYW2Q=
+github.com/ipfs/go-ipfs-api v0.7.0/go.mod h1:AIxsTNB0+ZhkqIfTZpdZ0VR/cpX5zrXjATa3prSay3g=
 github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw=
 github.com/ipfs/go-ipfs-util v0.0.3 h1:2RFdGez6bu2ZlZdI+rWfIdbQb1KudQp3VGwPtdNCmE0=
 github.com/ipfs/go-ipfs-util v0.0.3/go.mod h1:LHzG1a0Ig4G+iZ26UUOMjHd+lfM84LZCrn17xAKWBvs=
@@ -472,6 +480,7 @@ github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8Rv
 github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
 github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM=
 github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8=
+github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=

pkg/api/handlers_node.go

@@ -63,7 +63,7 @@ func (api *API) GetNodeDataHandler() gin.HandlerFunc {
 			"success":      true,
 			"data":         nodeDataPage.Data,
 			"pageNbr":      nodeDataPage.PageNumber,
-			"total":        nodeDataPage.TotalRecords,
+			"total":        nodeDataPage.TotalPages,
 			"totalRecords": nodeDataPage.TotalRecords,
 		})
 	}

pkg/api/routes.go

@@ -134,7 +134,7 @@ func SetupRoutes(node *masa.OracleNode) *gin.Engine {
 	//	@Title			Masa API
 	//	@Description	The Worlds Personal Data Network Masa Oracle Node API
 	//	@Host			https://api.masa.ai
-	//	@Version		0.0.9-beta
+	//	@Version		0.5.0
 	//	@contact.name	Masa API Support
 	//	@contact.url	https://masa.ai
 	//	@contact.email	[email protected]

pkg/chain/block.go

@@ -12,8 +12,8 @@ import (
 type Block struct {
 	Data  []byte //	this block's data
 	Hash  []byte //	this block's hash
-	Link  []byte //	the hash of the last block in the chain. this is the key part that links the blocks together
-	Nonce int64  //	the nonce used to sing the block. useful for verification
+	Link  []byte //	the hash of the last block in the chain
+	Nonce int64  //	the nonce used to sign the block for verification
 }
 
 func (b *Block) Build(data []byte, link []byte, stake *big.Int) {

pkg/config/welcome.go

@@ -6,10 +6,8 @@ import (
 
 func DisplayWelcomeMessage(multiAddr, ipAddr, publicKeyHex string, isStaked bool, isValidator bool, isTwitterScraper bool, isDiscordScraper bool, isWebScraper bool, version string) {
 	// ANSI escape code for yellow text
-	green := "\033[32m"
 	yellow := "\033[33m"
 	blue := "\033[34m"
-	// red := "\033[31m"
 	reset := "\033[0m"
 
 	fmt.Println("")
@@ -24,15 +22,15 @@ func DisplayWelcomeMessage(multiAddr, ipAddr, publicKeyHex string, isStaked bool
 	fmt.Println(yellow + "#                                     #" + reset)
 	fmt.Println(yellow + borderLine + reset)
 	fmt.Println("")
-	fmt.Printf(green+"Version:		%s\n"+reset, version)
-	// Displaying the multi-address and IP address in blue
-	fmt.Printf(blue+"Multiaddress:		%s\n"+reset, multiAddr)
-	fmt.Printf(blue+"IP Address:		%s\n"+reset, ipAddr)
-	fmt.Printf(blue+"Public Key:   		%s\n"+reset, publicKeyHex)
-	fmt.Printf(blue+"Is Staked:    		%t\n"+reset, isStaked)
-	fmt.Printf(blue+"Is Validator:    	%t\n"+reset, isValidator)
-	fmt.Printf(blue+"Is TwitterScraper:	%t\n"+reset, isTwitterScraper)
-	fmt.Printf(blue+"Is DiscordScraper:	%t\n"+reset, isDiscordScraper)
-	fmt.Printf(blue+"Is WebScraper:   	%t\n"+reset, isWebScraper)
+
+	fmt.Printf(blue+"%-20s %s\n"+reset, "Version:", yellow+version)
+	fmt.Printf(blue+"%-20s %s\n"+reset, "Multiaddress:", multiAddr)
+	fmt.Printf(blue+"%-20s %s\n"+reset, "IP Address:", ipAddr)
+	fmt.Printf(blue+"%-20s %s\n"+reset, "Public Key:", publicKeyHex)
+	fmt.Printf(blue+"%-20s %t\n"+reset, "Is Staked:", isStaked)
+	fmt.Printf(blue+"%-20s %t\n"+reset, "Is Validator:", isValidator)
+	fmt.Printf(blue+"%-20s %t\n"+reset, "Is TwitterScraper:", isTwitterScraper)
+	fmt.Printf(blue+"%-20s %t\n"+reset, "Is DiscordScraper:", isDiscordScraper)
+	fmt.Printf(blue+"%-20s %t\n"+reset, "Is WebScraper:", isWebScraper)
 	fmt.Println("")
 }

pkg/db/operations.go

@@ -1,9 +1,13 @@
 package db
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
+	"io"
+	"net/http"
+	"os"
 	"time"
 
 	masa "github.com/masa-finance/masa-oracle/pkg"
@@ -97,3 +101,48 @@ func ReadData(node *masa.OracleNode, key string) ([]byte, error) {
 
 	return val, nil
 }
+
+// SendToS3 sends a payload to an S3-compatible API.
+//
+// @param {string} uid - The unique identifier for the payload.
+// @param {map[string]string} payload - The payload to be sent, represented as a map of key-value pairs.
+// @returns {error} - Returns an error if the operation fails, otherwise returns nil.
+func SendToS3(uid string, payload map[string]string) error {
+
+	apiURL := os.Getenv("API_URL")
+	authToken := "your-secret-token"
+
+	// Creating the JSON payload
+	// payload := map[string]string{
+	// 	"key1": "value1",
+	// 	"key2": "value2",
+	// }
+
+	jsonPayload, err := json.Marshal(payload)
+	if err != nil {
+		return fmt.Errorf("failed to marshal JSON payload: %v", err)
+	}
+
+	req, err := http.NewRequest("POST", apiURL, bytes.NewBuffer(jsonPayload))
+	if err != nil {
+		return fmt.Errorf("failed to create HTTP request: %v", err)
+	}
+
+	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("Authorization", authToken)
+
+	// Send the request
+	client := &http.Client{}
+	resp, err := client.Do(req)
+	if err != nil {
+		return fmt.Errorf("failed to send HTTP request: %v", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		bodyBytes, _ := io.ReadAll(resp.Body)
+		return fmt.Errorf("received non-OK response: %s, body: %s", resp.Status, string(bodyBytes))
+	}
+
+	return nil
+}