Squashed commit of the following:

commit e8e1822
Author: Ettore Di Giacinto <[email protected]>
Date:   Fri Aug 23 17:54:21 2024 +0200

    ci: Rename static.yml to static.yml

    Signed-off-by: Ettore Di Giacinto <[email protected]>

commit efb84f9
Author: Ettore Di Giacinto <[email protected]>
Date:   Fri Aug 23 17:53:15 2024 +0200

    feat(tests): add baseline for an integration test (#527)

    * feat(tests): add baseline for an integration test

    Signed-off-by: mudler <[email protected]>

    * chore: drop obsoleted code

    Signed-off-by: mudler <[email protected]>

    * fix: HasBootnodes otherwise panics if no bootnodes are specified

    Signed-off-by: mudler <[email protected]>

    * chore(integration tests): make it possible to run the tests without generating identity keys

    - Introduce GetP2PMultiAddrs() to retrieve the node id multiaddress which
    is having p2p encapsulation, in order to be used as bootnode
    - Wire up some changes into the constructor of the oracle to allow to
      specify bootnodes and enable random identity generation

    Signed-off-by: mudler <[email protected]>

    * chore: drop unneeded code

    Signed-off-by: mudler <[email protected]>

    ---------

    Signed-off-by: mudler <[email protected]>

commit 6dae3fe
Author: Brendan Playford <[email protected]>
Date:   Mon Aug 19 14:01:08 2024 -0700

    hotfix: .env formatting and description plus runpod and Docker image

commit 5e5853e
Author: Ettore Di Giacinto <[email protected]>
Date:   Mon Aug 19 11:03:16 2024 +0200

    fix(ci): point correctly to main in `goreleaser.yml` (#516)

    * fix(ci): point correctly to main

    Otherwise goreleaser will fail building.

    Fixes: #514

    Signed-off-by: mudler <[email protected]>

    * detect goreleaser

    Signed-off-by: mudler <[email protected]>

    ---------

    Signed-off-by: mudler <[email protected]>

.env.example

@@ -1,61 +1,49 @@
-## note: all nodes need to be staked to participate
-
-## BOOTNODE configuration
-ENV=test
-FILE_PATH=.
-PORT=8080
-RPC_URL=https://ethereum-sepolia.publicnode.com
-
-## ORACLE without SCRAPER configuration
+## Note: All nodes need to be staked to participate
+## To stake your node:
+## 1. Ensure you have Sepolia ETH from a public faucet
+## 2. Ensure you have Sepolia MASA tokens run make faucet to get 1000 Sepolia MASA
+## 3. Start your node and copy the Public Key from the startup logs
+## 4. Use the following commands: make stake or ./masa-node --stake <amount> (e.g., ./masa-node --stake 1000 for 1000 MASA)
+## 5. Wait for the transaction to be confirmed on the blockchain
+## 6. Restart your node after staking
+## Minimum stake required: 1000 Sepolia MASA
+
+## Minimum .env configuration for running a staked node and getting data from the network
+## With this configuration, you can retrieve data but cannot provide data as a worker
 BOOTNODES=/ip4/35.223.224.220/udp/4001/quic-v1/p2p/16Uiu2HAmPxXXjR1XJEwckh6q1UStheMmGaGe8fyXdeRs3SejadSa
 ENV=test
 FILE_PATH=.
 PORT=8080
 RPC_URL=https://ethereum-sepolia.publicnode.com
 
-## VALIDATOR configuration
-BOOTNODES=/ip4/35.223.224.220/udp/4001/quic-v1/p2p/16Uiu2HAmPxXXjR1XJEwckh6q1UStheMmGaGe8fyXdeRs3SejadSa
-ENV=test
-FILE_PATH=.
-PORT=8080
-RPC_URL=https://ethereum-sepolia.publicnode.com
-VALIDATOR=true
 
-## ORACLE with SCRAPER configuration
-BOOTNODES=/ip4/35.223.224.220/udp/4001/quic-v1/p2p/16Uiu2HAmPxXXjR1XJEwckh6q1UStheMmGaGe8fyXdeRs3SejadSa
-ENV=test
-FILE_PATH=.
-PORT=8080
-RPC_URL=https://ethereum-sepolia.publicnode.com
-TWITTER_USERNAME= your twitter username
-TWITTER_PASSWORD= your twitter password
-TWITTER_2FA_CODE= your twitter 2fa code
-DISCORD_SCRAPER=true
-DISCORD_BOT_TOKEN= your discord bot token
-TWITTER_SCRAPER=true
-WEB_SCRAPER=true
-TELEGRAM_SCRAPER=false
-
-# Go to my.telegram.org/auth and after logging in click the developer option to get these
-TELEGRAM_APP_ID=
-TELEGRAM_APP_HASH=
+# Worker Configuration
+# Note: To become a worker and provide data to the network, you must configure the following settings
 
-## Optional added features for ORACLE or VALIDATOR nodes only
-
-# Claude API and Elevenlabs API
-CLAUDE_API_KEY=
-CLAUDE_API_URL=
-CLAUDE_API_VERSION=
-ELAB_KEY=
-
-# For use with ollama https://hub.docker.com/r/ollama/ollama
-LLM_SERVER=true
-LLM_CHAT_URL=http://localhost:11434/api/chat
+# Twitter Configuration
+# Note: A pro-paid Twitter account is required to run a Twitter worker
+TWITTER_SCRAPER=true
+TWITTER_USERNAME=your pro-paid twitter username (without the '@' symbol)
+TWITTER_PASSWORD=your twitter password
+# Important: If your 2FA code times out, you'll need to restart your node and login by submitting a request.
+# We recommend temporarily disabling 2FA to save your cookies locally to your .home or .masa directory, then re-enabling it afterwards.
+# This will help avoid frequent login requests and potential timeouts.
+TWITTER_2FA_CODE=your twitter 2fa code (if applicable)
+
+# Discord Configuration
+# Note: You must have a bot in a Discord guild to scrape Discord channel messages
+DISCORD_SCRAPER=true
+DISCORD_BOT_TOKEN=your discord bot token
 
-# Bring your own Cloudflare worker token
-LLM_CF_URL=
-LLM_CF_TOKEN=
+# Web Scraper Configuration
+WEB_SCRAPER=true
 
-# Bring your own OpenAI api key
-OPENAI_API_KEY=
-PROMPT="You are a helpful assistant."
+# Telegram Configuration
+# Note: You must configure a bot as a developer and add it to a channel to scrape Telegram channel messages
+TELEGRAM_SCRAPER=false
+# To obtain these credentials, go to my.telegram.org/auth, log in, and select the API development tools
+TELEGRAM_APP_ID=your telegram app id
+TELEGRAM_APP_HASH=your telegram app hash
+# Configure your Telegram bot and add it to the channel you want to scrape
+TELEGRAM_BOT_TOKEN=your telegram bot token
+TELEGRAM_CHANNEL_USERNAME=username of the channel to scrape (without the '@' symbol)

.github/workflows/build-goreleaser-test.yml

@@ -19,7 +19,7 @@ jobs:
       - name: Run GoReleaser
         uses: goreleaser/goreleaser-action@v6
         with:
-          version: v2.1.0
+          version: '~> v2'
           args: build --clean --snapshot
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/goreleaser.yml

@@ -24,7 +24,7 @@ jobs:
       - name: Run GoReleaser
         uses: goreleaser/goreleaser-action@v6
         with:
-          version: v2.1.0
+          version: '~> v2'
           args: release --clean
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.goreleaser.yml

@@ -1,7 +1,27 @@
 # Make sure to check the documentation at http://goreleaser.com
 version: 2
 builds:
-  - main: ./cmd/masa-node/main.go
+  - main: ./cmd/masa-node
+    id: "masa-node"
+    binary: masa-node
+    ldflags:
+      - -w -s
+      - -X github.com/masa-finance/masa-oracle/internal.Version={{.Tag}}
+      - -X github.com/masa-finance/masa-oracle/internal.Commit={{.Commit}}
+    env:
+      - CGO_ENABLED=0
+    goos:
+      - linux
+      - windows
+      - darwin
+      - freebsd
+    goarch:
+      - amd64
+      - arm
+      - arm64
+  - main: ./cmd/masa-node-cli
+    id: "masa-node-cli"
+    binary: masa-node-cli
     ldflags:
       - -w -s
       - -X github.com/masa-finance/masa-oracle/internal.Version={{.Tag}}

DOCKER.md

@@ -31,8 +31,8 @@ Example `.env` file content:
 BOOTNODES=/ip4/35.223.224.220/udp/4001/quic-v1/p2p/16Uiu2HAmPxXXjR1XJEwckh6q1UStheMmGaGe8fyXdeRs3SejadSa
 RPC_URL=https://ethereum-sepolia.publicnode.com
 ENV=test
-TWITTER_USER="your_username"
-TWITTER_PASS="your_password"
+TWITTER_USERNAME="your_username"
+TWITTER_PASSWORD="your_password"
 TWITTER_2FA_CODE="your_2fa_code"
 TWITTER_SCRAPER=True
 

Makefile

@@ -1,8 +1,22 @@
 VERSION := $(shell git describe --tags --abbrev=0)
 
+GORELEASER?=
+# check if goreleaser exists
+ifeq (, $(shell which goreleaser))
+	GORELEASER=curl -sfL https://goreleaser.com/static/run | bash -s --
+else
+	GORELEASER=$(shell which goreleaser)
+endif
+
 print-version:
 	@echo "Version: ${VERSION}"
 
+dev-dist:
+	$(GORELEASER) build --snapshot --single-target --clean
+
+dist:
+	$(GORELEASER) build --single-target --clean
+
 build:
 	@go build -v -ldflags "-X github.com/masa-finance/masa-oracle/internal/versioning.ApplicationVersion=${VERSION}" -o ./bin/masa-node ./cmd/masa-node
 	@go build -v -ldflags "-X github.com/masa-finance/masa-oracle/internal/versioning.ApplicationVersion=${VERSION}" -o ./bin/masa-node-cli ./cmd/masa-node-cli

cmd/masa-node/main.go

@@ -69,7 +69,7 @@ func main() {
 	isValidator := cfg.Validator
 
 	// Create a new OracleNode
-	node, err := masa.NewOracleNode(ctx, isStaked)
+	node, err := masa.NewOracleNode(ctx, config.EnableStaked)
 	if err != nil {
 		logrus.Fatal(err)
 	}
@@ -78,6 +78,7 @@ func main() {
 		logrus.Fatal(err)
 	}
 
+	node.NodeTracker.GetAllNodeData()
 	if cfg.TwitterScraper && cfg.DiscordScraper && cfg.WebScraper {
 		logrus.Warn("[+] Node is set as all types of scrapers. This may not be intended behavior.")
 	}

docs/guides-and-tutorials/runpod.md

@@ -0,0 +1,83 @@
+# Quickstart: Deploying Masa Oracle Node on RunPod
+
+## Prerequisites
+
+1. A RunPod account (sign up at https://www.runpod.io/)
+2. Basic knowledge of Docker and command-line interfaces
+
+## Step 1: Use the Official Masa Finance Docker Image
+
+Instead of building your own Docker image, we'll use the official Masa Finance image from Docker Hub.
+
+## Step 2: Create a RunPod Template
+
+1. Log in to your RunPod account.
+2. Go to "Templates" in the left sidebar.
+3. Click "New Template".
+4. Fill in the template details:
+   - Name: Masa Protocol Node
+   - Image: masafinance/masa-node:latest
+   - Container Disk: 10 GB (or as needed)
+   - Volume Disk: 20 GB (or as needed)
+   - Ports: 4001/tcp, 4001/udp, 8080/tcp
+
+5. In the "Docker Command" field, enter:
+   ```bash
+   /usr/bin/masa-node --bootnodes="$BOOTNODES" --env="$ENV" --validator="$VALIDATOR" --cachePath="$CACHE_PATH"
+   ```
+
+6. Add environment variables:
+   - BOOTNODES
+   - ENV
+   - RPC_URL
+   - FILE_PATH
+   - VALIDATOR
+   - CACHE_PATH
+   - TWITTER_PASSwORD
+   - TWITTER_USERNAME
+   - TWITTER_2FA_CODE
+   - TWITTER_SCRAPER
+
+7. Save the template.
+
+## Step 3: Deploy Your Node on RunPod
+
+1. Go to "Pods" in the left sidebar.
+2. Click "Deploy".
+3. Select your "Masa Oracle Node" template.
+4. Choose a GPU type (CPU-only might be sufficient for basic node operation).
+5. Set the values for your environment variables.
+6. Deploy the pod.
+
+## Step 4: Access Your Node
+
+1. Once deployed, go to the "Pods" page.
+2. Find your Masa Oracle Node pod and click on it.
+3. You can access the node's logs and terminal from this page.
+
+## Step 5: Verify Node Operation
+
+1. Use the provided terminal to check if your node is running correctly:
+   ```bash
+   docker logs masa-node
+   ```
+
+2. Look for the startup message indicating successful connection to the network.
+
+## Step 6: Stake Your Node (if not already staked)
+
+If your node isn't staked, you can stake it using the RunPod terminal:
+
+```bash
+docker exec masa-node /usr/bin/masa-node --stake 1000
+```
+
+Replace 1000 with the amount of MASA tokens you want to stake.
+
+## Conclusion
+
+You've now successfully deployed your Masa Oracle Node on RunPod using the official Masa Finance Docker image. Monitor your node's performance and logs regularly to ensure it's operating correctly. Remember to keep your environment variables and node software up to date.
+
+For more detailed information on node operation and troubleshooting, refer to the main Masa Oracle documentation.
+
+[Source: https://hub.docker.com/repositories/masafinance]

docs/oracle-node/quickstart.md

@@ -51,8 +51,8 @@ OPENAI_API_KEY=
 PROMPT="You are a helpful assistant."
 
 # X
-TWITTER_USER="yourusername"
-TWITTER_PASS="yourpassword"
+TWITTER_USERNAME="yourusername"
+TWITTER_PASSWORD="yourpassword"
 TWITTER_2FA_CODE="your2fa"
 
 # Worker node config; default = false

docs/worker-node/quickstart.md

@@ -51,8 +51,8 @@ OPENAI_API_KEY=
 PROMPT="You are a helpful assistant."
 
 # X
-TWITTER_USER="yourusername"
-TWITTER_PASS="yourpassword"
+TWITTER_USERNAME="yourusername"
+TWITTER_PASSWORD="yourpassword"
 TWITTER_2FA_CODE="your2fa"
 
 # Worker node config; default = false

go.mod

@@ -30,6 +30,8 @@ require (
 	github.com/multiformats/go-multiaddr v0.13.0
 	github.com/multiformats/go-multihash v0.2.3
 	github.com/ollama/ollama v0.3.1
+	github.com/onsi/ginkgo/v2 v2.16.0
+	github.com/onsi/gomega v1.30.0
 	github.com/rivo/tview v0.0.0-20240505185119-ed116790de0f
 	github.com/sashabaranov/go-openai v1.28.2
 	github.com/sirupsen/logrus v1.9.3
@@ -104,6 +106,7 @@ require (
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect
+	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/gopacket v1.1.19 // indirect
 	github.com/google/pprof v0.0.0-20240207164012-fb44976bdcd5 // indirect
 	github.com/gopherjs/gopherjs v0.0.0-20190812055157-5d271430af9f // indirect
@@ -170,7 +173,6 @@ require (
 	github.com/multiformats/go-multicodec v0.9.0 // indirect
 	github.com/multiformats/go-multistream v0.5.0 // indirect
 	github.com/multiformats/go-varint v0.0.7 // indirect
-	github.com/onsi/ginkgo/v2 v2.16.0 // indirect
 	github.com/opencontainers/runtime-spec v1.2.0 // indirect
 	github.com/opentracing/opentracing-go v1.2.0 // indirect
 	github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect

pkg/config/app.go

@@ -110,20 +110,25 @@ type AppConfig struct {
 // If the unmarshalling fails, the instance is set to nil.
 //
 // Subsequent calls to GetInstance will return the same initialized instance.
-func GetInstance() *AppConfig {
+func GetInstance(options ...Option) *AppConfig {
+	o := &AppOption{}
+	o.Apply(options...)
+
 	once.Do(func() {
 		instance = &AppConfig{}
 
 		instance.setDefaultConfig()
 		instance.setEnvVariableConfig()
+
 		instance.setFileConfig(viper.GetString("FILE_PATH"))
-		err := instance.setCommandLineConfig()
-		if err != nil {
-			logrus.Fatal(err)
+
+		if !o.DisableCLIParse {
+			if err := instance.setCommandLineConfig(); err != nil {
+				logrus.Fatal(err)
+			}
 		}
 
-		err = viper.Unmarshal(instance)
-		if err != nil {
+		if err := viper.Unmarshal(instance); err != nil {
 			logrus.Errorf("[-] Unable to unmarshal config into struct, %v", err)
 			instance = nil // Ensure instance is nil if unmarshalling fails
 		}
@@ -260,5 +265,9 @@ func (c *AppConfig) LogConfig() {
 // It returns true if there is at least one bootnode in the Bootnodes slice and it is not an empty string.
 // Otherwise, it returns false, indicating that no bootnodes are configured.
 func (c *AppConfig) HasBootnodes() bool {
+	if len(c.Bootnodes) == 0 {
+		return false
+	}
+
 	return c.Bootnodes[0] != ""
 }