chore(all): Cleanup and linting (#623)

.github/workflows/tests.yaml

@@ -9,6 +9,7 @@ on:
       - '**'
 jobs:
   test:
+
     runs-on: ubuntu-latest
 
     steps:
@@ -20,12 +21,16 @@ jobs:
       with:
         go-version: '^1.22'
 
-    - name: Install dependencies
-      run: go mod tidy
+    - name: Install golangci-lint
+      run: sudo snap install golangci-lint --classic
+
+    - name: Code formatting and linting
+      run: make ci-lint
+
     - name: Run tests
-      run: |
-        make test
+      run: make test
+
     - name: Upload coverage to Codecov
       uses: codecov/codecov-action@v4
       with:
-        token: ${{ secrets.CODECOV_TOKEN }}
+        token: ${{ secrets.CODECOV_TOKEN }}

.gitignore

@@ -73,3 +73,6 @@ snippets.txt
 # Build result of goreleaser
 dist/
 bp-todo.md
+
+# Result of running tests with coverage
+coverage.txt

Makefile

@@ -28,7 +28,7 @@ build: contracts/node_modules
 
 install:
 	@sh ./node_install.sh
-	
+
 run: build
 	@./bin/masa-node
 
@@ -44,16 +44,25 @@ stake: build
 client: build	
 	@./bin/masa-node-cli
 
+# TODO Add -race and fix race conditions
 test: contracts/node_modules
-	@go test -coverprofile=coverage.txt -covermode=atomic -v ./...
+	@go test -coverprofile=coverage.txt -covermode=atomic -v -count=1 -shuffle=on ./...
+
+ci-lint:
+	go mod tidy && git diff --exit-code
+	go mod download
+	go mod verify
+	gofmt -s -w . && git diff --exit-code
+	go vet ./...
+	golangci-lint run
 
 clean:
 	@rm -rf bin
-	
+
 	@if [ -d ~/.masa/blocks ]; then rm -rf ~/.masa/blocks; fi
 	@if [ -d ~/.masa/cache ]; then rm -rf ~/.masa/cache; fi	
 	@if [ -f masa_node.log ]; then rm masa_node.log; fi
-	
+
 proto:
 	sh pkg/workers/messages/build.sh
 

cmd/masa-node-cli/handlers.go

@@ -7,7 +7,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"log"
 	"net/http"
 	"os"
 	"os/exec"
@@ -43,21 +42,6 @@ func handleIPAddress(multiAddr string) string {
 	return ""
 }
 
-// handleOpenFile reads the content of a file specified by the filename 'f' and returns it as a string.
-// If the file cannot be read, the function logs a fatal error and exits the program.
-// Parameters:
-// - f: The name of the file to read.
-// Returns:
-// - A string containing the content of the file.
-// func handleOpenFile(f string) string {
-// 	dat, err := os.ReadFile(f)
-// 	if err != nil {
-// 		log.Print(err)
-// 		return ""
-// 	}
-// 	return string(dat)
-// }
-
 // handleSaveFile writes the provided content to a file specified by the filename 'f'.
 // It appends the content to the file if it already exists, or creates a new file with the content if it does not.
 // The file is created with permissions set to 0755.
@@ -66,9 +50,14 @@ func handleIPAddress(multiAddr string) string {
 // - content: The content to write to the file.
 func handleSaveFile(f string, content string) {
 	file, err := os.OpenFile(f, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0755)
-	file.WriteString(content + "\n")
 	if err != nil {
-		log.Println(err)
+		logrus.Errorf("[-] Error while opening file %s for writing: %v", f, err)
+		return
+	}
+
+	_, err = file.WriteString(content + "\n")
+	if err != nil {
+		logrus.Errorf("[-] Error while writing to file %s: %v", f, err)
 		return
 	}
 }
@@ -84,7 +73,7 @@ func handleSaveFile(f string, content string) {
 func handleGPT(prompt string, userMessage string) (string, error) {
 	key := os.Getenv("OPENAI_API_KEY")
 	if key == "" {
-		log.Println("OPENAI_API_KEY is not set. Please set the environment variable and try again.")
+		logrus.Println("OPENAI_API_KEY is not set. Please set the environment variable and try again.")
 		return "", errors.New("OPENAI_API_KEY is not set")
 	}
 	client := openai.NewClient(key)
@@ -105,7 +94,7 @@ func handleGPT(prompt string, userMessage string) (string, error) {
 		},
 	)
 	if err != nil {
-		log.Print(err)
+		logrus.Errorf("[-] Error while getting ChatGPT completion: %v", err)
 		return "", err
 	}
 	return resp.Choices[0].Message.Content, nil
@@ -131,7 +120,7 @@ func handleSpeak(response string) {
 
 	req, err := http.NewRequest(http.MethodPost, os.Getenv("ELAB_URL"), bytes.NewBuffer(buf))
 	if err != nil {
-		log.Print(err)
+		logrus.Errorf("[-] Error while creating HTTP request to %s: %v", os.Getenv("ELAB_URL"), err)
 		return
 	}
 	req.Header.Set("accept", "*/*")
@@ -140,27 +129,47 @@ func handleSpeak(response string) {
 
 	resp, err := http.DefaultClient.Do(req)
 	if err != nil {
-		log.Print(err)
+		logrus.Errorf("[-] Error while sending HTTP POST to %s: %v", os.Getenv("ELAB_URL"), err)
 		return
 	}
 	defer resp.Body.Close()
 
 	if resp.StatusCode == http.StatusOK {
 		bodyBytes, err := io.ReadAll(resp.Body)
 		if err != nil {
-			log.Print(err)
+			logrus.Errorf("[-] Error while reading HTTP reply from ElevenLabs API: %v", err)
 			return
 		}
+
+		// TODO: Configure filename
 		file, err := os.Create("output.mp3")
-		file.Write(bodyBytes)
 		if err != nil {
-			log.Print(err)
+			logrus.Errorf("[-] Error while opening output.mp3 for writing: %v", err)
+			return
+		}
+
+		_, err = file.Write(bodyBytes)
+		if err != nil {
+			logrus.Errorf("[-] Error while writing voice data to output.mp3: %v", err)
 			return
-		} else {
-			cmd := exec.Command("afplay", "output.mp3")
-			go cmd.Run()
-			go handleTranscribe("output.mp3", "transcription.txt")
 		}
+
+		// TODO: Is afplay available in all platforms? Perhaps configure?
+		cmd := exec.Command("afplay", "output.mp3")
+		go func() {
+			err := cmd.Run()
+			if err != nil {
+				logrus.Errorf("[-] Error while playing output using %s: %v", cmd, err)
+			}
+		}()
+		go func() {
+			err := handleTranscribe("output.mp3", "transcription.txt")
+			if err != nil {
+				logrus.Errorf("[-] Error while transcribing audio: %v", err)
+			}
+		}()
+
+		// TODO: perhaps rm output.mp3?
 	}
 }
 
@@ -169,22 +178,21 @@ func handleSpeak(response string) {
 func handleTranscribe(audioFile string, txtFile string) error {
 	key := os.Getenv("OPENAI_API_KEY")
 	if key == "" {
-		log.Println("OPENAI_API_KEY is not set. Please set the environment variable and try again.")
-		return errors.New("OPENAI_API_KEY is not set")
+		return errors.New("OPENAI_API_KEY is not set. Please set the environment variable and try again.")
 	}
 	client := openai.NewClient(key)
 	ctx := context.Background()
 	req := openai.AudioRequest{
 		Model:    openai.Whisper1,
 		FilePath: audioFile,
 	}
+
 	resp, err := client.CreateTranscription(ctx, req)
 	if err != nil {
-		fmt.Printf("Transcription error: %v\n", err)
 		return err
-	} else {
-		handleSaveFile(txtFile, resp.Text)
 	}
+
+	handleSaveFile(txtFile, resp.Text)
 	return nil
 }
 

cmd/masa-node/main.go

@@ -70,8 +70,6 @@ func main() {
 		logrus.Warn("No staking event found for this address")
 	}
 
-	isValidator := cfg.Validator
-
 	masaNodeOptions, workHandlerManager, pubKeySub := initOptions(cfg)
 	// Create a new OracleNode
 	masaNode, err := node.NewOracleNode(ctx, masaNodeOptions...)
@@ -80,8 +78,7 @@ func main() {
 		logrus.Fatal(err)
 	}
 
-	err = masaNode.Start()
-	if err != nil {
+	if err = masaNode.Start(); err != nil {
 		logrus.Fatal(err)
 	}
 
@@ -118,8 +115,11 @@ func main() {
 	// Get the multiaddress and IP address of the node
 	multiAddr := masaNode.GetMultiAddrs()                      // Get the multiaddress
 	ipAddr, err := multiAddr.ValueForProtocol(multiaddr.P_IP4) // Get the IP address
+	if err != nil {
+		logrus.Errorf("[-] Error while getting node IP address from %v: %v", multiAddr, err)
+	}
 	// Display the welcome message with the multiaddress and IP address
-	config.DisplayWelcomeMessage(multiAddr.String(), ipAddr, keyManager.EthAddress, isStaked, isValidator, cfg.TwitterScraper, cfg.TelegramScraper, cfg.DiscordScraper, cfg.WebScraper, versioning.ApplicationVersion, versioning.ProtocolVersion)
+	config.DisplayWelcomeMessage(multiAddr.String(), ipAddr, keyManager.EthAddress, isStaked, cfg.Validator, cfg.TwitterScraper, cfg.TelegramScraper, cfg.DiscordScraper, cfg.WebScraper, versioning.ApplicationVersion, versioning.ProtocolVersion)
 
 	<-ctx.Done()
 }

go.mod

@@ -23,7 +23,6 @@ require (
 	github.com/ipfs/go-ds-leveldb v0.5.0
 	github.com/ipfs/go-ipfs-api v0.7.0
 	github.com/joho/godotenv v1.5.1
-	github.com/lib/pq v1.10.9
 	github.com/libp2p/go-libp2p v0.36.3
 	github.com/libp2p/go-libp2p-kad-dht v0.26.1
 	github.com/libp2p/go-libp2p-pubsub v0.12.0

go.sum

@@ -404,8 +404,6 @@ github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7
 github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8=
 github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
 github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
-github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
-github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
 github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6cdF0Y8=
 github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg=
 github.com/libp2p/go-cidranger v1.1.0 h1:ewPN8EZ0dd1LSnrtuwd4709PXVcITVeuwbag38yPW7c=

node/oracle_node.go

@@ -33,8 +33,9 @@ import (
 )
 
 type OracleNode struct {
-	Host          host.Host
-	Protocol      protocol.ID
+	Host     host.Host
+	Protocol protocol.ID
+	// TODO: Rm from here and from NodeData? Should not be necessary
 	priorityAddrs multiaddr.Multiaddr
 	multiAddrs    []multiaddr.Multiaddr
 	DHT           *dht.IpfsDHT
@@ -134,9 +135,13 @@ func NewOracleNode(ctx context.Context, opts ...Option) (*OracleNode, error) {
 		return nil, err
 	}
 
+	ma, err := myNetwork.GetMultiAddressesForHost(hst)
+	if err != nil {
+		return nil, err
+	}
 	n := &OracleNode{
 		Host:          hst,
-		multiAddrs:    myNetwork.GetMultiAddressesForHostQuiet(hst),
+		multiAddrs:    ma,
 		PeerChan:      make(chan myNetwork.PeerEvent),
 		NodeTracker:   pubsub.NewNodeEventTracker(versioning.ProtocolVersion, o.Environment, hst.ID().String()),
 		Context:       ctx,
@@ -163,14 +168,22 @@ func (node *OracleNode) generateEthHexKeyForRandomIdentity() (string, error) {
 	return common.BytesToAddress(ethereumCrypto.Keccak256(rawKey[1:])[12:]).Hex(), nil
 }
 
-func (node *OracleNode) getNodeData(host host.Host, addr multiaddr.Multiaddr, publicEthAddress string) *pubsub.NodeData {
+func (node *OracleNode) getNodeData() *pubsub.NodeData {
 	// GetSelfNodeData converts the local node's data into a JSON byte array.
 	// It populates a NodeData struct with the node's ID, staking status, and Ethereum address.
 	// The NodeData struct is then marshalled into a JSON byte array.
 	// Returns nil if there is an error marshalling to JSON.
 	// Create and populate NodeData
-	nodeData := pubsub.NewNodeData(addr, host.ID(), publicEthAddress, pubsub.ActivityJoined)
-	nodeData.MultiaddrsString = addr.String()
+
+	var publicEthAddress string
+	if node.Options.RandomIdentity {
+		publicEthAddress, _ = node.generateEthHexKeyForRandomIdentity()
+	} else {
+		publicEthAddress = masacrypto.KeyManagerInstance().EthAddress
+	}
+
+	nodeData := pubsub.NewNodeData(node.priorityAddrs, node.Host.ID(), publicEthAddress, pubsub.ActivityJoined)
+	nodeData.MultiaddrsString = node.priorityAddrs.String()
 	nodeData.IsStaked = node.Options.IsStaked
 	nodeData.IsTwitterScraper = node.Options.IsTwitterScraper
 	nodeData.IsDiscordScraper = node.Options.IsDiscordScraper
@@ -211,26 +224,19 @@ func (node *OracleNode) Start() (err error) {
 	go node.handleDiscoveredPeers()
 	go node.NodeTracker.ClearExpiredWorkerTimeouts()
 
-	var publicKeyHex string
-	if node.Options.RandomIdentity {
-		publicKeyHex, _ = node.generateEthHexKeyForRandomIdentity()
-	} else {
-		publicKeyHex = masacrypto.KeyManagerInstance().EthAddress
-	}
-
-	myNodeData := node.getNodeData(node.Host, node.priorityAddrs, publicKeyHex)
+	myNodeData := node.getNodeData()
 
 	bootstrapNodes, err := myNetwork.GetBootNodesMultiAddress(node.Options.Bootnodes)
 	if err != nil {
 		return err
 	}
 
-	node.DHT, err = myNetwork.WithDHT(node.Context, node.Host, bootstrapNodes, node.Protocol, masaPrefix, node.PeerChan, myNodeData)
+	node.DHT, err = myNetwork.EnableDHT(node.Context, node.Host, bootstrapNodes, node.Protocol, masaPrefix, node.PeerChan, myNodeData)
 	if err != nil {
 		return err
 	}
 
-	err = myNetwork.WithMDNS(node.Host, config.Rendezvous, node.PeerChan)
+	err = myNetwork.EnableMDNS(node.Host, config.Rendezvous, node.PeerChan)
 	if err != nil {
 		return err
 	}
@@ -340,21 +346,6 @@ func (node *OracleNode) IsPublisher() bool {
 	return node.Signature != ""
 }
 
-// FromUnixTime converts a Unix timestamp into a formatted string.
-// The Unix timestamp is expected to be in seconds.
-// The returned string is in the format "2006-01-02T15:04:05.000Z".
-func (node *OracleNode) FromUnixTime(unixTime int64) string {
-	return time.Unix(unixTime, 0).Format("2006-01-02T15:04:05.000Z")
-}
-
-// ToUnixTime converts a formatted string time into a Unix timestamp.
-// The input string is expected to be in the format "2006-01-02T15:04:05.000Z".
-// The returned Unix timestamp is in seconds.
-func (node *OracleNode) ToUnixTime(stringTime string) int64 {
-	t, _ := time.Parse("2006-01-02T15:04:05.000Z", stringTime)
-	return t.Unix()
-}
-
 // Version returns the current version string of the oracle node software.
 func (node *OracleNode) Version() string {
 	return config.GetInstance().Version