Skip to content

Commit

Permalink
[DVT-920] add nodelist cmd (#115)
Browse files Browse the repository at this point in the history
* add nodelist cmd

* update desc

* update docs

* move around commands

* update docs

* export entity kinds

* fix mark flag required

* fix logging

* move files around again

* update docs

* remove extra docs
  • Loading branch information
minhd-vu authored Aug 18, 2023
1 parent cdf7659 commit 195dcf4
Show file tree
Hide file tree
Showing 8 changed files with 238 additions and 95 deletions.
62 changes: 62 additions & 0 deletions cmd/p2p/nodelist/nodelist.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
package nodelist

import (
"encoding/json"
"os"

"github.com/maticnetwork/polygon-cli/p2p/database"
"github.com/spf13/cobra"
)

const jsonIndent = " "

type (
nodeListParams struct {
ProjectID string
OutputFile string
Limit int
}
)

var (
inputNodeListParams nodeListParams
)

var NodeListCmd = &cobra.Command{
Use: "nodelist [nodes.json]",
Short: "Generate a node list to seed a node",
Args: cobra.MinimumNArgs(1),
PreRunE: func(cmd *cobra.Command, args []string) (err error) {
inputNodeListParams.OutputFile = args[0]
inputNodeListParams.ProjectID, err = cmd.Flags().GetString("project-id")
return err
},
RunE: func(cmd *cobra.Command, args []string) error {
ctx := cmd.Context()

db := database.NewDatastore(cmd.Context(), database.DatastoreOptions{
ProjectID: inputNodeListParams.ProjectID,
})

nodes, err := db.NodeList(ctx, inputNodeListParams.Limit)
if err != nil {
return err
}

bytes, err := json.MarshalIndent(nodes, "", jsonIndent)
if err != nil {
return err
}

if err = os.WriteFile(inputNodeListParams.OutputFile, bytes, 0644); err != nil {
return err
}

return nil
},
}

func init() {
NodeListCmd.PersistentFlags().IntVarP(&inputNodeListParams.Limit, "limit", "l", 100, "Number of unique nodes to return")
NodeListCmd.PersistentFlags().StringVarP(&inputNodeListParams.ProjectID, "project-id", "p", "", "GCP project ID")
}
4 changes: 3 additions & 1 deletion cmd/p2p/p2p.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
_ "embed"

"github.com/maticnetwork/polygon-cli/cmd/p2p/crawl"
"github.com/maticnetwork/polygon-cli/cmd/p2p/nodelist"
"github.com/maticnetwork/polygon-cli/cmd/p2p/ping"
"github.com/maticnetwork/polygon-cli/cmd/p2p/sensor"
)
Expand All @@ -20,7 +21,8 @@ var P2pCmd = &cobra.Command{
}

func init() {
P2pCmd.AddCommand(sensor.SensorCmd)
P2pCmd.AddCommand(crawl.CrawlCmd)
P2pCmd.AddCommand(nodelist.NodeListCmd)
P2pCmd.AddCommand(ping.PingCmd)
P2pCmd.AddCommand(sensor.SensorCmd)
}
54 changes: 27 additions & 27 deletions cmd/p2p/sensor/sensor.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ type (
ProjectID string
SensorID string
MaxPeers int
MaxConcurrentDatabaseWrites int
MaxDatabaseConcurrency int
ShouldWriteBlocks bool
ShouldWriteBlockEvents bool
ShouldWriteTransactions bool
Expand Down Expand Up @@ -147,7 +147,7 @@ var SensorCmd = &cobra.Command{
db := database.NewDatastore(cmd.Context(), database.DatastoreOptions{
ProjectID: inputSensorParams.ProjectID,
SensorID: inputSensorParams.SensorID,
MaxConcurrentWrites: inputSensorParams.MaxConcurrentDatabaseWrites,
MaxConcurrency: inputSensorParams.MaxDatabaseConcurrency,
ShouldWriteBlocks: inputSensorParams.ShouldWriteBlocks,
ShouldWriteBlockEvents: inputSensorParams.ShouldWriteBlockEvents,
ShouldWriteTransactions: inputSensorParams.ShouldWriteTransactions,
Expand Down Expand Up @@ -279,44 +279,44 @@ func getLatestBlock(url string) (*rpctypes.RawBlockResponse, error) {
}

func init() {
SensorCmd.PersistentFlags().StringVarP(&inputSensorParams.Bootnodes, "bootnodes", "b", "", "Comma separated nodes used for bootstrapping")
SensorCmd.PersistentFlags().Uint64VarP(&inputSensorParams.NetworkID, "network-id", "n", 0, "Filter discovered nodes by this network ID")
if err := SensorCmd.MarkPersistentFlagRequired("network-id"); err != nil {
SensorCmd.Flags().StringVarP(&inputSensorParams.Bootnodes, "bootnodes", "b", "", "Comma separated nodes used for bootstrapping")
SensorCmd.Flags().Uint64VarP(&inputSensorParams.NetworkID, "network-id", "n", 0, "Filter discovered nodes by this network ID")
if err := SensorCmd.MarkFlagRequired("network-id"); err != nil {
log.Error().Err(err).Msg("Failed to mark network-id as required persistent flag")
}
SensorCmd.PersistentFlags().StringVarP(&inputSensorParams.ProjectID, "project-id", "P", "", "GCP project ID")
SensorCmd.PersistentFlags().StringVarP(&inputSensorParams.SensorID, "sensor-id", "s", "", "Sensor ID when writing block/tx events")
if err := SensorCmd.MarkPersistentFlagRequired("sensor-id"); err != nil {
SensorCmd.PersistentFlags().StringVarP(&inputSensorParams.ProjectID, "project-id", "p", "", "GCP project ID")
SensorCmd.Flags().StringVarP(&inputSensorParams.SensorID, "sensor-id", "s", "", "Sensor ID when writing block/tx events")
if err := SensorCmd.MarkFlagRequired("sensor-id"); err != nil {
log.Error().Err(err).Msg("Failed to mark sensor-id as required persistent flag")
}
SensorCmd.PersistentFlags().IntVarP(&inputSensorParams.MaxPeers, "max-peers", "m", 200, "Maximum number of peers to connect to")
SensorCmd.PersistentFlags().IntVarP(&inputSensorParams.MaxConcurrentDatabaseWrites, "max-db-writes", "D", 10000,
`Maximum number of concurrent database writes to perform. Increasing this
SensorCmd.Flags().IntVarP(&inputSensorParams.MaxPeers, "max-peers", "m", 200, "Maximum number of peers to connect to")
SensorCmd.Flags().IntVarP(&inputSensorParams.MaxDatabaseConcurrency, "max-db-concurrency", "D", 10000,
`Maximum number of concurrent database operations to perform. Increasing this
will result in less chance of missing data (i.e. broken pipes) but can
significantly increase memory usage.`)
SensorCmd.PersistentFlags().BoolVarP(&inputSensorParams.ShouldWriteBlocks, "write-blocks", "B", true, "Whether to write blocks to the database")
SensorCmd.PersistentFlags().BoolVar(&inputSensorParams.ShouldWriteBlockEvents, "write-block-events", true, "Whether to write block events to the database")
SensorCmd.PersistentFlags().BoolVarP(&inputSensorParams.ShouldWriteTransactions, "write-txs", "t", true,
SensorCmd.Flags().BoolVarP(&inputSensorParams.ShouldWriteBlocks, "write-blocks", "B", true, "Whether to write blocks to the database")
SensorCmd.Flags().BoolVar(&inputSensorParams.ShouldWriteBlockEvents, "write-block-events", true, "Whether to write block events to the database")
SensorCmd.Flags().BoolVarP(&inputSensorParams.ShouldWriteTransactions, "write-txs", "t", true,
`Whether to write transactions to the database. This option could significantly
increase CPU and memory usage.`)
SensorCmd.PersistentFlags().BoolVar(&inputSensorParams.ShouldWriteTransactionEvents, "write-tx-events", true,
SensorCmd.Flags().BoolVar(&inputSensorParams.ShouldWriteTransactionEvents, "write-tx-events", true,
`Whether to write transaction events to the database. This option could
significantly increase CPU and memory usage.`)
SensorCmd.PersistentFlags().BoolVar(&inputSensorParams.ShouldRunPprof, "pprof", false, "Whether to run pprof")
SensorCmd.PersistentFlags().UintVar(&inputSensorParams.PprofPort, "pprof-port", 6060, "Port pprof runs on")
SensorCmd.PersistentFlags().StringVarP(&inputSensorParams.KeyFile, "key-file", "k", "", "Private key file")
SensorCmd.PersistentFlags().IntVar(&inputSensorParams.Port, "port", 30303, "TCP network listening port")
SensorCmd.PersistentFlags().IntVar(&inputSensorParams.DiscoveryPort, "discovery-port", 30303, "UDP P2P discovery port")
SensorCmd.PersistentFlags().StringVar(&inputSensorParams.RPC, "rpc", "https://polygon-rpc.com", "RPC endpoint used to fetch the latest block")
SensorCmd.PersistentFlags().StringVar(&inputSensorParams.GenesisFile, "genesis", "genesis.json", "Genesis file")
SensorCmd.PersistentFlags().StringVar(&inputSensorParams.GenesisHash, "genesis-hash", "0xa9c28ce2141b56c474f1dc504bee9b01eb1bd7d1a507580d5519d4437a97de1b", "The genesis block hash")
SensorCmd.PersistentFlags().IntVar(&inputSensorParams.DialRatio, "dial-ratio", 0,
SensorCmd.Flags().BoolVar(&inputSensorParams.ShouldRunPprof, "pprof", false, "Whether to run pprof")
SensorCmd.Flags().UintVar(&inputSensorParams.PprofPort, "pprof-port", 6060, "Port pprof runs on")
SensorCmd.Flags().StringVarP(&inputSensorParams.KeyFile, "key-file", "k", "", "Private key file")
SensorCmd.Flags().IntVar(&inputSensorParams.Port, "port", 30303, "TCP network listening port")
SensorCmd.Flags().IntVar(&inputSensorParams.DiscoveryPort, "discovery-port", 30303, "UDP P2P discovery port")
SensorCmd.Flags().StringVar(&inputSensorParams.RPC, "rpc", "https://polygon-rpc.com", "RPC endpoint used to fetch the latest block")
SensorCmd.Flags().StringVar(&inputSensorParams.GenesisFile, "genesis", "genesis.json", "Genesis file")
SensorCmd.Flags().StringVar(&inputSensorParams.GenesisHash, "genesis-hash", "0xa9c28ce2141b56c474f1dc504bee9b01eb1bd7d1a507580d5519d4437a97de1b", "The genesis block hash")
SensorCmd.Flags().IntVar(&inputSensorParams.DialRatio, "dial-ratio", 0,
`Ratio of inbound to dialed connections. A dial ratio of 2 allows 1/2 of
connections to be dialed. Setting this to 0 defaults it to 3.`)
SensorCmd.PersistentFlags().StringVar(&inputSensorParams.NAT, "nat", "any", "NAT port mapping mechanism (any|none|upnp|pmp|pmp:<IP>|extip:<IP>)")
SensorCmd.PersistentFlags().BoolVar(&inputSensorParams.QuickStart, "quick-start", false,
SensorCmd.Flags().StringVar(&inputSensorParams.NAT, "nat", "any", "NAT port mapping mechanism (any|none|upnp|pmp|pmp:<IP>|extip:<IP>)")
SensorCmd.Flags().BoolVar(&inputSensorParams.QuickStart, "quick-start", false,
`Whether to load the nodes.json as static nodes to quickly start the network.
This produces faster development cycles but can prevent the sensor from being to
connect to new peers if the nodes.json file is large.`)
SensorCmd.PersistentFlags().StringVar(&inputSensorParams.TrustedNodesFile, "trusted-nodes", "", "Trusted nodes file")
SensorCmd.Flags().StringVar(&inputSensorParams.TrustedNodesFile, "trusted-nodes", "", "Trusted nodes file")
}
2 changes: 2 additions & 0 deletions doc/polycli_p2p.md
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,8 @@ The command also inherits flags from parent commands.
- [polycli](polycli.md) - A Swiss Army knife of blockchain tools.
- [polycli p2p crawl](polycli_p2p_crawl.md) - Crawl a network on the devp2p layer and generate a nodes JSON file.

- [polycli p2p nodelist](polycli_p2p_nodelist.md) - Generate a node list to seed a node

- [polycli p2p ping](polycli_p2p_ping.md) - Ping node(s) and return the output.

- [polycli p2p sensor](polycli_p2p_sensor.md) - Start a devp2p sensor that discovers other peers and will receive blocks and transactions.
Expand Down
44 changes: 44 additions & 0 deletions doc/polycli_p2p_nodelist.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
# `polycli p2p nodelist`

> Auto-generated documentation.
## Table of Contents

- [Description](#description)
- [Usage](#usage)
- [Flags](#flags)
- [See Also](#see-also)

## Description

Generate a node list to seed a node

```bash
polycli p2p nodelist [nodes.json] [flags]
```

## Flags

```bash
-h, --help help for nodelist
-l, --limit int Number of unique nodes to return (default 100)
-p, --project-id string GCP project ID
```

The command also inherits flags from parent commands.

```bash
--config string config file (default is $HOME/.polygon-cli.yaml)
--pretty-logs Should logs be in pretty format or JSON (default true)
-v, --verbosity int 0 - Silent
100 Fatal
200 Error
300 Warning
400 Info
500 Debug
600 Trace (default 400)
```

## See also

- [polycli p2p](polycli_p2p.md) - Set of commands related to devp2p.
60 changes: 30 additions & 30 deletions doc/polycli_p2p_sensor.md
Original file line number Diff line number Diff line change
Expand Up @@ -23,36 +23,36 @@ If no nodes.json file exists, it will be created.
## Flags

```bash
-b, --bootnodes string Comma separated nodes used for bootstrapping
--dial-ratio int Ratio of inbound to dialed connections. A dial ratio of 2 allows 1/2 of
connections to be dialed. Setting this to 0 defaults it to 3.
--discovery-port int UDP P2P discovery port (default 30303)
--genesis string Genesis file (default "genesis.json")
--genesis-hash string The genesis block hash (default "0xa9c28ce2141b56c474f1dc504bee9b01eb1bd7d1a507580d5519d4437a97de1b")
-h, --help help for sensor
-k, --key-file string Private key file
-D, --max-db-writes int Maximum number of concurrent database writes to perform. Increasing this
will result in less chance of missing data (i.e. broken pipes) but can
significantly increase memory usage. (default 10000)
-m, --max-peers int Maximum number of peers to connect to (default 200)
--nat string NAT port mapping mechanism (any|none|upnp|pmp|pmp:<IP>|extip:<IP>) (default "any")
-n, --network-id uint Filter discovered nodes by this network ID
--port int TCP network listening port (default 30303)
--pprof Whether to run pprof
--pprof-port uint Port pprof runs on (default 6060)
-P, --project-id string GCP project ID
--quick-start Whether to load the nodes.json as static nodes to quickly start the network.
This produces faster development cycles but can prevent the sensor from being to
connect to new peers if the nodes.json file is large.
--rpc string RPC endpoint used to fetch the latest block (default "https://polygon-rpc.com")
-s, --sensor-id string Sensor ID when writing block/tx events
--trusted-nodes string Trusted nodes file
--write-block-events Whether to write block events to the database (default true)
-B, --write-blocks Whether to write blocks to the database (default true)
--write-tx-events Whether to write transaction events to the database. This option could
significantly increase CPU and memory usage. (default true)
-t, --write-txs Whether to write transactions to the database. This option could significantly
increase CPU and memory usage. (default true)
-b, --bootnodes string Comma separated nodes used for bootstrapping
--dial-ratio int Ratio of inbound to dialed connections. A dial ratio of 2 allows 1/2 of
connections to be dialed. Setting this to 0 defaults it to 3.
--discovery-port int UDP P2P discovery port (default 30303)
--genesis string Genesis file (default "genesis.json")
--genesis-hash string The genesis block hash (default "0xa9c28ce2141b56c474f1dc504bee9b01eb1bd7d1a507580d5519d4437a97de1b")
-h, --help help for sensor
-k, --key-file string Private key file
-D, --max-db-concurrency int Maximum number of concurrent database operations to perform. Increasing this
will result in less chance of missing data (i.e. broken pipes) but can
significantly increase memory usage. (default 10000)
-m, --max-peers int Maximum number of peers to connect to (default 200)
--nat string NAT port mapping mechanism (any|none|upnp|pmp|pmp:<IP>|extip:<IP>) (default "any")
-n, --network-id uint Filter discovered nodes by this network ID
--port int TCP network listening port (default 30303)
--pprof Whether to run pprof
--pprof-port uint Port pprof runs on (default 6060)
-p, --project-id string GCP project ID
--quick-start Whether to load the nodes.json as static nodes to quickly start the network.
This produces faster development cycles but can prevent the sensor from being to
connect to new peers if the nodes.json file is large.
--rpc string RPC endpoint used to fetch the latest block (default "https://polygon-rpc.com")
-s, --sensor-id string Sensor ID when writing block/tx events
--trusted-nodes string Trusted nodes file
--write-block-events Whether to write block events to the database (default true)
-B, --write-blocks Whether to write blocks to the database (default true)
--write-tx-events Whether to write transaction events to the database. This option could
significantly increase CPU and memory usage. (default true)
-t, --write-txs Whether to write transactions to the database. This option could significantly
increase CPU and memory usage. (default true)
```
The command also inherits flags from parent commands.
Expand Down
3 changes: 3 additions & 0 deletions p2p/database/database.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,4 +44,7 @@ type Database interface {
ShouldWriteBlockEvents() bool
ShouldWriteTransactions() bool
ShouldWriteTransactionEvents() bool

// NodeList will return a list of enode URLs.
NodeList(ctx context.Context, limit int) ([]string, error)
}
Loading

0 comments on commit 195dcf4

Please sign in to comment.