Merge pull request #25 from pinax-network/feature/port-solana-to-parq…

…uet-files

Solana Parquet

.gitignore

@@ -825,4 +825,7 @@ adapters/**/*.csv
 adapters/**/*.jsonl
 
 # Sublime Text
-*.sublime*
+*.sublime*
+
+# parquet-sink-files state file
+*state.yaml

Cargo.toml

@@ -5,6 +5,7 @@ members = [
   "blocks/solana",
   "blocks/cosmos",
   "blocks/beacon",
+  "blocks/solana-parquet",
   "common",
 ]
 resolver = "2"
@@ -22,5 +23,6 @@ substreams-solana = "0.14"
 substreams-cosmos = "0.2"
 prost = "0.13"
 prost-types = "0.13"
+serde_json = "1.0"
 # substreams-bitcoin = "1.0"
 # substreams-near = "0.9"

blocks/solana-parquet/Cargo.toml

@@ -0,0 +1,16 @@
+[package]
+name = "raw_blocks_solana_parquet"
+edition = { workspace = true }
+version = { workspace = true }
+
+[lib]
+crate-type = ["cdylib"]
+
+[dependencies]
+common = { path = "../../common" }
+substreams-solana = { workspace = true }
+substreams-database-change = { workspace = true }
+substreams = { workspace = true }
+serde_json = { workspace = true }
+prost = { workspace = true }
+prost-types = { workspace = true }

blocks/solana-parquet/Makefile

@@ -0,0 +1,64 @@
+.PHONY: all
+all:
+	make build
+	make pack
+	make graph
+	make info
+
+.PHONY: build
+build:
+	cargo build --target wasm32-unknown-unknown --release
+
+.PHONY: pack
+pack:
+	substreams pack
+
+.PHONY: graph
+graph:
+	substreams graph
+
+.PHONY: info
+info:
+	substreams info
+
+.PHONY: run
+run:
+	substreams run substreams.yaml -e solana.substreams.pinax.network:443 ch_out -s 203100000 -o json
+
+.PHONY: protogen
+protogen:
+	substreams protogen --exclude-paths sf/substreams,google
+
+.PHONY: cache
+cache:
+	substreams gui substreams.yaml -e solana.substreams.pinax.network:443 ch_out -s 203100000 -t 203200000 --production-mode
+
+.PHONY: cache-foundational
+cache-foundational:
+	substreams gui https://github.com/streamingfast/substreams-foundational-modules/releases/download/solana-common-v0.3.0/solana-common-v0.3.0.spkg -e solana.substreams.pinax.network:443 blocks_without_votes -s 203100000 -t 203200000 --production-mode
+
+.PHONY: gui
+gui:
+	substreams gui substreams.yaml -e solana.substreams.pinax.network:443 ch_out -s 203100000
+
+.PHONY: sql-setup
+sql-setup:
+	# EVM blocks
+	substreams-sink-sql setup clickhouse://default:default@localhost:9000/solana substreams.yaml
+
+.PHONY: sql-run-solana
+sql-run-solana:
+	substreams-sink-sql run clickhouse://default:default@localhost:9000/solana substreams.yaml -e solana.substreams.pinax.network:443 203100000:203100100 --final-blocks-only --undo-buffer-size 1 --on-module-hash-mistmatch=warn  --batch-block-flush-interval 1 --development-mode
+
+.PHONY: sql-run-solana-blocks
+sql-run-solana-blocks:
+	substreams-sink-sql run clickhouse://default:default@localhost:9000/solana substreams.yaml -e solana.substreams.pinax.network:443 2: --final-blocks-only --undo-buffer-size 1 --on-module-hash-mistmatch=warn --batch-block-flush-interval 100 --params ch_out=blocks
+
+.PHONY: parquet
+parquet:
+	substreams-sink-files run solana.substreams.pinax.network:443 substreams.yaml ch_out_without_votes './out' 203100000:203100500 --encoder parquet --file-block-count 100 --development-mode
+
+.PHONY: deploy
+deploy:
+	graph build
+	graph deploy --studio clock

blocks/solana-parquet/README.md

@@ -0,0 +1,29 @@
+## `Solana` Raw Blockchain Data
+
+> Solana
+> [`sf.solana.type.v1.Block`](https://buf.build/streamingfast/firehose-solana/docs/main:sf.solana.type.v1)
+
+- [x] **Blocks**
+- [x] **Transactions**
+- [x] **Instruction Calls**
+- [x] **Account Activity**
+- [x] **Rewards**
+- [x] **Vote Transactions**
+- [x] **Vote Instruction Calls**
+- [x] **Vote Account Activity**
+- ~~[ ] **Discriminators**~~
+
+
+```mermaid
+graph TD;
+  raw[sf.solana.type.v1.Block];
+  raw --> blocks;
+  raw --> transactions;
+  raw --> instruction_calls;
+  raw --> account_activity;
+  raw --> rewards;
+  raw --> vote_transactions;
+  raw --> vote_instruction_calls;
+  raw --> vote_account_activity;
+  raw --> discriminators;
+```

blocks/solana-parquet/proto/solana.rawblocks.proto

@@ -0,0 +1,154 @@
+syntax = "proto3";
+
+package solana.rawblocks;
+
+import "google/protobuf/timestamp.proto";
+
+message Events {
+  repeated Block blocks = 1;
+  repeated Reward rewards = 2; 
+  repeated Transaction transactions = 3;
+  repeated InstructionCall instruction_calls = 4;
+  repeated AccountActivity account_activity = 5;
+  repeated Transaction vote_transactions = 6;
+  repeated InstructionCall vote_instruction_calls = 7;
+  repeated AccountActivity vote_account_activity = 8;
+}
+
+message Block {
+  // clock
+  google.protobuf.Timestamp time = 1;
+  string date = 2;
+  string hash = 3;
+
+  // block
+  uint64 slot = 4;
+  uint64 height = 5;
+  string previous_block_hash = 6;
+  uint64 parent_slot = 7;
+
+  // counters
+  uint64 total_transactions = 8;
+  uint64 successful_transactions = 9;
+  uint64 failed_transactions = 10;
+  uint64 total_vote_transactions = 11;
+  uint64 total_non_vote_transactions = 12;
+  uint64 successful_vote_transactions = 13;
+  uint64 successful_non_vote_transactions = 14;
+  uint64 failed_vote_transactions = 15;
+  uint64 failed_non_vote_transactions = 16;
+}
+
+message Reward {
+  // clock
+  google.protobuf.Timestamp block_time = 1;
+  string block_date = 2;
+  string block_hash = 3;
+
+  // block
+  uint64 block_slot = 4;
+  uint64 block_height = 5;
+  string block_previous_block_hash = 6;
+  uint64 block_parent_slot = 7;
+
+  // reward
+  string pubkey = 8;
+  int64 lamports = 9;
+  uint64 pre_balance = 10;
+  uint64 post_balance = 11;
+  string reward_type = 12;
+  string commission = 13;
+}
+
+message Transaction {
+  // clock
+  google.protobuf.Timestamp block_time = 1;
+  string block_hash = 2;
+  string block_date = 3;
+
+  // block
+  uint64 block_slot = 4;
+  uint64 block_height = 5;
+  string block_previous_block_hash = 6;
+  uint64 block_parent_slot = 7;
+
+  // transaction
+  string id = 8;
+  uint32 index = 9;
+  uint64 fee = 10;
+  uint32 required_signatures = 11;
+  uint32 required_signed_accounts = 12;
+  uint32 required_unsigned_accounts = 13;
+  string signature = 14;
+  bool success = 15;
+  string error = 16;
+  string recent_block_hash = 17;
+  repeated string account_keys = 18;
+  string log_messages = 19;
+  repeated string pre_balances = 20;
+  repeated string post_balances = 21;
+  repeated string signatures = 22;
+  string signer = 23;
+  repeated string signers = 24;
+}
+
+message InstructionCall {
+  // clock
+ google.protobuf.Timestamp block_time = 1;
+  string block_hash = 2;
+  string block_date = 3;
+
+  // block
+  uint64 block_slot = 4;
+  uint64 block_height = 5;
+  string block_previous_block_hash = 6;
+  uint64 block_parent_slot = 7;
+
+  // transaction
+  string tx_id = 8;
+  uint32 tx_index = 9;
+  string tx_signer = 10;
+  bool tx_success = 11;
+  repeated string log_messages = 12;
+
+  // instruction
+  uint32 outer_instruction_index = 13;
+  int32 inner_instruction_index = 14;
+  string inner_executing_account = 15;
+  string outer_executing_account = 16;
+  string executing_account = 17;
+  bool is_inner = 18;
+  string data = 19;
+  repeated string account_arguments = 20;
+  // string representing a nested array. Switch to nested array type when supported by sink-files
+  string inner_instructions = 21;
+}
+
+message AccountActivity {
+  // clock
+  google.protobuf.Timestamp block_time = 1;
+  string block_hash = 2;
+  string block_date = 3;
+
+  // block
+  uint64 block_slot = 4;
+  uint64 block_height = 5;
+  string block_previous_block_hash = 6;
+  uint64 block_parent_slot = 7;
+
+  string address = 8;
+  uint32 tx_index = 9;
+  string tx_id = 10;
+  bool tx_success = 11;
+  bool signed = 12;
+  bool writable = 13;
+  optional string token_mint_address = 14;
+
+  uint64 pre_balance = 15;
+  uint64 post_balance = 16;
+  int64 balance_change = 17;
+  optional double pre_token_balance = 18;
+  optional double post_token_balance = 19;
+  optional double token_balance_change = 20;
+  optional string token_balance_owner = 21;
+}