diff --git a/.gitignore b/.gitignore index f9a0987..102cf0b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ .idea/ vendor/ -*.conf +.DS_Store +sync-iris +sync-iris-unix + diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..ff4f202 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,38 @@ +FROM alpine:edge + +# Set up dependencies +ENV PACKAGES go make git libc-dev bash + +# Set up GOPATH & PATH + +ENV PROJECT_NAME irishub-sync +ENV GOPATH /root/go +ENV BASE_PATH $GOPATH/src/github.com/irisnet +ENV REPO_PATH $BASE_PATH/$PROJECT_NAME +ENV LOG_DIR /$PROJECT_NAME/log +ENV PATH $GOPATH/bin:$PATH + +# Set volumes + +VOLUME $LOG_DIR + +# Link expected Go repo path + +RUN mkdir -p $GOPATH/pkg $GOPATH/bin $BASE_PATH $REPO_PATH $LOG_DIR + +# Add source files + +COPY . $REPO_PATH + +# Install minimum necessary dependencies, build irishub-server +RUN apk add --no-cache $PACKAGES && \ + cd $REPO_PATH && make all && \ + mv $REPO_PATH/$PROJECT_NAME $GOPATH/bin && \ + rm -rf $REPO_PATH/vendor && \ + rm -rf $GOPATH/src/github.com/golang $GOPATH/bin/dep $GOPATH/pkg/* && \ + apk del $PACKAGES + +VOLUME ["$LOG_DIR"] + + +CMD irishub-sync > $LOG_DIR/debug.log && tail -f $LOG_DIR/debug.log \ No newline at end of file diff --git a/Gopkg.lock b/Gopkg.lock new file mode 100644 index 0000000..52d18d6 --- /dev/null +++ b/Gopkg.lock @@ -0,0 +1,505 @@ +# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. + + +[[projects]] + name = "github.com/bgentry/speakeasy" + packages = ["."] + revision = "4aabc24848ce5fd31929f7d1e4ea74d3709c14cd" + version = "v0.1.0" + +[[projects]] + branch = "master" + name = "github.com/btcsuite/btcd" + packages = ["btcec"] + revision = "fdfc19097e7ac6b57035062056f5b7b4638b8898" + +[[projects]] + branch = "master" + name = "github.com/btcsuite/btcutil" + packages = ["bech32"] + revision = "ab6388e0c60ae4834a1f57511e20c17b5f78be4b" + +[[projects]] + name = "github.com/cosmos/cosmos-sdk" + packages = [ + "client", + "client/context", + "client/keys", + "store", + "types", + "wire", + "x/auth", + "x/auth/client/cli", + "x/bank", + "x/ibc", + "x/slashing", + "x/stake" + ] + revision = "1e6d26ad3dce18fd1dde9bbee7d2aa192c196310" + version = "v0.19.1-rc1" + +[[projects]] + name = "github.com/davecgh/go-spew" + packages = ["spew"] + revision = "346938d642f2ec3594ed81d874461961cd0faa76" + version = "v1.1.0" + +[[projects]] + branch = "master" + name = "github.com/ebuchman/fail-test" + packages = ["."] + revision = "95f809107225be108efcf10a3509e4ea6ceef3c4" + +[[projects]] + name = "github.com/fsnotify/fsnotify" + packages = ["."] + revision = "c2828203cd70a50dcccfb2761f8b1f8ceef9a8e9" + version = "v1.4.7" + +[[projects]] + name = "github.com/go-kit/kit" + packages = [ + "log", + "log/level", + "log/term" + ] + revision = "4dc7be5d2d12881735283bcab7352178e190fc71" + version = "v0.6.0" + +[[projects]] + name = "github.com/go-logfmt/logfmt" + packages = ["."] + revision = "390ab7935ee28ec6b286364bba9b4dd6410cb3d5" + version = "v0.3.0" + +[[projects]] + name = "github.com/go-stack/stack" + packages = ["."] + revision = "259ab82a6cad3992b4e21ff5cac294ccb06474bc" + version = "v1.7.0" + +[[projects]] + name = "github.com/gogo/protobuf" + packages = [ + "gogoproto", + "jsonpb", + "proto", + "protoc-gen-gogo/descriptor", + "sortkeys", + "types" + ] + revision = "1adfc126b41513cc696b209667c8656ea7aac67c" + version = "v1.0.0" + +[[projects]] + name = "github.com/golang/protobuf" + packages = [ + "proto", + "ptypes", + "ptypes/any", + "ptypes/duration", + "ptypes/timestamp" + ] + revision = "925541529c1fa6821df4e44ce2723319eb2be768" + version = "v1.0.0" + +[[projects]] + branch = "master" + name = "github.com/golang/snappy" + packages = ["."] + revision = "2e65f85255dbc3072edf28d6b5b8efc472979f5a" + +[[projects]] + name = "github.com/gorilla/context" + packages = ["."] + revision = "08b5f424b9271eedf6f9f0ce86cb9396ed337a42" + version = "v1.1.1" + +[[projects]] + name = "github.com/gorilla/mux" + packages = ["."] + revision = "e3702bed27f0d39777b0b37b664b6280e8ef8fbf" + version = "v1.6.2" + +[[projects]] + name = "github.com/gorilla/websocket" + packages = ["."] + revision = "ea4d1f681babbce9545c9c5f3d5194a789c89f5b" + version = "v1.2.0" + +[[projects]] + branch = "master" + name = "github.com/hashicorp/hcl" + packages = [ + ".", + "hcl/ast", + "hcl/parser", + "hcl/printer", + "hcl/scanner", + "hcl/strconv", + "hcl/token", + "json/parser", + "json/scanner", + "json/token" + ] + revision = "ef8a98b0bbce4a65b5aa4c368430a80ddc533168" + +[[projects]] + branch = "master" + name = "github.com/howeyc/crc16" + packages = ["."] + revision = "2b2a61e366a66d3efb279e46176e7291001e0354" + +[[projects]] + name = "github.com/inconshreveable/mousetrap" + packages = ["."] + revision = "76626ae9c91c4f2a10f34cad8ce83ea42c93bb75" + version = "v1.0" + +[[projects]] + branch = "master" + name = "github.com/jmhodges/levigo" + packages = ["."] + revision = "c42d9e0ca023e2198120196f842701bb4c55d7b9" + +[[projects]] + branch = "master" + name = "github.com/kr/logfmt" + packages = ["."] + revision = "b84e30acd515aadc4b783ad4ff83aff3299bdfe0" + +[[projects]] + name = "github.com/magiconair/properties" + packages = ["."] + revision = "c2353362d570a7bfa228149c62842019201cfb71" + version = "v1.8.0" + +[[projects]] + name = "github.com/mattn/go-isatty" + packages = ["."] + revision = "0360b2af4f38e8d38c7fce2a9f4e702702d73a39" + version = "v0.0.3" + +[[projects]] + branch = "master" + name = "github.com/mitchellh/mapstructure" + packages = ["."] + revision = "bb74f1db0675b241733089d5a1faa5dd8b0ef57b" + +[[projects]] + name = "github.com/pelletier/go-toml" + packages = ["."] + revision = "c01d1270ff3e442a8a57cddc1c92dc1138598194" + version = "v1.2.0" + +[[projects]] + name = "github.com/pkg/errors" + packages = ["."] + revision = "645ef00459ed84a119197bfb8d8205042c6df63d" + version = "v0.8.0" + +[[projects]] + name = "github.com/pmezard/go-difflib" + packages = ["difflib"] + revision = "792786c7400a136282c1664665ae0a8db921c6c2" + version = "v1.0.0" + +[[projects]] + branch = "master" + name = "github.com/rcrowley/go-metrics" + packages = ["."] + revision = "e2704e165165ec55d062f5919b4b29494e9fa790" + +[[projects]] + name = "github.com/robfig/cron" + packages = ["."] + revision = "b41be1df696709bb6395fe435af20370037c0b4c" + version = "v1.1" + +[[projects]] + name = "github.com/spf13/afero" + packages = [ + ".", + "mem" + ] + revision = "787d034dfe70e44075ccc060d346146ef53270ad" + version = "v1.1.1" + +[[projects]] + name = "github.com/spf13/cast" + packages = ["."] + revision = "8965335b8c7107321228e3e3702cab9832751bac" + version = "v1.2.0" + +[[projects]] + name = "github.com/spf13/cobra" + packages = ["."] + revision = "ef82de70bb3f60c65fb8eebacbb2d122ef517385" + version = "v0.0.3" + +[[projects]] + branch = "master" + name = "github.com/spf13/jwalterweatherman" + packages = ["."] + revision = "7c0cea34c8ece3fbeb2b27ab9b59511d360fb394" + +[[projects]] + name = "github.com/spf13/pflag" + packages = ["."] + revision = "583c0c0531f06d5278b7d917446061adc344b5cd" + version = "v1.0.1" + +[[projects]] + name = "github.com/spf13/viper" + packages = ["."] + revision = "b5e8006cbee93ec955a89ab31e0e3ce3204f3736" + version = "v1.0.2" + +[[projects]] + name = "github.com/stretchr/testify" + packages = [ + "assert", + "require" + ] + revision = "f35b8ab0b5a2cef36673838d662e249dd9c94686" + version = "v1.2.2" + +[[projects]] + branch = "master" + name = "github.com/syndtr/goleveldb" + packages = [ + "leveldb", + "leveldb/cache", + "leveldb/comparer", + "leveldb/errors", + "leveldb/filter", + "leveldb/iterator", + "leveldb/journal", + "leveldb/memdb", + "leveldb/opt", + "leveldb/storage", + "leveldb/table", + "leveldb/util" + ] + revision = "c4c61651e9e37fa117f53c5a906d3b63090d8445" + +[[projects]] + name = "github.com/tendermint/abci" + packages = [ + "client", + "example/code", + "example/kvstore", + "types" + ] + revision = "198dccf0ddfd1bb176f87657e3286a05a6ed9540" + version = "v0.12.0" + +[[projects]] + branch = "master" + name = "github.com/tendermint/ed25519" + packages = [ + ".", + "edwards25519", + "extra25519" + ] + revision = "d8387025d2b9d158cf4efb07e7ebf814bcce2057" + +[[projects]] + name = "github.com/tendermint/go-amino" + packages = ["."] + revision = "ed62928576cfcaf887209dc96142cd79cdfff389" + version = "0.9.9" + +[[projects]] + name = "github.com/tendermint/go-crypto" + packages = [ + ".", + "keys", + "keys/bcrypt", + "keys/words", + "keys/words/wordlist" + ] + revision = "915416979bf70efa4bcbf1c6cd5d64c5fff9fc19" + version = "v0.6.2" + +[[projects]] + name = "github.com/tendermint/iavl" + packages = [ + ".", + "sha256truncated" + ] + revision = "c9206995e8f948e99927f5084a88a7e94ca256da" + version = "v0.8.0-rc0" + +[[projects]] + name = "github.com/tendermint/tendermint" + packages = [ + "blockchain", + "cmd/tendermint/commands", + "config", + "consensus", + "consensus/types", + "evidence", + "libs/events", + "libs/pubsub", + "libs/pubsub/query", + "lite", + "lite/client", + "lite/errors", + "lite/files", + "lite/proxy", + "mempool", + "node", + "p2p", + "p2p/conn", + "p2p/pex", + "p2p/upnp", + "privval", + "proxy", + "rpc/client", + "rpc/core", + "rpc/core/types", + "rpc/grpc", + "rpc/lib", + "rpc/lib/client", + "rpc/lib/server", + "rpc/lib/types", + "state", + "state/txindex", + "state/txindex/kv", + "state/txindex/null", + "types", + "version" + ] + revision = "46369a1ab76f274ab47179c4176221842b8207b4" + version = "v0.21.0" + +[[projects]] + name = "github.com/tendermint/tmlibs" + packages = [ + "autofile", + "bech32", + "cli", + "cli/flags", + "clist", + "common", + "db", + "flowrate", + "log", + "merkle", + "merkle/tmhash" + ] + revision = "49596e0a1f48866603813df843c9409fc19805c6" + version = "v0.9.0" + +[[projects]] + branch = "master" + name = "golang.org/x/crypto" + packages = [ + "blowfish", + "curve25519", + "internal/subtle", + "nacl/box", + "nacl/secretbox", + "openpgp/armor", + "openpgp/errors", + "poly1305", + "ripemd160", + "salsa20/salsa" + ] + revision = "a49355c7e3f8fe157a85be2f77e6e269a0f89602" + +[[projects]] + branch = "master" + name = "golang.org/x/net" + packages = [ + "context", + "http/httpguts", + "http2", + "http2/hpack", + "idna", + "internal/timeseries", + "trace" + ] + revision = "039a4258aec0ad3c79b905677cceeab13b296a77" + +[[projects]] + branch = "master" + name = "golang.org/x/sys" + packages = ["unix"] + revision = "1b2967e3c290b7c545b3db0deeda16e9be4f98a2" + +[[projects]] + name = "golang.org/x/text" + packages = [ + "collate", + "collate/build", + "internal/colltab", + "internal/gen", + "internal/tag", + "internal/triegen", + "internal/ucd", + "language", + "secure/bidirule", + "transform", + "unicode/bidi", + "unicode/cldr", + "unicode/norm", + "unicode/rangetable" + ] + revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0" + version = "v0.3.0" + +[[projects]] + name = "google.golang.org/genproto" + packages = ["googleapis/rpc/status"] + revision = "7fd901a49ba6a7f87732eb344f6e3c5b19d1b200" + +[[projects]] + name = "google.golang.org/grpc" + packages = [ + ".", + "balancer", + "codes", + "connectivity", + "credentials", + "grpclb/grpc_lb_v1/messages", + "grpclog", + "internal", + "keepalive", + "metadata", + "naming", + "peer", + "resolver", + "stats", + "status", + "tap", + "transport" + ] + revision = "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + version = "v1.7.5" + +[[projects]] + branch = "v2" + name = "gopkg.in/mgo.v2" + packages = [ + ".", + "bson", + "internal/json", + "internal/sasl", + "internal/scram" + ] + revision = "9856a29383ce1c59f308dd1cf0363a79b5bef6b5" + +[[projects]] + name = "gopkg.in/yaml.v2" + packages = ["."] + revision = "5420a8b6744d3b0345ab293f6fcba19c978f1183" + version = "v2.2.1" + +[solve-meta] + analyzer-name = "dep" + analyzer-version = 1 + inputs-digest = "165c8f6c97ce24d4415c9721fa0e86eccd01edd9a77268153465ddbc15ef378a" + solver-name = "gps-cdcl" + solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml new file mode 100644 index 0000000..e051a91 --- /dev/null +++ b/Gopkg.toml @@ -0,0 +1,75 @@ +# Gopkg.toml example +# +# Refer to https://golang.github.io/dep/docs/Gopkg.toml.html +# for detailed Gopkg.toml documentation. +# +# required = ["github.com/user/thing/cmd/thing"] +# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"] +# +# [[constraint]] +# name = "github.com/user/project" +# version = "1.0.0" +# +# [[constraint]] +# name = "github.com/user/project2" +# branch = "dev" +# source = "github.com/myfork/project2" +# +# [[override]] +# name = "github.com/x/y" +# version = "2.4.0" +# +# [prune] +# non-go = false +# go-tests = true +# unused-packages = true + + +[[constraint]] + name = "github.com/robfig/cron" + version = "1.1.0" + +[[constraint]] + name = "github.com/cosmos/cosmos-sdk" + version = "v0.19.1-rc1" + +[[override]] + name = "github.com/tendermint/abci" + version = "=0.12.0" + +[[override]] + name = "github.com/tendermint/go-crypto" + version = "=0.6.2" + +[[override]] + name = "github.com/tendermint/go-amino" + version = "=0.9.9" + +[[override]] + name = "github.com/tendermint/iavl" + version = "=0.8.0-rc0" + +[[override]] + name = "github.com/tendermint/tendermint" + version = "=0.21.0" + +[[override]] + name = "github.com/tendermint/tmlibs" + version = "=v0.9.0" + +# this got updated and broke, so locked to an old working commit ... +[[override]] + name = "google.golang.org/genproto" + revision = "7fd901a49ba6a7f87732eb344f6e3c5b19d1b200" + +[[constraint]] + branch = "v2" + name = "gopkg.in/mgo.v2" + +[[constraint]] + branch = "v2" + name = "github.com/stretchr/testify" + +[prune] + go-tests = true + unused-packages = true diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..5fe6801 --- /dev/null +++ b/Makefile @@ -0,0 +1,43 @@ +GOCMD=go +GOBUILD=$(GOCMD) build +GOCLEAN=$(GOCMD) clean +GOTEST=$(GOCMD) test +GOGET=$(GOCMD) get +BINARY_NAME=irishub-sync +BINARY_UNIX=$(BINARY_NAME)-unix + +all: get_tools get_deps build + +get_deps: + @rm -rf vendor/ + @echo "--> Running dep ensure" + @dep ensure -v + +build: + $(GOBUILD) -o $(BINARY_NAME) -v + +clean: + $(GOCLEAN) + rm -f $(BINARY_NAME) + rm -f $(BINARY_UNIX) + +run: + $(GOBUILD) -o $(BINARY_NAME) -v + ./$(BINARY_NAME) + + +# Cross compilation +build-linux: + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 $(GOBUILD) -o $(BINARY_UNIX) -v + +###################################### +## Tools + +check_tools: + cd tools && $(MAKE) check_tools + +get_tools: + cd tools && $(MAKE) get_tools + +update_tools: + cd tools && $(MAKE) update_tools \ No newline at end of file diff --git a/README.md b/README.md index 4c3da43..865a594 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,24 @@ -# irishub-sync -A daemon that synchronizes IRIS hub data into database +# IRISHUB-SYNC +A server that synchronize IRIS blockChain data into a database + +# Structure + +- `conf`: config of project +- `module`: project module +- `mongodb`: mongodb script to create database +- `service`: main logic of sync-server, sync data from blockChain and write to database +- `store`: database model +- `util`: common constants and helper functions +- `main.go`: bootstrap project + +# SetUp + +## Create mongodb database + +run script `mongodb.js` in `mongodb` folder to create database before run project + +# Build And Run + +- Build: `make all` +- Run: `make run` +- Cross compilation: `make build-linux` diff --git a/conf/db/types.go b/conf/db/types.go index 93bee57..d7fe50c 100644 --- a/conf/db/types.go +++ b/conf/db/types.go @@ -1,7 +1,53 @@ package db -const ( - Host = "" - Port = "" - Database = "" +import ( + "os" + "github.com/irisnet/irishub-sync/util/constant" + "github.com/irisnet/irishub-sync/module/logger" ) + +var ( + Host = "127.0.0.1" + Port = "27217" + User = "user" + Passwd = "passwd" + Database = "sync_irishub" +) + +// get value of env var +func init() { + host, found := os.LookupEnv(constant.EnvNameDbHost) + if found { + Host = host + logger.Info.Printf("The value of env var %v is %v\n", + constant.EnvNameDbHost, host) + } + + port, found := os.LookupEnv(constant.EnvNameDbPort) + if found { + Port = port + logger.Info.Printf("The value of env var %v is %v\n", + constant.EnvNameDbPort, port) + } + + user, found := os.LookupEnv(constant.EnvNameDbUser) + if found { + User = user + logger.Info.Printf("The value of env var %v is %v\n", + constant.EnvNameDbUser, user) + } + + passwd, found := os.LookupEnv(constant.EnvNameDbPassWd) + if found { + Passwd = passwd + logger.Info.Printf("The value of env var %v is %v\n", + constant.EnvNameDbPassWd, passwd) + } + + database, found := os.LookupEnv(constant.EnvNameDbDataBase) + if found { + Database = database + logger.Info.Printf("The value of env var %v is %v\n", + constant.EnvNameDbDataBase, database) + } +} diff --git a/conf/server/types.go b/conf/server/types.go index 4f96102..538589d 100644 --- a/conf/server/types.go +++ b/conf/server/types.go @@ -1,13 +1,70 @@ package server -const ( - BlockChainMonitorUrl = "tcp://localhost:46657" - Token = "iris" - InitConnectionNum = 100 - MaxConnectionNum = 200 - ChainId = "test" - SyncCron = "1 * * * * *" +import ( + "os" + "strconv" + + "github.com/irisnet/irishub-sync/util/constant" + "github.com/irisnet/irishub-sync/module/logger" +) - SyncMaxGoroutine = 2000 - SyncBlockNumFastSync = 2000 +var ( + BlockChainMonitorUrl = "tcp://127.0.0.1:46657" + ChainId = "test" + Token = "iris" + + InitConnectionNum = 100 // fast init num of tendermint client pool + MaxConnectionNum = 1000 // max size of tendermint client pool + SyncCron = "0-59 * * * * *" + + SyncMaxGoroutine = 60 // max go routine in server + SyncBlockNumFastSync = 8000 // sync block num each goroutine ) + +// get value of env var +func init() { + nodeUrl, found := os.LookupEnv(constant.EnvNameSerNetworkNodeUrl) + if found { + BlockChainMonitorUrl = nodeUrl + logger.Info.Printf("The value of env var %v is %v\n", + constant.EnvNameSerNetworkNodeUrl, nodeUrl) + } + + chainId, found := os.LookupEnv(constant.EnvNameSerNetworkChainId) + if found { + ChainId = chainId + logger.Info.Printf("The value of env var %v is %v\n", + constant.EnvNameSerNetworkChainId, chainId) + } + + token, found := os.LookupEnv(constant.EnvNameSerNetworkToken) + if found { + Token = token + logger.Info.Printf("The value of env var %v is %v\n", + constant.EnvNameSerNetworkToken, token) + } + + maxGoroutine, found := os.LookupEnv(constant.EnvNameSerMaxGoRoutine) + if found { + var err error + SyncMaxGoroutine, err = strconv.Atoi(maxGoroutine) + if err != nil { + logger.Error.Fatalf("Convert str to int failed, env var is %v\n", + constant.EnvNameSerMaxGoRoutine) + } + logger.Info.Printf("The value of env var %v is %v\n", + constant.EnvNameSerMaxGoRoutine, maxGoroutine) + } + + syncBlockNum, found := os.LookupEnv(constant.EnvNameSerSyncBlockNum) + if found { + var err error + SyncBlockNumFastSync, err = strconv.Atoi(syncBlockNum) + if err != nil { + logger.Error.Fatalf("Convert str to int failed, env var is %v\n", + constant.EnvNameSerSyncBlockNum) + } + logger.Info.Printf("The value of env var %v is %v\n", + constant.EnvNameSerSyncBlockNum, SyncBlockNumFastSync) + } +} diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml new file mode 100644 index 0000000..8cf83fc --- /dev/null +++ b/docker/docker-compose.yml @@ -0,0 +1,21 @@ +version: "2" +services: + irishub-server: + image: irisnet/irishub-sync:develop + container_name: c_irishub-sync-develop + volumes: + - /mnt/data/irishub-sync/log:/irishub-sync/log + environment: + ENV: dev + DB_HOST: 127.0.0.1 + DB_PORT: 27117 + DB_USER: user + DB_PASSWD: passwd + DB_DATABASE: sync_irishub + + + SER_BC_NODE_URL: tcp://127.0.0.1:46657 + SER_BC_CHAIN_ID: test + SER_BC_TOKEN: iris + SER_MAX_GOROUTINE: 60 + SER_SYNC_BLOCK_NUM: 8000 \ No newline at end of file diff --git a/glide.lock b/glide.lock deleted file mode 100644 index d0a8da6..0000000 --- a/glide.lock +++ /dev/null @@ -1,309 +0,0 @@ -hash: 195b6bc507f3be7f05cbfeab8a493ee8188892f8fc266853ea43d2090b718298 -updated: 2018-03-20T09:31:27.390489+08:00 -imports: -- name: github.com/bgentry/speakeasy - version: 4aabc24848ce5fd31929f7d1e4ea74d3709c14cd -- name: github.com/btcsuite/btcd - version: 2e60448ffcc6bf78332d1fe590260095f554dd78 - subpackages: - - btcec -- name: github.com/BurntSushi/toml - version: b26d9c308763d68093482582cea63d69be07a0f0 -- name: github.com/cosmos/cosmos-sdk - version: f1b1d36324812dc221da87f56734a5c25d8754aa - subpackages: - - app - - client - - client/commands - - client/commands/commits - - client/commands/keys - - client/commands/proxy - - client/commands/query - - client/commands/rpc - - client/commands/search - - client/commands/txs - - client/rest - - errors - - genesis - - modules/auth - - modules/auth/commands - - modules/base - - modules/base/commands - - modules/coin - - modules/coin/commands - - modules/coin/rest - - modules/fee - - modules/fee/commands - - modules/ibc - - modules/ibc/commands - - modules/nonce - - modules/nonce/commands - - modules/nonce/rest - - modules/roles - - modules/roles/commands - - modules/roles/rest - - server/commands - - stack - - state - - version -- name: github.com/cosmos/gaia - version: 2ac7ed920446ef8edf582c2b3e890cb7676a7d77 - subpackages: - - modules/stake - - modules/stake/commands - - modules/stake/rest - - version -- name: github.com/ebuchman/fail-test - version: 95f809107225be108efcf10a3509e4ea6ceef3c4 -- name: github.com/fsnotify/fsnotify - version: 4da3e2cfbabc9f751898f250b49f2439785783a1 -- name: github.com/go-kit/kit - version: 4dc7be5d2d12881735283bcab7352178e190fc71 - subpackages: - - log - - log/level - - log/term -- name: github.com/go-logfmt/logfmt - version: 390ab7935ee28ec6b286364bba9b4dd6410cb3d5 -- name: github.com/go-playground/locales - version: e4cbcb5d0652150d40ad0646651076b6bd2be4f6 - subpackages: - - currency -- name: github.com/go-playground/universal-translator - version: 71201497bace774495daed26a3874fd339e0b538 -- name: github.com/go-stack/stack - version: 259ab82a6cad3992b4e21ff5cac294ccb06474bc -- name: github.com/gogo/protobuf - version: 1adfc126b41513cc696b209667c8656ea7aac67c - subpackages: - - gogoproto - - jsonpb - - proto - - protoc-gen-gogo/descriptor - - sortkeys - - types -- name: github.com/golang/protobuf - version: 925541529c1fa6821df4e44ce2723319eb2be768 - subpackages: - - proto - - ptypes - - ptypes/any - - ptypes/duration - - ptypes/timestamp -- name: github.com/golang/snappy - version: 553a641470496b2327abcac10b36396bd98e45c9 -- name: github.com/gorilla/context - version: 08b5f424b9271eedf6f9f0ce86cb9396ed337a42 -- name: github.com/gorilla/mux - version: 53c1911da2b537f792e7cafcb446b05ffe33b996 -- name: github.com/gorilla/websocket - version: ea4d1f681babbce9545c9c5f3d5194a789c89f5b -- name: github.com/hashicorp/hcl - version: 23c074d0eceb2b8a5bfdbb271ab780cde70f05a8 - subpackages: - - hcl/ast - - hcl/parser - - hcl/scanner - - hcl/strconv - - hcl/token - - json/parser - - json/scanner - - json/token -- name: github.com/howeyc/crc16 - version: 2b2a61e366a66d3efb279e46176e7291001e0354 -- name: github.com/inconshreveable/mousetrap - version: 76626ae9c91c4f2a10f34cad8ce83ea42c93bb75 -- name: github.com/jmhodges/levigo - version: c42d9e0ca023e2198120196f842701bb4c55d7b9 -- name: github.com/kr/logfmt - version: b84e30acd515aadc4b783ad4ff83aff3299bdfe0 -- name: github.com/magiconair/properties - version: 49d762b9817ba1c2e9d0c69183c2b4a8b8f1d934 -- name: github.com/mattn/go-isatty - version: 0360b2af4f38e8d38c7fce2a9f4e702702d73a39 -- name: github.com/mitchellh/mapstructure - version: 06020f85339e21b2478f756a78e295255ffa4d6a -- name: github.com/pelletier/go-toml - version: 4e9e0ee19b60b13eb79915933f44d8ed5f268bdd -- name: github.com/pkg/errors - version: 645ef00459ed84a119197bfb8d8205042c6df63d -- name: github.com/rcrowley/go-metrics - version: e181e095bae94582363434144c61a9653aff6e50 -- name: github.com/spf13/afero - version: 8d919cbe7e2627e417f3e45c3c0e489a5b7e2536 - subpackages: - - mem -- name: github.com/spf13/cast - version: acbeb36b902d72a7a4c18e8f3241075e7ab763e4 -- name: github.com/spf13/cobra - version: 7b2c5ac9fc04fc5efafb60700713d4fa609b777b -- name: github.com/spf13/jwalterweatherman - version: 12bd96e66386c1960ab0f74ced1362f66f552f7b -- name: github.com/spf13/pflag - version: e57e3eeb33f795204c1ca35f56c44f83227c6e66 -- name: github.com/spf13/viper - version: 25b30aa063fc18e48662b86996252eabdcf2f0c7 -- name: github.com/syndtr/goleveldb - version: adf24ef3f94bd13ec4163060b21a5678f22b429b - subpackages: - - leveldb - - leveldb/cache - - leveldb/comparer - - leveldb/errors - - leveldb/filter - - leveldb/iterator - - leveldb/journal - - leveldb/memdb - - leveldb/opt - - leveldb/storage - - leveldb/table - - leveldb/util -- name: github.com/tendermint/abci - version: 3d5f0a8b94bbbfa70ecd4072fc55854f8210425b - subpackages: - - client - - example/code - - example/kvstore - - server - - types -- name: github.com/tendermint/ed25519 - version: d8387025d2b9d158cf4efb07e7ebf814bcce2057 - subpackages: - - edwards25519 - - extra25519 -- name: github.com/tendermint/go-crypto - version: 2017856384589234024cda848a6332157d1f527c - subpackages: - - keys - - keys/cryptostore - - keys/storage/filestorage - - keys/wordlist -- name: github.com/tendermint/go-wire - version: b6fc872b42d41158a60307db4da051dd6f179415 - subpackages: - - data -- name: github.com/tendermint/iavl - version: 39de8f0b4ee758fdd5bb3a9afc6dd9bf42c04785 -- name: github.com/tendermint/tendermint - version: 9c5937df969de23e9087f183c52b5088d4b6f268 - subpackages: - - blockchain - - cmd/tendermint/commands - - config - - consensus - - consensus/types - - evidence - - lite - - lite/client - - lite/errors - - lite/files - - lite/proxy - - mempool - - client - - p2p - - p2p/conn - - p2p/pex - - p2p/trust - - p2p/upnp - - proxy - - rpc/client - - rpc/core - - rpc/core/types - - rpc/grpc - - rpc/lib - - rpc/lib/client - - rpc/lib/server - - rpc/lib/types - - state - - state/txindex - - state/txindex/kv - - state/txindex/null - - types - - version -- name: github.com/tendermint/tmlibs - version: 26f2ab65f82cfc6873c312e8030104c47c05f10e - subpackages: - - autofile - - cli - - cli/flags - - clist - - common - - db - - flowrate - - log - - merkle - - pubsub - - pubsub/query -- name: golang.org/x/crypto - version: 94eea52f7b742c7cbe0b03b22f0c4c8631ece122 - subpackages: - - curve25519 - - nacl/box - - nacl/secretbox - - openpgp/armor - - openpgp/errors - - poly1305 - - ripemd160 - - salsa20/salsa -- name: golang.org/x/net - version: 5ccada7d0a7ba9aeb5d3aca8d3501b4c2a509fec - subpackages: - - context - - http2 - - http2/hpack - - idna - - internal/timeseries - - lex/httplex - - trace -- name: golang.org/x/sys - version: 8b4580aae2a0dd0c231a45d3ccb8434ff533b840 - subpackages: - - unix -- name: golang.org/x/text - version: 57961680700a5336d15015c8c50686ca5ba362a4 - subpackages: - - secure/bidirule - - transform - - unicode/bidi - - unicode/norm -- name: google.golang.org/genproto - version: a8101f21cf983e773d0c1133ebc5424792003214 - repo: https://github.com/google/go-genproto - vcs: git - subpackages: - - googleapis/rpc/status -- name: google.golang.org/grpc - version: 401e0e00e4bb830a10496d64cd95e068c5bf50de - repo: https://github.com/grpc/grpc-go - vcs: git - subpackages: - - balancer - - codes - - connectivity - - credentials - - grpclb/grpc_lb_v1/messages - - grpclog - - internal - - keepalive - - metadata - - naming - - peer - - resolver - - stats - - status - - tap - - transport -- name: gopkg.in/go-playground/validator.v9 - version: 1b8c8e19cd250435025214492d9a08411d760fdd -- name: gopkg.in/yaml.v2 - version: 287cf08546ab5e7e37d55a84f7ed3fd1db036de5 -testImports: -- name: github.com/stretchr/testify - version: 2aa2c176b9dab406a6970f6a55f513e8a8c8b18f - subpackages: - - assert - - require -- name: gopkg.in/mgo.v2 - version: master -- name: github.com/robfig/cron -version: master \ No newline at end of file diff --git a/glide.yaml b/glide.yaml deleted file mode 100644 index 4a1d605..0000000 --- a/glide.yaml +++ /dev/null @@ -1,53 +0,0 @@ -package: github.com/irisnet/iris-sync-server -import: -- package: github.com/gorilla/websocket -- package: github.com/pkg/errors - version: ^0.8.0 -- package: github.com/spf13/cobra -- package: github.com/spf13/pflag -- package: github.com/spf13/viper -- package: github.com/tendermint/abci - version: develop - subpackages: - - server - - types -- package: github.com/cosmos/cosmos-sdk - version: develop -- package: github.com/tendermint/iavl - version: develop -- package: github.com/tendermint/go-crypto - version: remove_ledger - subpackages: - - cmd - - keys -- package: github.com/tendermint/go-wire - version: develop - subpackages: - - data -- package: github.com/tendermint/tendermint - version: v0.15.0 - subpackages: - - config - - node - - proxy - - rpc/client - - rpc/core/types - - rpc/lib/client - - rpc/lib/types - - types -- package: github.com/tendermint/tmlibs - version: develop - subpackages: - - cli - - cli/flags - - common - - events - - log - - logger -- package: github.com/gorilla/mux - version: ^1.5.0 -testImport: -- package: github.com/stretchr/testify - subpackages: - - assert - - require diff --git a/main.go b/main.go index a4d141d..0d58df8 100644 --- a/main.go +++ b/main.go @@ -1,14 +1,15 @@ package main import ( - "fmt" - - "github.com/irisnet/iris-sync-server/module/logger" + "github.com/irisnet/irishub-sync/service" + "sync" ) func main() { - i := 5 - j := 2 - logger.Info.Printf("result of j / i is %v", i/j) - fmt.Println("This is test") + var wg sync.WaitGroup + wg.Add(2) + + service.Start() + + wg.Wait() } diff --git a/model/store/document/account.go b/model/store/document/account.go deleted file mode 100644 index f808f35..0000000 --- a/model/store/document/account.go +++ /dev/null @@ -1,55 +0,0 @@ -package document - -import ( - "github.com/cosmos/cosmos-sdk/modules/coin" - "github.com/irisnet/iris-sync-server/model/store" - "gopkg.in/mgo.v2" - "gopkg.in/mgo.v2/bson" - "time" - "github.com/irisnet/iris-sync-server/module/logger" -) - -const ( - CollectionNmAccount = "account" -) - -type Account struct { - Address string `bson:"address"` - Amount coin.Coins `bson:"amount"` - Time time.Time `bson:"time"` - Height int64 `bson:"height"` -} - -func (a Account) Name() string { - return CollectionNmAccount -} - -func (a Account) PkKvPair() map[string]interface{} { - return bson.M{"address": a.Address} -} - -func (a Account) Index() []mgo.Index { - return []mgo.Index{ - { - Key: []string{"address"}, - Unique: true, - DropDups: false, - Background: true, - }, - } -} - -func QueryAccount(address string) (Account, error) { - var result Account - query := func(c *mgo.Collection) error { - err := c.Find(bson.M{"address": address}).Sort("-amount.amount").One(&result) - return err - } - - if store.ExecCollection(CollectionNmAccount, query) != nil { - logger.Info.Println("Account is Empty") - return result, nil - } - - return result, nil -} diff --git a/model/store/document/block.go b/model/store/document/block.go deleted file mode 100644 index 44f4520..0000000 --- a/model/store/document/block.go +++ /dev/null @@ -1,36 +0,0 @@ -package document - -import ( - "gopkg.in/mgo.v2" - "gopkg.in/mgo.v2/bson" - "time" -) - -const ( - CollectionNmBlock = "block" -) - -type Block struct { - Height int64 `bson:"height"` - Time time.Time `bson:"time"` - TxNum int64 `bson:"tx_num"` -} - -func (d Block) Name() string { - return CollectionNmBlock -} - -func (d Block) PkKvPair() map[string]interface{} { - return bson.M{"height": d.Height} -} - -func (d Block) Index() []mgo.Index { - return []mgo.Index{ - { - Key: []string{"height"}, - Unique: true, - DropDups: true, - Background: true, - }, - } -} diff --git a/model/store/document/stake_role_candidate.go b/model/store/document/stake_role_candidate.go deleted file mode 100644 index 1632847..0000000 --- a/model/store/document/stake_role_candidate.go +++ /dev/null @@ -1,76 +0,0 @@ -package document - -import ( - "errors" - "github.com/irisnet/iris-sync-server/model/store" - "gopkg.in/mgo.v2" - "gopkg.in/mgo.v2/bson" - "log" -) - -const ( - CollectionNmStakeRoleCandidate = "stake_role_candidate" -) - -type Candidate struct { - Address string `bson:"address"` // owner - PubKey string `bson:"pub_key"` - Shares int64 `bson:"shares"` - VotingPower uint64 `bson:"voting_power"` // Voting power if pubKey is a considered a validator - Description Description `bson:"description"` // Description terms for the candidate -} - -func (d Candidate) Name() string { - return CollectionNmStakeRoleCandidate -} - -func (d Candidate) PkKvPair() map[string]interface{} { - return bson.M{"pub_key": d.PubKey} -} - -func (d Candidate) Index() []mgo.Index { - return []mgo.Index { - { - Key: []string{"pub_key"}, - Unique: true, - DropDups: false, - Background: true, - }, - { - Key: []string{"address"}, - Unique: false, - DropDups: false, - Background: true, - }, - } -} - -//func QueryCandidateByAddressAndPubkey(address string, pubKey string) (Candidate, error) { -// var result Candidate -// query := func(c *mgo.Collection) error { -// err := c.Find(bson.M{"address": address, "pub_key": pubKey}).Sort("-shares").One(&result) -// return err -// } -// -// if store.ExecCollection(CollectionNmStakeRoleDelegator, query) != nil { -// log.Printf("delegator is Empty") -// return result, errors.New("delegator is Empty") -// } -// -// return result, nil -//} - -func QueryCandidateByPubkey(pubKey string) (Candidate, error) { - var result Candidate - query := func(c *mgo.Collection) error { - err := c.Find(bson.M{"pub_key": pubKey}).One(&result) - return err - } - - if store.ExecCollection(CollectionNmStakeRoleCandidate, query) != nil { - log.Printf("candidate is Empty") - return result, errors.New("candidate is Empty") - } - - return result, nil -} diff --git a/model/store/document/stake_role_delegator.go b/model/store/document/stake_role_delegator.go deleted file mode 100644 index 37d565f..0000000 --- a/model/store/document/stake_role_delegator.go +++ /dev/null @@ -1,65 +0,0 @@ -package document - -import ( - "errors" - "github.com/irisnet/iris-sync-server/model/store" - "gopkg.in/mgo.v2" - "gopkg.in/mgo.v2/bson" - "log" -) - -const ( - CollectionNmStakeRoleDelegator = "stake_role_delegator" -) - -type Delegator struct { - Address string `bson:"address"` - PubKey string `bson:"pub_key"` - Shares int64 `bson:"shares"` -} - -func (d Delegator) Name() string { - return CollectionNmStakeRoleDelegator -} - -func (d Delegator) PkKvPair() map[string]interface{} { - return bson.M{"address": d.Address, "pub_key": d.PubKey} -} - -func (d Delegator) Index() []mgo.Index { - return []mgo.Index{ - { - Key: []string{"address"}, - Unique: false, - DropDups: false, - Background: true, - }, - { - Key: []string{"pub_key"}, - Unique: false, - DropDups: false, - Background: true, - }, - { - Key: []string{"address", "pub_key"}, - Unique: true, - DropDups: false, - Background: true, - }, - } -} - -func QueryDelegatorByAddressAndPubkey(address string, pubKey string) (Delegator, error) { - var result Delegator - query := func(c *mgo.Collection) error { - err := c.Find(bson.M{"address": address, "pub_key": pubKey}).Sort("-shares").One(&result) - return err - } - - if store.ExecCollection(CollectionNmStakeRoleDelegator, query) != nil { - log.Printf("delegator is Empty") - return result, errors.New("delegator is Empty") - } - - return result, nil -} diff --git a/model/store/document/tx_coin.go b/model/store/document/tx_coin.go deleted file mode 100644 index fb401d2..0000000 --- a/model/store/document/tx_coin.go +++ /dev/null @@ -1,66 +0,0 @@ -package document - -import ( - "github.com/cosmos/cosmos-sdk/modules/coin" - "gopkg.in/mgo.v2" - "gopkg.in/mgo.v2/bson" - "time" -) - -const ( - CollectionNmCoinTx = "tx_coin" -) - -//Coin交易 -type CoinTx struct { - TxHash string `bson:"tx_hash"` - Time time.Time `bson:"time"` - Height int64 `bson:"height"` - From string `bson:"from"` - To string `bson:"to"` - Amount coin.Coins `bson:"amount"` -} - -func (c CoinTx) Name() string { - return CollectionNmCoinTx -} - -func (c CoinTx) PkKvPair() map[string]interface{} { - return bson.M{"tx_hash": c.TxHash} -} - -func (c CoinTx) Index() []mgo.Index { - return []mgo.Index{ - { - Key: []string{"tx_hash"}, - Unique: true, - DropDups: false, - Background: true, - }, - { - Key: []string{"from"}, - Unique: false, - DropDups: false, - Background: true, - }, - { - Key: []string{"to"}, - Unique: false, - DropDups: false, - Background: true, - }, - { - Key: []string{"-height"}, - Unique: false, - DropDups: false, - Background: true, - }, - { - Key: []string{"from", "to"}, - Unique: false, - DropDups: false, - Background: true, - }, - - } -} diff --git a/model/store/document/tx_stake.go b/model/store/document/tx_stake.go deleted file mode 100644 index 4167101..0000000 --- a/model/store/document/tx_stake.go +++ /dev/null @@ -1,72 +0,0 @@ -package document - -import ( - "github.com/cosmos/cosmos-sdk/modules/coin" - "gopkg.in/mgo.v2" - "gopkg.in/mgo.v2/bson" - "time" -) - -const ( - CollectionNmStakeTx = "tx_stake" -) - -// StakeTx -type StakeTx struct { - TxHash string `bson:"tx_hash"` - Time time.Time `bson:"time"` - Height int64 `bson:"height"` - From string `bson:"from"` - PubKey string `bson:"pub_key"` - Type string `bson:"type"` - Amount coin.Coin `bson:"amount"` -} - -func (c StakeTx) Name() string { - return CollectionNmStakeTx -} - -func (c StakeTx) PkKvPair() map[string]interface{} { - return bson.M{"tx_hash": c.TxHash} -} - -func (c StakeTx) Index() []mgo.Index { - return []mgo.Index{ - { - Key: []string{"tx_hash"}, - Unique: true, - DropDups: false, - Background: true, - }, - { - Key: []string{"from"}, - Unique: false, - DropDups: false, - Background: true, - }, - { - Key: []string{"pub_key"}, - Unique: false, - DropDups: false, - Background: true, - }, - { - Key: []string{"-height"}, - Unique: false, - DropDups: false, - Background: true, - }, - { - Key: []string{"type"}, - Unique: false, - DropDups: false, - Background: true, - }, - { - Key: []string{"from", "pub_key", "type"}, - Unique: false, - DropDups: false, - Background: true, - }, - } -} diff --git a/model/store/document/tx_stake_declare_candidacy.go b/model/store/document/tx_stake_declare_candidacy.go deleted file mode 100644 index dc0163e..0000000 --- a/model/store/document/tx_stake_declare_candidacy.go +++ /dev/null @@ -1,42 +0,0 @@ -package document - -import ( - "gopkg.in/mgo.v2" - "gopkg.in/mgo.v2/bson" -) - -const ( - CollectionNmStakeTxDeclareCandidacy = "tx_stake" -) - -// Description -type Description struct { - Moniker string `bson:"moniker"` - Identity string `bson:"identity"` - Website string `bson:"website"` - Details string `bson:"details"` -} - -type StakeTxDeclareCandidacy struct { - StakeTx `bson:"stake_tx"` - Description `bson:"description"` -} - -func (s StakeTxDeclareCandidacy) Name() string { - return CollectionNmStakeTxDeclareCandidacy -} - -func (s StakeTxDeclareCandidacy) PkKvPair() map[string]interface{} { - return bson.M{"stake_tx.tx_hash": s.TxHash} -} - -func (s StakeTxDeclareCandidacy) Index() []mgo.Index { - return []mgo.Index{ - { - Key: []string{"description.moniker"}, - Unique: false, - DropDups: false, - Background: true, - }, - } -} \ No newline at end of file diff --git a/model/store/store.go b/model/store/store.go deleted file mode 100644 index 42177e9..0000000 --- a/model/store/store.go +++ /dev/null @@ -1,154 +0,0 @@ -// init mongodb session and provide common functions - -package store - -import ( - "fmt" - "time" - - conf "github.com/irisnet/iris-sync-server/conf/db" - "github.com/irisnet/iris-sync-server/module/logger" - - "gopkg.in/mgo.v2" - "gopkg.in/mgo.v2/bson" -) - -var ( - session *mgo.Session - docs []Docs -) - -func RegisterDocs(d Docs) { - docs = append(docs, d) -} - -func Init() { - if session == nil { - url := fmt.Sprintf("mongodb://%s:%s", conf.Host, conf.Port) - - logger.Info.Printf("Mgo start on %s\n", url) - - var err error - session, err = mgo.Dial(url) - if err != nil { - logger.Error.Fatalln(err) - } - session.SetMode(mgo.Monotonic, true) - - index() - } -} - -func InitWithAuth(addrs []string, username, password string) { - dialInfo := &mgo.DialInfo{ - Addrs: addrs, //[]string{"192.168.6.122"} - Direct: false, - Timeout: time.Second * 1, - Database: conf.Database, - Username: username, - Password: password, - PoolLimit: 4096, // Session.SetPoolLimit - } - - session, err := mgo.DialWithInfo(dialInfo) - session.SetMode(mgo.Monotonic, true) - if nil != err { - panic(err) - } - index() -} - -func getSession() *mgo.Session { - // max session num is 4096 - return session.Clone() -} - -// get collection object -func ExecCollection(collection string, s func(*mgo.Collection) error) error { - session := getSession() - defer session.Close() - c := session.DB(conf.Database).C(collection) - return s(c) -} - -func Find(collection string, query interface{}) *mgo.Query { - session := getSession() - defer session.Close() - c := session.DB(conf.Database).C(collection) - return c.Find(query) -} - -func index() { - if len(docs) == 0 { - return - } - for _, h := range docs { - indexKey := func(c *mgo.Collection) error { - for _, i := range h.Index() { - err := c.EnsureIndex(i) - if err != nil { - logger.Error.Println(err) - return err - } - } - return nil - } - ExecCollection(h.Name(), indexKey) - } -} - -func Save(h Docs) error { - save := func(c *mgo.Collection) error { - //先按照关键字查询，如果存在，直接返回 - n, _ := c.Find(h.PkKvPair()).Count() - if n >= 1 { - logger.Info.Println("db: record existed while save data") - return nil - } - logger.Info.Printf("insert %s %+v\n", h.Name(), h) - return c.Insert(h) - } - - return ExecCollection(h.Name(), save) -} - -func SaveOrUpdate(h Docs) error { - save := func(c *mgo.Collection) error { - //先按照关键字查询，如果存在，直接返回 - n, err := c.Find(h.PkKvPair()).Count() - logger.Info.Printf("Count:%d err:%+v\n", n, err) - if n >= 1 { - return Update(h) - } - logger.Info.Printf("insert %s %+v\n", h.Name(), h) - return c.Insert(h) - } - - return ExecCollection(h.Name(), save) -} - -func Update(h Docs) error { - update := func(c *mgo.Collection) error { - key := h.PkKvPair() - logger.Info.Printf("update %s set %+v where %+v\n", h.Name(), h, key) - return c.Update(h.PkKvPair(), h) - } - return ExecCollection(h.Name(), update) -} - -/** - * 执行查询，此方法可拆分做为公共方法 - * [SearchPerson description] - * @param {[type]} collectionName string [description] - * @param {[type]} query bson.M [description] - * @param {[type]} sort bson.M [description] - * @param {[type]} fields bson.M [description] - * @param {[type]} skip int [description] - * @param {[type]} limit int) (results []interface{}, err error [description] - */ -func Query(collectionName string, query bson.M, sort string, fields bson.M, skip int, limit int) (results []interface{}, err error) { - exop := func(c *mgo.Collection) error { - return c.Find(query).Sort(sort).Select(fields).Skip(skip).Limit(limit).All(&results) - } - return results, ExecCollection(collectionName, exop) -} diff --git a/module/codec/codec.go b/module/codec/codec.go new file mode 100644 index 0000000..cd171ee --- /dev/null +++ b/module/codec/codec.go @@ -0,0 +1,29 @@ +package codec + +import ( + "github.com/cosmos/cosmos-sdk/wire" + "github.com/cosmos/cosmos-sdk/x/ibc" + "github.com/cosmos/cosmos-sdk/x/bank" + "github.com/cosmos/cosmos-sdk/x/stake" + "github.com/cosmos/cosmos-sdk/x/slashing" + "github.com/cosmos/cosmos-sdk/x/auth" + sdktypes "github.com/cosmos/cosmos-sdk/types" +) + +var ( + Cdc *wire.Codec +) + +func init() { + Cdc = wire.NewCodec() + + ibc.RegisterWire(Cdc) + bank.RegisterWire(Cdc) + stake.RegisterWire(Cdc) + slashing.RegisterWire(Cdc) + auth.RegisterWire(Cdc) + + sdktypes.RegisterWire(Cdc) + + wire.RegisterCrypto(Cdc) +} diff --git a/module/logger/logger.go b/module/logger/logger.go index d574889..f75f11a 100644 --- a/module/logger/logger.go +++ b/module/logger/logger.go @@ -4,34 +4,68 @@ import ( "io" "log" "os" + "io/ioutil" ) var ( - Info *log.Logger // Important information - Error *log.Logger // Critical problem + Trace *log.Logger // Trace db log + Info *log.Logger // Important information + Warning *log.Logger // Warning information + Error *log.Logger // Critical problem ) const ( - errFile = ".sync_server_err.log" + // traceDbFile = ".sync_server_db.log" + // debugFile = ".sync_server_debug.log" + errFile = ".sync_server_err.log" + warningFile = ".sync_server_warning.log" ) func init() { - errFile, err := os.OpenFile(os.ExpandEnv("$HOME/" + errFile), + // traceDbFile, err := os.OpenFile(os.ExpandEnv("$HOME/"+traceDbFile), + // os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) + // if err != nil { + // log.Fatalln("Failed to open trace db log file:", err) + // } + // + // debugFile, err := os.OpenFile(os.ExpandEnv("$HOME/"+debugFile), + // os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) + // if err != nil { + // log.Fatalln("Failed to open debug log file:", err) + // } + + warningFile, err := os.OpenFile(os.ExpandEnv("$HOME/"+warningFile), + os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) + if err != nil { + log.Fatalln("Failed to open warning log file:", err) + } + + errFile, err := os.OpenFile(os.ExpandEnv("$HOME/"+errFile), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) if err != nil { log.Fatalln("Failed to open error log file:", err) } + Trace = log.New(ioutil.Discard, + "TRACE: ", + log.Ldate|log.Ltime|log.Lshortfile) + Info = log.New(os.Stdout, "INFO: ", log.Ldate|log.Ltime|log.Lshortfile) + Warning = log.New(io.MultiWriter(warningFile, os.Stderr), + "WARNING: ", + log.Ldate|log.Ltime|log.Lshortfile) + Error = log.New(io.MultiWriter(errFile, os.Stderr), "ERROR: ", log.Ldate|log.Ltime|log.Lshortfile) } func test() { + Trace.Println("This is trace info...") Info.Println("This is info info...") + Warning.Println("This is warning info...") Error.Println("This is err info...") } diff --git a/module/stake/errors.go b/module/stake/errors.go deleted file mode 100644 index 900077b..0000000 --- a/module/stake/errors.go +++ /dev/null @@ -1,53 +0,0 @@ -// nolint -package stake - -import ( - "fmt" - - "github.com/cosmos/cosmos-sdk/errors" -) - -var ( - errCandidateEmpty = fmt.Errorf("Cannot bond to an empty candidate") - errBadBondingDenom = fmt.Errorf("Invalid coin denomination") - errBadBondingAmount = fmt.Errorf("Amount must be > 0") - errNoBondingAcct = fmt.Errorf("No bond account for this (address, validator) pair") - errCommissionNegative = fmt.Errorf("Commission must be positive") - errCommissionHuge = fmt.Errorf("Commission cannot be more than 100%") - - errBadValidatorAddr = fmt.Errorf("Validator does not exist for that address") - errCandidateExistsAddr = fmt.Errorf("Candidate already exist, cannot re-declare candidacy") - errMissingSignature = fmt.Errorf("Missing signature") - errBondNotNominated = fmt.Errorf("Cannot bond to non-nominated account") - errNoCandidateForAddress = fmt.Errorf("Validator does not exist for that address") - errNoDelegatorForAddress = fmt.Errorf("Delegator does not contain validator bond") - errInsufficientFunds = fmt.Errorf("Insufficient bond shares") - errBadRemoveValidator = fmt.Errorf("Error removing validator") - - invalidInput = errors.CodeTypeBaseInvalidInput -) - -func ErrBadValidatorAddr() error { - return errors.WithCode(errBadValidatorAddr, errors.CodeTypeBaseUnknownAddress) -} -func ErrCandidateExistsAddr() error { - return errors.WithCode(errCandidateExistsAddr, errors.CodeTypeBaseInvalidInput) -} -func ErrMissingSignature() error { - return errors.WithCode(errMissingSignature, errors.CodeTypeUnauthorized) -} -func ErrBondNotNominated() error { - return errors.WithCode(errBondNotNominated, errors.CodeTypeBaseInvalidOutput) -} -func ErrNoCandidateForAddress() error { - return errors.WithCode(errNoCandidateForAddress, errors.CodeTypeBaseUnknownAddress) -} -func ErrNoDelegatorForAddress() error { - return errors.WithCode(errNoDelegatorForAddress, errors.CodeTypeBaseInvalidInput) -} -func ErrInsufficientFunds() error { - return errors.WithCode(errInsufficientFunds, errors.CodeTypeBaseInvalidInput) -} -func ErrBadRemoveValidator() error { - return errors.WithCode(errBadRemoveValidator, errors.CodeTypeInternalErr) -} diff --git a/module/stake/state.go b/module/stake/state.go deleted file mode 100644 index f26e989..0000000 --- a/module/stake/state.go +++ /dev/null @@ -1,22 +0,0 @@ -package stake - -import "github.com/tendermint/go-crypto" - -var ( - // Keys for store prefixes - CandidatesPubKeysKey = []byte{0x01} // key for all candidates' pubkeys - ParamKey = []byte{0x02} // key for global parameters relating to staking - - // Key prefixes - CandidateKeyPrefix = []byte{0x03} // prefix for each key to a candidate - DelegatorBondKeyPrefix = []byte{0x04} // prefix for each key to a delegator's bond - DelegatorBondsKeyPrefix = []byte{0x05} // prefix for each key to a delegator's bond -) - -// DelegatorBond represents the bond with tokens held by an account. It is -// owned by one delegator, and is associated with the voting power of one -// pubKey. -type DelegatorBond struct { - PubKey crypto.PubKey - Shares uint64 -} diff --git a/module/stake/tx.go b/module/stake/tx.go deleted file mode 100644 index 2202687..0000000 --- a/module/stake/tx.go +++ /dev/null @@ -1,168 +0,0 @@ -package stake - -import ( - "fmt" - - sdk "github.com/cosmos/cosmos-sdk" - "github.com/cosmos/cosmos-sdk/modules/coin" - crypto "github.com/tendermint/go-crypto" -) - -//TODO 当cosmos-sdk加入stake模块时需要删除该文件，防止在启动时，重复加载ByteTxDeclareCandidacy，ByteTxEditCandidacy等类型，导致启动失败 - -const stakingModuleName = "stake" - -// Description - description fields for a candidate -type Description struct { - Moniker string `json:"moniker"` - Identity string `json:"identity"` - Website string `json:"website"` - Details string `json:"details"` -} - -// Name is the name of the modules. -func Name() string { - return stakingModuleName -} - -// Tx -//-------------------------------------------------------------------------------- - -// register the tx type with its validation logic -// make sure to use the name of the handler as the prefix in the tx type, -// so it gets routed properly -const ( - ByteTxDeclareCandidacy = 0x55 - ByteTxEditCandidacy = 0x56 - ByteTxDelegate = 0x57 - ByteTxUnbond = 0x58 - TypeTxDeclareCandidacy = stakingModuleName + "/declareCandidacy" - TypeTxEditCandidacy = stakingModuleName + "/editCandidacy" - TypeTxDelegate = stakingModuleName + "/delegate" - TypeTxUnbond = stakingModuleName + "/unbond" -) - -func init() { - sdk.TxMapper.RegisterImplementation(TxDeclareCandidacy{}, TypeTxDeclareCandidacy, ByteTxDeclareCandidacy) - sdk.TxMapper.RegisterImplementation(TxEditCandidacy{}, TypeTxEditCandidacy, ByteTxEditCandidacy) - sdk.TxMapper.RegisterImplementation(TxDelegate{}, TypeTxDelegate, ByteTxDelegate) - sdk.TxMapper.RegisterImplementation(TxUnbond{}, TypeTxUnbond, ByteTxUnbond) -} - -//Verify interface at compile time -var _, _, _, _ sdk.TxInner = &TxDeclareCandidacy{}, &TxEditCandidacy{}, &TxDelegate{}, &TxUnbond{} - -// BondUpdate - struct for bonding or unbonding transactions -type BondUpdate struct { - PubKey crypto.PubKey `json:"pub_key"` - Bond coin.Coin `json:"amount"` -} - -// ValidateBasic - Check for non-empty candidate, and valid coins -func (tx BondUpdate) ValidateBasic() error { - if tx.PubKey.Empty() { - return errCandidateEmpty - } - - coins := coin.Coins{tx.Bond} - if !coins.IsValid() { - return coin.ErrInvalidCoins() - } - if !coins.IsPositive() { - return fmt.Errorf("Amount must be > 0") - } - return nil -} - -// TxDeclareCandidacy - struct for unbonding transactions -type TxDeclareCandidacy struct { - BondUpdate - Description -} - -// NewTxDeclareCandidacy - new TxDeclareCandidacy -func NewTxDeclareCandidacy(bond coin.Coin, pubKey crypto.PubKey, description Description) sdk.Tx { - return TxDeclareCandidacy{ - BondUpdate{ - PubKey: pubKey, - Bond: bond, - }, - description, - }.Wrap() -} - -// Wrap - Wrap a Tx as a Basecoin Tx -func (tx TxDeclareCandidacy) Wrap() sdk.Tx { return sdk.Tx{tx} } - -// TxEditCandidacy - struct for editing a candidate -type TxEditCandidacy struct { - PubKey crypto.PubKey `json:"pub_key"` - Description -} - -// NewTxEditCandidacy - new TxEditCandidacy -func NewTxEditCandidacy(pubKey crypto.PubKey, description Description) sdk.Tx { - return TxEditCandidacy{ - PubKey: pubKey, - Description: description, - }.Wrap() -} - -// Wrap - Wrap a Tx as a Basecoin Tx -func (tx TxEditCandidacy) Wrap() sdk.Tx { return sdk.Tx{tx} } - -// ValidateBasic - Check for non-empty candidate, -func (tx TxEditCandidacy) ValidateBasic() error { - if tx.PubKey.Empty() { - return errCandidateEmpty - } - - empty := Description{} - if tx.Description == empty { - return fmt.Errorf("Transaction must include some information to modify") - } - return nil -} - -// TxDelegate - struct for bonding transactions -type TxDelegate struct{ BondUpdate } - -// NewTxDelegate - new TxDelegate -func NewTxDelegate(bond coin.Coin, pubKey crypto.PubKey) sdk.Tx { - return TxDelegate{BondUpdate{ - PubKey: pubKey, - Bond: bond, - }}.Wrap() -} - -// Wrap - Wrap a Tx as a Basecoin Tx -func (tx TxDelegate) Wrap() sdk.Tx { return sdk.Tx{tx} } - -// TxUnbond - struct for unbonding transactions -type TxUnbond struct { - PubKey crypto.PubKey `json:"pub_key"` - Shares uint64 `json:"amount"` -} - -// NewTxUnbond - new TxUnbond -func NewTxUnbond(shares uint64, pubKey crypto.PubKey) sdk.Tx { - return TxUnbond{ - PubKey: pubKey, - Shares: shares, - }.Wrap() -} - -// Wrap - Wrap a Tx as a Basecoin Tx -func (tx TxUnbond) Wrap() sdk.Tx { return sdk.Tx{tx} } - -// ValidateBasic - Check for non-empty candidate, positive shares -func (tx TxUnbond) ValidateBasic() error { - if tx.PubKey.Empty() { - return errCandidateEmpty - } - - if tx.Shares == 0 { - return fmt.Errorf("Shares must be > 0") - } - return nil -} diff --git a/mongodb/mongodb.js b/mongodb/mongodb.js new file mode 100644 index 0000000..152e3bb --- /dev/null +++ b/mongodb/mongodb.js @@ -0,0 +1,69 @@ +// create collections +db.createCollection("account"); +db.createCollection("block"); +db.createCollection("stake_role_candidate"); +db.createCollection("stake_role_delegator"); +db.createCollection("sync_task"); +db.createCollection("tx_stake"); +db.createCollection("tx_common"); + +// create index +db.account.createIndex({"address": 1}, {"unique": true}); +db.block.createIndex({"height": -1}, {"unique": true}); + +db.stake_role_candidate.createIndex({"address": 1}, {"unique": true}); +db.stake_role_candidate.createIndex({"pub_key": 1}); + +db.stake_role_delegator.createIndex({"validator_addr": 1}); +db.stake_role_delegator.createIndex({"address": 1}); +db.stake_role_delegator.createIndex({"address": 1, "validator_addr": 1}, {"unique": true}); + +db.sync_task.createIndex({"chain_id": 1}, {"unique": true}); + +db.tx_stake.createIndex({"height": -1}); +db.tx_stake.createIndex({"time": -1}); +db.tx_stake.createIndex({"tx_hash": 1}); +db.tx_stake.createIndex({"stake_tx.tx_hash": 1}); +db.tx_stake.createIndex({"description.moniker": 1}); +db.tx_stake.createIndex({"from": 1}); +db.tx_stake.createIndex({"to": 1}); +db.tx_stake.createIndex({"pub_key": 1}); +db.tx_stake.createIndex({"type": 1}); +db.tx_stake.createIndex({"status": 1}); +db.tx_stake.createIndex({"from": 1, "to": 1, "type": 1, "status": 1, "time": -1}); + +db.tx_common.createIndex({"height": -1}); +db.tx_common.createIndex({"time": -1}); +db.tx_common.createIndex({"tx_hash": 1}, {"unique": true}); +db.tx_common.createIndex({"from": 1}); +db.tx_common.createIndex({"to": 1}); +db.tx_common.createIndex({"type": 1}); +db.tx_common.createIndex({"status": 1}); + +// drop collection +// db.account.drop(); +// db.block.drop(); +// db.stake_role_candidate.drop(); +// db.stake_role_delegator.drop(); +// db.sync_task.drop(); +// db.tx_stake.drop(); +// db.tx_common.drop(); +// +// remove collection data +// db.account.remove({}); +// db.block.remove({}); +// db.stake_role_candidate.remove({}); +// db.stake_role_delegator.remove({}); +// db.sync_task.remove({}); +// db.tx_stake.remove({}); +// db.tx_common.remove({}); + + + + + + + + + + diff --git a/service/handler/account.go b/service/handler/account.go new file mode 100644 index 0000000..19285e6 --- /dev/null +++ b/service/handler/account.go @@ -0,0 +1,174 @@ +package handler + +import ( + "github.com/irisnet/irishub-sync/module/logger" + "github.com/irisnet/irishub-sync/store" + "github.com/irisnet/irishub-sync/store/document" + "github.com/irisnet/irishub-sync/util/constant" + "github.com/irisnet/irishub-sync/util/helper" + "sync" + "time" +) + +// save account +func SaveAccount(docTx store.Docs, mutex sync.Mutex) { + var ( + address string + updateTime time.Time + height int64 + methodName = "SaveAccount: " + ) + logger.Info.Printf("Start %v\n", methodName) + + // save account + fun := func(address string, updateTime time.Time, height int64) { + account := document.Account{ + Address: address, + Time: updateTime, + Height: height, + } + + if err := store.Save(account); err != nil { + logger.Trace.Printf("%v Record exists, account is %v, err is %s\n", + methodName, account.Address, err.Error()) + } + } + + txType := GetTxType(docTx) + if txType == "" { + logger.Error.Printf("%v get docTx type failed, docTx is %v\n", + methodName, docTx) + return + } + + switch txType { + case constant.TxTypeBank: + docTx, r := docTx.(document.CommonTx) + if !r { + logger.Error.Printf("%v get docuemnt from docTx failed. docTx type is %v\n", + methodName, txType) + break + } + updateTime = docTx.Time + height = docTx.Height + + fun(docTx.From, updateTime, height) + fun(docTx.To, updateTime, height) + break + case constant.TxTypeStakeCreate: + docTx, r := docTx.(document.StakeTxDeclareCandidacy) + if !r { + logger.Error.Printf("%v get docuemnt from docTx failed. docTx type is %v\n", + methodName, txType) + break + } + address = docTx.ValidatorAddr + updateTime = docTx.Time + height = docTx.Height + + fun(address, updateTime, height) + break + case constant.TxTypeStakeEdit: + docTx, r := docTx.(document.StakeTxEditCandidacy) + if !r { + logger.Error.Printf("%v get docuemnt from docTx failed. docTx type is %v\n", + methodName, txType) + break + } + address = docTx.ValidatorAddr + updateTime = docTx.Time + height = docTx.Height + + fun(address, updateTime, height) + break + case constant.TxTypeStakeDelegate, constant.TxTypeStakeUnbond: + stakeTx, r := docTx.(document.StakeTx) + if !r { + logger.Error.Printf("%v get docuemnt from docTx failed. docTx type is %v\n", + methodName, txType) + break + } + updateTime = stakeTx.Time + height = stakeTx.Height + + fun(stakeTx.ValidatorAddr, updateTime, height) + fun(stakeTx.DelegatorAddr, updateTime, height) + break + } + + logger.Info.Printf("End %v\n", methodName) +} + +// update account balance +func UpdateBalance(docTx store.Docs, mutex sync.Mutex) { + var ( + methodName = "UpdateBalance: " + ) + logger.Info.Printf("Start %v\n", methodName) + + fun := func(address string) { + account, err := document.QueryAccount(address) + if err != nil { + logger.Error.Printf("%v updateAccountBalance failed, account is %v and err is %v", + methodName, account, err.Error()) + return + } + + // query balance of account + account.Amount = helper.QueryAccountBalance(address) + if err := store.Update(account); err != nil { + logger.Error.Printf("%v account:[%q] balance update failed,%s\n", + methodName, account.Address, err) + } + } + + txType := GetTxType(docTx) + if txType == "" { + logger.Error.Printf("%v get docTx type failed, docTx is %v\n", + methodName, docTx) + return + } + + switch txType { + case constant.TxTypeBank: + docTx, r := docTx.(document.CommonTx) + if !r { + logger.Error.Printf("%v get docuemnt from docTx failed. docTx type is %v\n", + methodName, txType) + break + } + fun(docTx.From) + fun(docTx.To) + break + case constant.TxTypeStakeCreate: + docTx, r := docTx.(document.StakeTxDeclareCandidacy) + if !r { + logger.Error.Printf("%v get docuemnt from docTx failed. docTx type is %v\n", + methodName, txType) + break + } + fun(docTx.ValidatorAddr) + break + case constant.TxTypeStakeEdit: + docTx, r := docTx.(document.StakeTxEditCandidacy) + if !r { + logger.Error.Printf("%v get docuemnt from docTx failed. docTx type is %v\n", + methodName, txType) + break + } + fun(docTx.ValidatorAddr) + break + case constant.TxTypeStakeDelegate, constant.TxTypeStakeUnbond: + docTx, r := docTx.(document.StakeTx) + if !r { + logger.Error.Printf("%v get docuemnt from docTx failed. docTx type is %v\n", + methodName, txType) + break + } + fun(docTx.ValidatorAddr) + fun(docTx.DelegatorAddr) + break + } + + logger.Info.Printf("End %v\n", methodName) +} diff --git a/service/handler/account_test.go b/service/handler/account_test.go new file mode 100644 index 0000000..1dd6229 --- /dev/null +++ b/service/handler/account_test.go @@ -0,0 +1,124 @@ +package handler + +import ( + "sync" + "testing" + + "github.com/irisnet/irishub-sync/store" +) + +func TestSaveAccount(t *testing.T) { + docTxBank := buildDocData(1762) + docTxStakeCreate := buildDocData(46910) + docTxStakeEdit := buildDocData(49388) + docTxStakeDelegate := buildDocData(47349) + docTxStakeUnBond := buildDocData(34241) + + type args struct { + docTx store.Docs + mutex sync.Mutex + } + tests := []struct { + name string + args args + }{ + { + name: "tx bank", + args: args{ + docTx: docTxBank, + mutex: sync.Mutex{}, + }, + }, + { + name: "tx stake/create", + args: args{ + docTx: docTxStakeCreate, + mutex: sync.Mutex{}, + }, + }, + { + name: "tx stake/edit", + args: args{ + docTx: docTxStakeEdit, + mutex: sync.Mutex{}, + }, + }, + { + name: "tx stake/delegate", + args: args{ + docTx: docTxStakeDelegate, + mutex: sync.Mutex{}, + }, + }, + { + name: "tx stake/unbond", + args: args{ + docTx: docTxStakeUnBond, + mutex: sync.Mutex{}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + SaveAccount(tt.args.docTx, tt.args.mutex) + }) + } +} + +func TestUpdateBalance(t *testing.T) { + docTxBank := buildDocData(1762) + docTxStakeCreate := buildDocData(46910) + docTxStakeEdit := buildDocData(49388) + docTxStakeDelegate := buildDocData(47349) + docTxStakeUnBond := buildDocData(34241) + + type args struct { + docTx store.Docs + mutex sync.Mutex + } + tests := []struct { + name string + args args + }{ + { + name: "tx bank", + args: args{ + docTx: docTxBank, + mutex: sync.Mutex{}, + }, + }, + { + name: "tx stake/create", + args: args{ + docTx: docTxStakeCreate, + mutex: sync.Mutex{}, + }, + }, + { + name: "tx stake/edit", + args: args{ + docTx: docTxStakeEdit, + mutex: sync.Mutex{}, + }, + }, + { + name: "tx stake/delegate", + args: args{ + docTx: docTxStakeDelegate, + mutex: sync.Mutex{}, + }, + }, + { + name: "tx stake/unbond", + args: args{ + docTx: docTxStakeUnBond, + mutex: sync.Mutex{}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + UpdateBalance(tt.args.docTx, tt.args.mutex) + }) + } +} diff --git a/service/handler/block.go b/service/handler/block.go new file mode 100644 index 0000000..66133fd --- /dev/null +++ b/service/handler/block.go @@ -0,0 +1,113 @@ +package handler + +import ( + "encoding/json" + "github.com/irisnet/irishub-sync/module/codec" + "github.com/irisnet/irishub-sync/module/logger" + "github.com/irisnet/irishub-sync/store" + "github.com/irisnet/irishub-sync/store/document" + "github.com/irisnet/irishub-sync/util/helper" + "github.com/tendermint/tendermint/types" +) + +func SaveBlock(meta *types.BlockMeta, block *types.Block, validators []*types.Validator) { + + hexFunc := func(bytes []byte) string { + return helper.BuildHex(bytes) + } + + docBlock := document.Block{ + Height: meta.Header.Height, + Hash: hexFunc(meta.BlockID.Hash), + Time: meta.Header.Time, + NumTxs: meta.Header.NumTxs, + } + + lastBlockId := document.BlockID{ + Hash: hexFunc(meta.Header.LastBlockID.Hash), + PartsHeader: document.PartSetHeader{ + Total: meta.Header.LastBlockID.PartsHeader.Total, + Hash: hexFunc(meta.Header.LastBlockID.PartsHeader.Hash), + }, + } + + // blockMeta + blockMeta := document.BlockMeta{ + BlockID: document.BlockID{ + Hash: hexFunc(meta.BlockID.Hash), + PartsHeader: document.PartSetHeader{ + Total: meta.BlockID.PartsHeader.Total, + Hash: hexFunc(meta.BlockID.PartsHeader.Hash), + }, + }, + Header: document.Header{ + ChainID: meta.Header.ChainID, + Height: meta.Header.Height, + Time: meta.Header.Time, + NumTxs: meta.Header.NumTxs, + LastBlockID: lastBlockId, + TotalTxs: meta.Header.TotalTxs, + LastCommitHash: hexFunc(meta.Header.LastCommitHash), + DataHash: hexFunc(meta.Header.DataHash), + ValidatorsHash: hexFunc(meta.Header.ValidatorsHash), + ConsensusHash: hexFunc(meta.Header.ConsensusHash), + AppHash: hexFunc(meta.Header.AppHash), + LastResultsHash: hexFunc(meta.Header.LastResultsHash), + EvidenceHash: hexFunc(meta.Header.EvidenceHash), + }, + } + + // block + var ( + preCommits []document.Vote + ) + + if len(block.LastCommit.Precommits) > 0 { + for _, v := range block.LastCommit.Precommits { + var sig document.Signature + out, _ := codec.Cdc.MarshalJSON(v.Signature) + json.Unmarshal(out, &sig) + preCommit := document.Vote{ + ValidatorAddress: v.ValidatorAddress.String(), + ValidatorIndex: v.ValidatorIndex, + Height: v.Height, + Round: v.Round, + Timestamp: v.Timestamp, + Type: v.Type, + BlockID: lastBlockId, + Signature: sig, + } + preCommits = append(preCommits, preCommit) + } + } + + blockContent := document.BlockContent{ + LastCommit: document.Commit{ + BlockID: lastBlockId, + Precommits: preCommits, + }, + } + + // validators + var vals []document.Validator + if len(validators) > 0 { + for _, v := range validators { + validator := document.Validator{ + Address: v.Address.String(), + VotingPower: v.VotingPower, + Accum: v.Accum, + PubKey: hexFunc(v.PubKey.Bytes()), + } + vals = append(vals, validator) + } + } + + docBlock.Meta = blockMeta + docBlock.Block = blockContent + docBlock.Validators = vals + + err := store.Save(docBlock) + if err != nil { + logger.Error.Println(err) + } +} diff --git a/service/handler/block_test.go b/service/handler/block_test.go new file mode 100644 index 0000000..32928af --- /dev/null +++ b/service/handler/block_test.go @@ -0,0 +1,75 @@ +package handler + +import ( + "testing" + + "github.com/irisnet/irishub-sync/module/logger" + "github.com/irisnet/irishub-sync/util/helper" + "github.com/tendermint/tendermint/types" +) + +func buildBlock(blockHeight int64) (*types.BlockMeta, *types.Block, []*types.Validator) { + + client := helper.GetClient() + // release client + defer client.Release() + + block, err := client.Client.Block(&blockHeight) + + if err != nil { + logger.Error.Fatalln(err) + } + + validators, err := client.Client.Validators(&blockHeight) + if err != nil { + logger.Error.Fatalln(err) + } + + return block.BlockMeta, block.Block, validators.Validators +} + +func TestSaveBlock(t *testing.T) { + meta1, block1, vals1 := buildBlock(28558) + meta2, block2, vals2 := buildBlock(96319) + meta3, block3, vals3 := buildBlock(34241) + + type args struct { + meta *types.BlockMeta + block *types.Block + vals []*types.Validator + } + tests := []struct { + name string + args args + }{ + { + name: "test save block", + args: args{ + meta: meta1, + block: block1, + vals: vals1, + }, + }, + { + name: "test save block", + args: args{ + meta: meta2, + block: block2, + vals: vals2, + }, + }, + { + name: "test save block", + args: args{ + meta: meta3, + block: block3, + vals: vals3, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + SaveBlock(tt.args.meta, tt.args.block, tt.args.vals) + }) + } +} diff --git a/service/handler/tx.go b/service/handler/tx.go new file mode 100644 index 0000000..743e16c --- /dev/null +++ b/service/handler/tx.go @@ -0,0 +1,285 @@ +package handler + +import ( + "github.com/irisnet/irishub-sync/module/logger" + "github.com/irisnet/irishub-sync/store" + "github.com/irisnet/irishub-sync/store/document" + "github.com/irisnet/irishub-sync/util/constant" + "sync" +) + +// save Tx document into collection +func SaveTx(docTx store.Docs, mutex sync.Mutex) { + var ( + methodName = "SaveTx: " + ) + + // save docTx document into database + storeTxDocFunc := func(doc store.Docs) { + err := store.Save(doc) + if err != nil { + logger.Error.Printf("%v Save failed. doc is %+v, err is %v", + methodName, doc, err.Error()) + } + } + + // save common docTx document + saveCommonTx := func(commonTx document.CommonTx) { + err := store.Save(commonTx) + if err != nil { + logger.Error.Printf("%v Save commonTx failed. doc is %+v, err is %v", + methodName, commonTx, err.Error()) + } + } + + txType := GetTxType(docTx) + if txType == "" { + logger.Error.Printf("%v get docTx type failed, docTx is %v\n", + methodName, docTx) + return + } + + saveCommonTx(buildCommonTxData(docTx, txType)) + + switch txType { + case constant.TxTypeBank: + break + case constant.TxTypeStakeCreate: + docTx, r := docTx.(document.StakeTxDeclareCandidacy) + if !r { + logger.Error.Printf("%v get docuemnt from docTx failed. docTx type is %v\n", + methodName, txType) + break + } + storeTxDocFunc(docTx) + + mutex.Lock() + logger.Info.Printf("%v saveOrUpdate cndidates get lock\n", methodName) + + cd, err := document.QueryCandidateByAddress(docTx.ValidatorAddr) + + candidate := document.Candidate{ + Address: docTx.ValidatorAddr, + PubKey: docTx.PubKey, + } + + if err == nil && cd.PubKey == "" { + // candidate exist + logger.Warning.Printf("%v Replace candidate from %+v to %+v\n", methodName, cd, candidate) + // TODO: in further share not equal amount + candidate.Shares = cd.Shares + docTx.Amount.Amount + candidate.VotingPower = int64(candidate.Shares) + + // description of candidate is empty + if cd.Description.Moniker == "" { + candidate.Description = docTx.Description + } + } else { + // candidate not exist + candidate.Shares = docTx.Amount.Amount + candidate.VotingPower = int64(candidate.Shares) + candidate.Description = docTx.Description + } + candidate.UpdateTime = docTx.Time + + store.SaveOrUpdate(candidate) + + mutex.Unlock() + logger.Info.Printf("%v saveOrUpdate cndidates release lock\n", methodName) + break + + case constant.TxTypeStakeEdit: + docTx, r := docTx.(document.StakeTxEditCandidacy) + if !r { + logger.Error.Printf("%v get docuemnt from docTx failed. docTx type is %v\n", + methodName, txType) + break + } + storeTxDocFunc(docTx) + + mutex.Lock() + logger.Info.Printf("%v saveOrUpdate cndidates get lock\n", methodName) + + cd, err := document.QueryCandidateByAddress(docTx.ValidatorAddr) + + var candidate document.Candidate + + if err != nil { + // candidate not exist + candidate = document.Candidate{ + Address: docTx.ValidatorAddr, + Description: docTx.Description, + } + } else { + // candidate exist + cd.Description = docTx.Description + candidate = cd + } + + store.SaveOrUpdate(candidate) + + mutex.Unlock() + logger.Info.Printf("%v saveOrUpdate cndidates release lock\n", methodName) + break + case constant.TxTypeStakeDelegate: + docTx, r := docTx.(document.StakeTx) + if !r { + logger.Error.Printf("%v get docuemnt from docTx failed. docTx type is %v\n", + methodName, txType) + break + } + storeTxDocFunc(docTx) + + mutex.Lock() + logger.Info.Printf("%v saveOrUpdate docTx type %v get lock\n", + methodName, txType) + + candidate, err := document.QueryCandidateByAddress(docTx.ValidatorAddr) + // candidate is not exist + if err != nil { + logger.Warning.Printf("%v candidate is not exist while delegate, addrdelegator = %s ,addrvalidator = %s\n", + methodName, docTx.DelegatorAddr, docTx.ValidatorAddr) + candidate = document.Candidate{ + Address: docTx.ValidatorAddr, + } + } + + delegator, err := document.QueryDelegatorByAddressAndValAddr(docTx.DelegatorAddr, docTx.ValidatorAddr) + // delegator is not exist + if err != nil { + delegator = document.Delegator{ + Address: docTx.DelegatorAddr, + ValidatorAddr: docTx.ValidatorAddr, + } + } + // TODO: in further share not equal amount + delegator.Shares += docTx.Amount.Amount + delegator.UpdateTime = docTx.Time + store.SaveOrUpdate(delegator) + + candidate.Shares += docTx.Amount.Amount + candidate.VotingPower += int64(docTx.Amount.Amount) + candidate.UpdateTime = docTx.Time + store.SaveOrUpdate(candidate) + + mutex.Unlock() + logger.Info.Printf("%v saveOrUpdate docTx type %v release lock\n", + methodName, txType) + break + + case constant.TxTypeStakeUnbond: + docTx, r := docTx.(document.StakeTx) + if !r { + logger.Error.Printf("%v get docuemnt from docTx failed. docTx type is %v\n", + methodName, txType) + break + } + storeTxDocFunc(docTx) + + mutex.Lock() + logger.Info.Printf("%v saveOrUpdate docTx type %v get lock\n", + methodName, txType) + + delegator, err := document.QueryDelegatorByAddressAndValAddr(docTx.DelegatorAddr, docTx.ValidatorAddr) + // delegator is not exist + if err != nil { + logger.Warning.Printf("%v delegator is not exist while unBond,add = %s,valAddr=%s\n", + methodName, docTx.DelegatorAddr, docTx.ValidatorAddr) + delegator = document.Delegator{ + Address: docTx.DelegatorAddr, + ValidatorAddr: docTx.ValidatorAddr, + } + } + delegator.Shares -= docTx.Amount.Amount + delegator.UpdateTime = docTx.Time + store.SaveOrUpdate(delegator) + + candidate, err2 := document.QueryCandidateByAddress(docTx.ValidatorAddr) + // candidate is not exist + if err2 != nil { + logger.Warning.Printf("%v candidate is not exist while unBond,add = %s,valAddr=%s\n", + methodName, docTx.DelegatorAddr, docTx.ValidatorAddr) + candidate = document.Candidate{ + Address: docTx.ValidatorAddr, + } + } + candidate.Shares -= docTx.Amount.Amount + candidate.VotingPower -= int64(docTx.Amount.Amount) + candidate.UpdateTime = docTx.Time + store.SaveOrUpdate(candidate) + + mutex.Unlock() + logger.Info.Printf("%v saveOrUpdate docTx type %v release lock\n", + methodName, txType) + break + } +} + +// build common tx data through parse tx +func buildCommonTxData(docTx store.Docs, txType string) document.CommonTx { + var commonTx document.CommonTx + + if txType == "" { + txType = GetTxType(docTx) + } + switch txType { + case constant.TxTypeBank: + doc := docTx.(document.CommonTx) + commonTx = document.CommonTx{ + TxHash: doc.TxHash, + Time: doc.Time, + Height: doc.Height, + From: doc.From, + To: doc.To, + Amount: doc.Amount, + Type: doc.Type, + Fee: doc.Fee, + Status: doc.Status, + } + break + case constant.TxTypeStakeCreate: + doc := docTx.(document.StakeTxDeclareCandidacy) + commonTx = document.CommonTx{ + TxHash: doc.TxHash, + Time: doc.Time, + Height: doc.Height, + From: doc.ValidatorAddr, + To: "", + Amount: []store.Coin{doc.Amount}, + Type: doc.Type, + Fee: doc.Fee, + Status: doc.Status, + } + break + case constant.TxTypeStakeEdit: + doc := docTx.(document.StakeTxEditCandidacy) + commonTx = document.CommonTx{ + TxHash: doc.TxHash, + Time: doc.Time, + Height: doc.Height, + From: doc.ValidatorAddr, + To: "", + Amount: []store.Coin{doc.Amount}, + Type: doc.Type, + Fee: doc.Fee, + Status: doc.Status, + } + break + case constant.TxTypeStakeDelegate, constant.TxTypeStakeUnbond: + doc := docTx.(document.StakeTx) + commonTx = document.CommonTx{ + TxHash: doc.TxHash, + Time: doc.Time, + Height: doc.Height, + From: doc.DelegatorAddr, + To: doc.ValidatorAddr, + Amount: []store.Coin{doc.Amount}, + Type: doc.Type, + Fee: doc.Fee, + Status: doc.Status, + } + break + } + + return commonTx +} diff --git a/service/handler/tx_test.go b/service/handler/tx_test.go new file mode 100644 index 0000000..b949c4a --- /dev/null +++ b/service/handler/tx_test.go @@ -0,0 +1,97 @@ +package handler + +import ( + "sync" + "testing" + + "github.com/irisnet/irishub-sync/module/codec" + "github.com/irisnet/irishub-sync/module/logger" + "github.com/irisnet/irishub-sync/store" + "github.com/irisnet/irishub-sync/util/helper" +) + +func init() { + helper.InitClientPool() + store.InitWithAuth() +} + +func buildDocData(blockHeight int64) store.Docs { + + client := helper.GetClient() + // release client + defer client.Release() + + block, err := client.Client.Block(&blockHeight) + + if err != nil { + logger.Error.Panic(err) + } + + if block.BlockMeta.Header.NumTxs > 0 { + txs := block.Block.Data.Txs + txByte := txs[0] + docTx := helper.ParseTx(codec.Cdc, txByte, block.Block) + + return docTx + + } + return nil +} + +func TestSaveTx(t *testing.T) { + docTxBank := buildDocData(1762) + docTxStakeCreate := buildDocData(46910) + docTxStakeEdit := buildDocData(49388) + docTxStakeDelegate := buildDocData(47349) + docTxStakeUnBond := buildDocData(96319) + + type args struct { + docTx store.Docs + mutex sync.Mutex + } + tests := []struct { + name string + args args + }{ + { + name: "tx bank", + args: args{ + docTx: docTxBank, + mutex: sync.Mutex{}, + }, + }, + { + name: "tx stake/create", + args: args{ + docTx: docTxStakeCreate, + mutex: sync.Mutex{}, + }, + }, + { + name: "tx stake/edit", + args: args{ + docTx: docTxStakeEdit, + mutex: sync.Mutex{}, + }, + }, + { + name: "tx stake/delegate", + args: args{ + docTx: docTxStakeDelegate, + mutex: sync.Mutex{}, + }, + }, + { + name: "tx stake/unbond", + args: args{ + docTx: docTxStakeUnBond, + mutex: sync.Mutex{}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + SaveTx(tt.args.docTx, tt.args.mutex) + }) + } +} diff --git a/service/handler/types.go b/service/handler/types.go new file mode 100644 index 0000000..8607136 --- /dev/null +++ b/service/handler/types.go @@ -0,0 +1,27 @@ +package handler + +import ( + "github.com/irisnet/irishub-sync/module/logger" + "github.com/irisnet/irishub-sync/store" + "github.com/irisnet/irishub-sync/util/helper" + "reflect" + "sync" +) + +// get tx type +func GetTxType(docTx store.Docs) string { + if !reflect.ValueOf(docTx).FieldByName("Type").IsValid() { + logger.Error.Printf("type which is field name of stake docTx is missed, docTx is %+v\n", + helper.ToJson(docTx)) + return "" + } + txType := reflect.ValueOf(docTx).FieldByName("Type").String() + + return txType +} + +func Handle(docTx store.Docs, mutex sync.Mutex, funChains []func(tx store.Docs, mutex sync.Mutex)) { + for _, fun := range funChains { + fun(docTx, mutex) + } +} diff --git a/service/sync.go b/service/sync.go new file mode 100644 index 0000000..e1ff783 --- /dev/null +++ b/service/sync.go @@ -0,0 +1,243 @@ +package service + +import ( + conf "github.com/irisnet/irishub-sync/conf/server" + "github.com/irisnet/irishub-sync/module/codec" + "github.com/irisnet/irishub-sync/module/logger" + "github.com/irisnet/irishub-sync/service/handler" + "github.com/irisnet/irishub-sync/store" + "github.com/irisnet/irishub-sync/util/helper" + + "github.com/irisnet/irishub-sync/store/document" + "github.com/robfig/cron" + rpcClient "github.com/tendermint/tendermint/rpc/client" + ctypes "github.com/tendermint/tendermint/rpc/core/types" + + "github.com/tendermint/tendermint/types" + "sync" +) + +var ( + // how many block each goroutine need to sync when do fast sync + syncBlockNumFastSync = int64(conf.SyncBlockNumFastSync) + + // limit max goroutine + limitChan = make(chan int64, conf.SyncMaxGoroutine) + + mutex sync.Mutex + mutexWatchBlock sync.Mutex +) + +// start sync server +func Start() { + var ( + status *ctypes.ResultStatus + err error + i = 1 + ) + Init() + c := helper.GetClient().Client + + for { + logger.Info.Printf("Begin %v time fast sync task", i) + syncLatestHeight := fastSync(c) + status, err = c.Status() + if err != nil { + logger.Error.Printf("TmClient err and try again, %v\n", err.Error()) + c := helper.GetClient().Client + status, err = c.Status() + if err != nil { + logger.Error.Fatalf("TmClient err and exit, %v\n", err.Error()) + } + } + latestHeight := status.SyncInfo.LatestBlockHeight + if syncLatestHeight >= latestHeight-60 { + logger.Info.Println("All fast sync task complete!") + break + } + logger.Info.Printf("End %v time fast sync task", i) + i++ + } + + startCron(c) +} + +func Init() { + store.InitWithAuth() + + chainId := conf.ChainId + syncTask, err := document.QuerySyncTask() + + if err != nil { + if chainId == "" { + logger.Error.Fatalln("sync process start failed,chainId is empty") + } + syncTask = document.SyncTask{ + Height: 0, + ChainID: chainId, + } + store.Save(syncTask) + } + + // init client pool + helper.InitClientPool() +} + +// start cron scheduler +func startCron(client rpcClient.Client) { + spec := conf.SyncCron + c := cron.New() + c.AddFunc(spec, func() { + watchBlock(client) + }) + go c.Start() +} + +func watchBlock(c rpcClient.Client) { + mutexWatchBlock.Lock() + + syncTask, _ := document.QuerySyncTask() + status, _ := c.Status() + latestBlockHeight := status.SyncInfo.LatestBlockHeight + + funcChain := []func(tx store.Docs, mutex sync.Mutex){ + handler.SaveTx, handler.SaveAccount, handler.UpdateBalance, + } + + ch := make(chan int64) + limitChan <- 1 + + go syncBlock(syncTask.Height+1, latestBlockHeight, funcChain, ch, 0) + + select { + case <-ch: + logger.Info.Printf("Watch block, current height is %v \n", latestBlockHeight) + block, _ := c.Block(&latestBlockHeight) + syncTask.Height = block.Block.Height + syncTask.Time = block.Block.Time + err := store.Update(syncTask) + if err != nil { + logger.Error.Printf("Update syncTask fail, err is %v", + err.Error()) + } + } + + mutexWatchBlock.Unlock() +} + +// fast sync data from blockChain +func fastSync(c rpcClient.Client) int64 { + syncTaskDoc, _ := document.QuerySyncTask() + status, _ := c.Status() + latestBlockHeight := status.SyncInfo.LatestBlockHeight + + funcChain := []func(tx store.Docs, mutex sync.Mutex){ + handler.SaveTx, handler.SaveAccount, handler.UpdateBalance, + } + + ch := make(chan int64) + + goroutineNum := (latestBlockHeight - syncTaskDoc.Height) / syncBlockNumFastSync + + if goroutineNum == 0 { + goroutineNum = 20 + syncBlockNumFastSync = (latestBlockHeight - syncTaskDoc.Height) / goroutineNum + } + activeGoroutineNum := goroutineNum + + for i := int64(1); i <= goroutineNum; i++ { + limitChan <- i + var ( + start = syncTaskDoc.Height + (i-1)*syncBlockNumFastSync + 1 + end = syncTaskDoc.Height + i*syncBlockNumFastSync + ) + if i == goroutineNum { + end = latestBlockHeight + } + go syncBlock(start, end, funcChain, ch, i) + } + + for { + select { + case threadNo := <-ch: + activeGoroutineNum = activeGoroutineNum - 1 + logger.Info.Printf("ThreadNo[%d] is over and active thread num is %d\n", threadNo, activeGoroutineNum) + if activeGoroutineNum == 0 { + goto end + } + } + } + +end: + { + logger.Info.Println("This fastSync task complete!") + // update syncTask document + block, _ := c.Block(&latestBlockHeight) + syncTaskDoc.Height = block.Block.Height + syncTaskDoc.Time = block.Block.Time + err := store.Update(syncTaskDoc) + if err != nil { + logger.Error.Printf("Update syncTask fail, err is %v", + err.Error()) + } + return syncTaskDoc.Height + } +} + +func syncBlock(start int64, end int64, funcChain []func(tx store.Docs, mutex sync.Mutex), ch chan int64, threadNum int64) { + logger.Info.Printf("ThreadNo[%d] begin sync block from %d to %d\n", + threadNum, start, end) + + client := helper.GetClient() + // release client + defer client.Release() + + for j := start; j <= end; j++ { + block, err := client.Client.Block(&j) + if err != nil { + logger.Error.Printf("Invalid block height %d and err is %v, try again\n", j, err.Error()) + // try again + client2 := helper.GetClient() + block, err = client2.Client.Block(&j) + if err != nil { + ch <- threadNum + logger.Error.Fatalf("Invalid block height %d and err is %v\n", j, err.Error()) + } + } + if block.BlockMeta.Header.NumTxs > 0 { + txs := block.Block.Data.Txs + for _, txByte := range txs { + docTx := helper.ParseTx(codec.Cdc, txByte, block.Block) + txHash := helper.BuildHex(txByte.Hash()) + if txHash == "" { + logger.Warning.Printf("Tx has no hash, skip this tx."+ + ""+"tx is %v\n", helper.ToJson(docTx)) + continue + } + logger.Info.Printf("===========threadNo[%d] find tx, txHash=%s\n", threadNum, txHash) + + handler.Handle(docTx, mutex, funcChain) + } + } + + // get validatorSet at given height + var validators []*types.Validator + res, err := client.Client.Validators(&j) + if err != nil { + logger.Error.Printf("Can't get validatorSet at %v\n", j) + } else { + validators = res.Validators + } + + // save block info + handler.SaveBlock(block.BlockMeta, block.Block, validators) + } + + logger.Info.Printf("ThreadNo[%d] finish sync block from %d to %d\n", + threadNum, start, end) + + <-limitChan + ch <- threadNum + logger.Info.Printf("Send threadNum into channel: %v\n", threadNum) + +} diff --git a/service/sync_test.go b/service/sync_test.go new file mode 100644 index 0000000..949da08 --- /dev/null +++ b/service/sync_test.go @@ -0,0 +1,69 @@ +package service + +import ( + "github.com/robfig/cron" + "testing" + + conf "github.com/irisnet/irishub-sync/conf/server" + + "github.com/irisnet/irishub-sync/module/logger" + "sync" + "time" +) + +func TestStart(t *testing.T) { + var ( + limitChan chan int + unBufferChan chan int + ) + limitChan = make(chan int, 3) + unBufferChan = make(chan int) + goroutineNum := 5 + activeGoroutineNum := goroutineNum + for i := 1; i <= goroutineNum; i++ { + limitChan <- i + go func(goroutineNum int, ch chan int) { + logger.Info.Println("release limitChan") + <-limitChan + defer func() { + logger.Info.Printf("%v goroutine send data to channel\n", + goroutineNum) + ch <- goroutineNum + }() + + }(i, nil) + } + + for { + select { + case <-unBufferChan: + activeGoroutineNum = activeGoroutineNum - 1 + logger.Info.Printf("active goroutine num is %v", activeGoroutineNum) + if activeGoroutineNum == 0 { + logger.Info.Println("All goroutine complete") + break + } + } + } + +} + +func Test_startCron(t *testing.T) { + var wg sync.WaitGroup + var mutex sync.Mutex + wg.Add(2) + + spec := conf.SyncCron + c := cron.New() + c.AddFunc(spec, func() { + mutex.Lock() + var sleepSecond time.Duration + sleepSecond = 3 + time.Sleep(time.Second * sleepSecond) + logger.Info.Printf("awake up after %v second\n", sleepSecond) + mutex.Unlock() + }) + go c.Start() + + wg.Wait() +} diff --git a/store/document/account.go b/store/document/account.go new file mode 100644 index 0000000..a980159 --- /dev/null +++ b/store/document/account.go @@ -0,0 +1,43 @@ +package document + +import ( + "github.com/irisnet/irishub-sync/store" + "gopkg.in/mgo.v2" + "gopkg.in/mgo.v2/bson" + "time" +) + +const ( + CollectionNmAccount = "account" +) + +type Account struct { + Address string `bson:"address"` + Amount store.Coins `bson:"amount"` + Time time.Time `bson:"time"` + Height int64 `bson:"height"` +} + +func (a Account) Name() string { + return CollectionNmAccount +} + +func (a Account) PkKvPair() map[string]interface{} { + return bson.M{"address": a.Address} +} + +func QueryAccount(address string) (Account, error) { + var result Account + query := func(c *mgo.Collection) error { + err := c.Find(bson.M{"address": address}).Sort("-amount.amount").One(&result) + return err + } + + err := store.ExecCollection(CollectionNmAccount, query) + + if err != nil { + return result, err + } + + return result, nil +} diff --git a/store/document/block.go b/store/document/block.go new file mode 100644 index 0000000..5e373a8 --- /dev/null +++ b/store/document/block.go @@ -0,0 +1,104 @@ +package document + +import ( + "gopkg.in/mgo.v2/bson" + "time" +) + +const ( + CollectionNmBlock = "block" +) + +type Block struct { + Height int64 `bson:"height"` + Hash string `bson:"hash"` + Time time.Time `bson:"time"` + NumTxs int64 `bson:"num_txs"` + Meta BlockMeta `bson:"meta"` + Block BlockContent `bson:"block"` + Validators []Validator `bson:"validators"` +} + +type BlockMeta struct { + BlockID BlockID `bson:"block_id"` + Header Header `bson:"header"` +} + +type BlockID struct { + Hash string `bson:"hash"` + PartsHeader PartSetHeader `bson:"parts"` +} + +type PartSetHeader struct { + Total int `bson:"total"` + Hash string `bson:"hash"` +} + +type Header struct { + // basic block info + ChainID string `bson:"chain_id"` + Height int64 `bson:"height"` + Time time.Time `bson:"time"` + NumTxs int64 `bson:"num_txs"` + + // prev block info + LastBlockID BlockID `bson:"last_block_id"` + TotalTxs int64 `bson:"total_txs"` + + // hashes of block data + LastCommitHash string `bson:"last_commit_hash"` // commit from validators from the last block + DataHash string `bson:"data_hash"` // transactions + + // hashes from the app output from the prev block + ValidatorsHash string `bson:"validators_hash"` // validators for the current block + ConsensusHash string `bson:"consensus_hash"` // consensus params for current block + AppHash string `bson:"app_hash"` // state after txs from the previous block + LastResultsHash string `bson:"last_results_hash"` // root hash of all results from the txs from the previous block + + // consensus info + EvidenceHash string `bson:"evidence_hash"` // evidence included in the block +} + +type BlockContent struct { + LastCommit Commit `bson:"last_commit"` +} + +type Commit struct { + // NOTE: The Precommits are in order of address to preserve the bonded ValidatorSet order. + // Any peer with a block can gossip precommits by index with a peer without recalculating the + // active ValidatorSet. + BlockID BlockID `bson:"block_id"` + Precommits []Vote `bson:"precommits"` +} + +// Represents a prevote, precommit, or commit vote from validators for consensus. +type Vote struct { + ValidatorAddress string `bson:"validator_address"` + ValidatorIndex int `bson:"validator_index"` + Height int64 `bson:"height"` + Round int `bson:"round"` + Timestamp time.Time `bson:"timestamp"` + Type byte `bson:"type"` + BlockID BlockID `bson:"block_id"` // zero if vote is nil. + Signature Signature `bson:"signature"` +} + +type Signature struct { + Type string `bson:"type"` + Value string `bson:"value"` +} + +type Validator struct { + Address string `bson:"address"` + PubKey string `bson:"pub_key"` + VotingPower int64 `bson:"voting_power"` + Accum int64 `bson:"accum"` +} + +func (d Block) Name() string { + return CollectionNmBlock +} + +func (d Block) PkKvPair() map[string]interface{} { + return bson.M{"height": d.Height} +} diff --git a/model/store/document/doc.go b/store/document/doc.go similarity index 77% rename from model/store/document/doc.go rename to store/document/doc.go index 27705a9..04ac999 100644 --- a/model/store/document/doc.go +++ b/store/document/doc.go @@ -1,14 +1,14 @@ package document import ( - "github.com/irisnet/iris-sync-server/model/store" + "github.com/irisnet/irishub-sync/store" ) func init() { store.RegisterDocs(new(Account)) - store.RegisterDocs(new(CoinTx)) store.RegisterDocs(new(StakeTx)) store.RegisterDocs(new(StakeTxDeclareCandidacy)) + store.RegisterDocs(new(StakeTxEditCandidacy)) store.RegisterDocs(new(Candidate)) store.RegisterDocs(new(Delegator)) store.RegisterDocs(new(Block)) diff --git a/store/document/stake_role_candidate.go b/store/document/stake_role_candidate.go new file mode 100644 index 0000000..650107b --- /dev/null +++ b/store/document/stake_role_candidate.go @@ -0,0 +1,46 @@ +package document + +import ( + "errors" + "github.com/irisnet/irishub-sync/module/logger" + "github.com/irisnet/irishub-sync/store" + "gopkg.in/mgo.v2" + "gopkg.in/mgo.v2/bson" + "time" +) + +const ( + CollectionNmStakeRoleCandidate = "stake_role_candidate" +) + +type Candidate struct { + Address string `bson:"address"` // owner, identity key + PubKey string `bson:"pub_key"` + Shares int64 `bson:"shares"` + VotingPower int64 `bson:"voting_power"` // Voting power if pubKey is a considered a validator + Description Description `bson:"description"` // Description terms for the candidate + UpdateTime time.Time `bson:"update_time"` +} + +func (d Candidate) Name() string { + return CollectionNmStakeRoleCandidate +} + +func (d Candidate) PkKvPair() map[string]interface{} { + return bson.M{"address": d.Address} +} + +func QueryCandidateByAddress(address string) (Candidate, error) { + var result Candidate + query := func(c *mgo.Collection) error { + err := c.Find(bson.M{"address": address}).One(&result) + return err + } + + if store.ExecCollection(CollectionNmStakeRoleCandidate, query) != nil { + logger.Info.Println("candidate is Empty") + return result, errors.New("candidate is Empty") + } + + return result, nil +} diff --git a/store/document/stake_role_delegator.go b/store/document/stake_role_delegator.go new file mode 100644 index 0000000..d6ac316 --- /dev/null +++ b/store/document/stake_role_delegator.go @@ -0,0 +1,42 @@ +package document + +import ( + "errors" + "github.com/irisnet/irishub-sync/store" + "gopkg.in/mgo.v2" + "gopkg.in/mgo.v2/bson" + "time" +) + +const ( + CollectionNmStakeRoleDelegator = "stake_role_delegator" +) + +type Delegator struct { + Address string `bson:"address"` + ValidatorAddr string `bson:"validator_addr"` // validatorAddr + Shares int64 `bson:"shares"` + UpdateTime time.Time `bson:"update_time"` +} + +func (d Delegator) Name() string { + return CollectionNmStakeRoleDelegator +} + +func (d Delegator) PkKvPair() map[string]interface{} { + return bson.M{"address": d.Address, "validator_addr": d.ValidatorAddr} +} + +func QueryDelegatorByAddressAndValAddr(address string, valAddr string) (Delegator, error) { + var result Delegator + query := func(c *mgo.Collection) error { + err := c.Find(bson.M{"address": address, "validator_addr": valAddr}).Sort("-shares").One(&result) + return err + } + + if store.ExecCollection(CollectionNmStakeRoleDelegator, query) != nil { + return result, errors.New("delegator is Empty") + } + + return result, nil +} diff --git a/model/store/document/sync_task.go b/store/document/sync_task.go similarity index 75% rename from model/store/document/sync_task.go rename to store/document/sync_task.go index 61c8a7f..62052ba 100644 --- a/model/store/document/sync_task.go +++ b/store/document/sync_task.go @@ -1,7 +1,7 @@ package document import ( - "github.com/irisnet/iris-sync-server/model/store" + "github.com/irisnet/irishub-sync/store" "gopkg.in/mgo.v2" "gopkg.in/mgo.v2/bson" "time" @@ -26,17 +26,6 @@ func (c SyncTask) PkKvPair() map[string]interface{} { return bson.M{"chain_id": c.ChainID} } -func (c SyncTask) Index() []mgo.Index { - return []mgo.Index{ - { - Key: []string{"chain_id"}, - Unique: true, - DropDups: false, - Background: true, - }, - } -} - func QuerySyncTask() (SyncTask, error) { result := SyncTask{} diff --git a/store/document/tx_common.go b/store/document/tx_common.go new file mode 100644 index 0000000..dd49f71 --- /dev/null +++ b/store/document/tx_common.go @@ -0,0 +1,31 @@ +package document + +import ( + "github.com/irisnet/irishub-sync/store" + "gopkg.in/mgo.v2/bson" + "time" +) + +const ( + CollectionNmCommonTx = "tx_common" +) + +type CommonTx struct { + TxHash string `bson:"tx_hash"` + Time time.Time `bson:"time"` + Height int64 `bson:"height"` + From string `bson:"from"` + To string `bson:"to"` + Amount store.Coins `bson:"amount"` + Type string `bson:"type"` + Fee store.Fee `bson:"fee"` + Status string `bson:"status"` +} + +func (d CommonTx) Name() string { + return CollectionNmCommonTx +} + +func (d CommonTx) PkKvPair() map[string]interface{} { + return bson.M{"tx_hash": d.TxHash} +} diff --git a/store/document/tx_stake.go b/store/document/tx_stake.go new file mode 100644 index 0000000..8494d70 --- /dev/null +++ b/store/document/tx_stake.go @@ -0,0 +1,73 @@ +package document + +import ( + "github.com/irisnet/irishub-sync/store" + "gopkg.in/mgo.v2/bson" + "time" +) + +const ( + CollectionNmStakeTx = "tx_stake" + CollectionNmStakeTxDeclareCandidacy = CollectionNmStakeTx + CollectionNmStakeTxEditCandidacy = CollectionNmStakeTx +) + +// StakeTx +type StakeTx struct { + TxHash string `bson:"tx_hash"` + Time time.Time `bson:"time"` + Height int64 `bson:"height"` + DelegatorAddr string `bson:"from"` + ValidatorAddr string `bson:"to"` + PubKey string `bson:"pub_key"` + Type string `bson:"type"` + Amount store.Coin `bson:"amount"` + Fee store.Fee `bson:"fee"` + Status string `bson:"status"` +} + +func (c StakeTx) Name() string { + return CollectionNmStakeTx +} + +func (c StakeTx) PkKvPair() map[string]interface{} { + return bson.M{"tx_hash": c.TxHash} +} + +// ====== + +// Description +type Description struct { + Moniker string `bson:"moniker"` + Identity string `bson:"identity"` + Website string `bson:"website"` + Details string `bson:"details"` +} + +type StakeTxDeclareCandidacy struct { + StakeTx `bson:"stake_tx"` + Description `bson:"description"` +} + +func (s StakeTxDeclareCandidacy) Name() string { + return CollectionNmStakeTxDeclareCandidacy +} + +func (s StakeTxDeclareCandidacy) PkKvPair() map[string]interface{} { + return bson.M{"stake_tx.tx_hash": s.TxHash} +} + +// ====== + +type StakeTxEditCandidacy struct { + StakeTx `bson:"stake_tx"` + Description `bson:"description"` +} + +func (s StakeTxEditCandidacy) Name() string { + return CollectionNmStakeTxEditCandidacy +} + +func (s StakeTxEditCandidacy) PkKvPair() map[string]interface{} { + return bson.M{"stake_tx.tx_hash": s.TxHash} +} diff --git a/store/store.go b/store/store.go new file mode 100644 index 0000000..2fd7188 --- /dev/null +++ b/store/store.go @@ -0,0 +1,128 @@ +// init mongodb session and provide common functions + +package store + +import ( + "time" + + conf "github.com/irisnet/irishub-sync/conf/db" + "github.com/irisnet/irishub-sync/module/logger" + + "errors" + "fmt" + "gopkg.in/mgo.v2" + "gopkg.in/mgo.v2/bson" +) + +var ( + session *mgo.Session + docs []Docs +) + +func RegisterDocs(d Docs) { + docs = append(docs, d) +} + +//func Init() { +// if session == nil { +// url := fmt.Sprintf("mongodb://%s:%s", conf.Host, conf.Port) +// +// var err error +// session, err = mgo.Dial(url) +// if err != nil { +// logger.Error.Fatalln(err) +// } +// logger.Info.Printf("Mgo start on %s\n", url) +// session.SetMode(mgo.Monotonic, true) +// } +//} + +func InitWithAuth() { + addr := fmt.Sprintf("%s:%s", conf.Host, conf.Port) + addrs := []string{addr} + + dialInfo := &mgo.DialInfo{ + Addrs: addrs, // []string{"192.168.6.122"} + Database: conf.Database, + Username: conf.User, + Password: conf.Passwd, + Direct: false, + Timeout: time.Second * 10, + PoolLimit: 4096, // Session.SetPoolLimit + } + + var err error + session, err = mgo.DialWithInfo(dialInfo) + session.SetMode(mgo.Monotonic, true) + if err != nil { + logger.Error.Panicln(err) + } +} + +func getSession() *mgo.Session { + // max session num is 4096 + return session.Clone() +} + +// get collection object +func ExecCollection(collection string, s func(*mgo.Collection) error) error { + session := getSession() + defer session.Close() + c := session.DB(conf.Database).C(collection) + return s(c) +} + +func Find(collection string, query interface{}) *mgo.Query { + session := getSession() + defer session.Close() + c := session.DB(conf.Database).C(collection) + return c.Find(query) +} + +func Save(h Docs) error { + save := func(c *mgo.Collection) error { + pk := h.PkKvPair() + n, _ := c.Find(pk).Count() + if n >= 1 { + errMsg := fmt.Sprintf("Record existed while save %v, data is %+v\n", + h.Name(), h) + return errors.New(errMsg) + } + // logger.Info.Printf("insert %s %+v\n", h.Name(), h) + return c.Insert(h) + } + return ExecCollection(h.Name(), save) +} + +func SaveOrUpdate(h Docs) error { + save := func(c *mgo.Collection) error { + n, err := c.Find(h.PkKvPair()).Count() + if err != nil { + logger.Error.Printf("Count:%d err:%+v\n", n, err) + } + + if n >= 1 { + return Update(h) + } + // logger.Trace.Printf("insert %s %+v\n", h.Name(), h) + return c.Insert(h) + } + + return ExecCollection(h.Name(), save) +} + +func Update(h Docs) error { + update := func(c *mgo.Collection) error { + key := h.PkKvPair() + // logger.Trace.Printf("update %s set %+v where %+v\n", h.Name(), h, key) + return c.Update(key, h) + } + return ExecCollection(h.Name(), update) +} + +func Query(collectionName string, query bson.M, sort string, fields bson.M, skip int, limit int) (results []interface{}, err error) { + exop := func(c *mgo.Collection) error { + return c.Find(query).Sort(sort).Select(fields).Skip(skip).Limit(limit).All(&results) + } + return results, ExecCollection(collectionName, exop) +} diff --git a/store/store_test.go b/store/store_test.go new file mode 100644 index 0000000..bd1d857 --- /dev/null +++ b/store/store_test.go @@ -0,0 +1,20 @@ +// init mongodb session and provide common functions + +package store + +import "testing" + +func TestInitWithAuth(t *testing.T) { + tests := []struct { + name string + }{ + { + name: "tets initWithAuth", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + InitWithAuth() + }) + } +} diff --git a/model/store/types.go b/store/types.go similarity index 55% rename from model/store/types.go rename to store/types.go index dabb41f..eaea9c1 100644 --- a/model/store/types.go +++ b/store/types.go @@ -2,15 +2,21 @@ package store -import ( - "gopkg.in/mgo.v2" -) - type Docs interface { // collection name Name() string // primary key pair(used to find a unique record) PkKvPair() map[string]interface{} - // add index for collection - Index() []mgo.Index +} + +type Coin struct { + Denom string `json:"denom"` + Amount int64 `json:"amount"` +} + +type Coins []Coin + +type Fee struct { + Amount Coins + Gas int64 } diff --git a/sync/handler.go b/sync/handler.go deleted file mode 100644 index f0f8249..0000000 --- a/sync/handler.go +++ /dev/null @@ -1,239 +0,0 @@ -package sync - -import ( - - "reflect" - "time" - "sync" - - "github.com/irisnet/iris-sync-server/model/store" - "github.com/irisnet/iris-sync-server/module/logger" - "github.com/irisnet/iris-sync-server/module/stake" - "github.com/irisnet/iris-sync-server/util/helper" - "github.com/irisnet/iris-sync-server/model/store/document" - "github.com/irisnet/iris-sync-server/util/constant" - -) - -var ( - delay = false - mutex sync.Mutex -) - - -func handle(tx store.Docs, funChains []func(tx store.Docs)) { - for _, fun := range funChains { - fun(tx) - } -} - -// save Tx document into collection -func saveTx(tx store.Docs) { - err := store.Save(tx) - - if err != nil { - logger.Error.Println(err) - } - - mutex.Lock() - - { - if tx.Name() == document.CollectionNmStakeTx { - if !reflect.ValueOf(tx).FieldByName("Type").IsValid() { - logger.Error.Println("type which is field name of stake tx is missed") - return - } - stakeType := constant.TxTypeStake + "/" + reflect.ValueOf(tx).FieldByName("Type").String() - - switch stakeType { - case stake.TypeTxDeclareCandidacy: - stakeTxDeclareCandidacy, _ := tx.(document.StakeTxDeclareCandidacy) - candidate, err := document.QueryCandidateByPubkey(stakeTxDeclareCandidacy.PubKey) - // new candidate - if err != nil { - candidate = document.Candidate { - Address: stakeTxDeclareCandidacy.From, - PubKey: stakeTxDeclareCandidacy.PubKey, - Description: stakeTxDeclareCandidacy.Description, - } - } - // TODO: in further share not equal amount - candidate.Shares += stakeTxDeclareCandidacy.Amount.Amount - candidate.VotingPower += uint64(stakeTxDeclareCandidacy.Amount.Amount) - store.SaveOrUpdate(candidate) - break - case stake.TypeTxDelegate: - stakeTx, _ := tx.(document.StakeTx) - candidate, err := document.QueryCandidateByPubkey(stakeTx.PubKey) - // candidate is not exist - if err != nil { - logger.Error.Printf("candidate is not exist while delegate, add = %s ,pub_key = %s\n", - stakeTx.From, stakeTx.PubKey) - return - } - - delegator, err := document.QueryDelegatorByAddressAndPubkey(stakeTx.From, stakeTx.PubKey) - // delegator is not exist - if err != nil { - delegator = document.Delegator{ - Address: stakeTx.From, - PubKey: stakeTx.PubKey, - } - } - // TODO: in further share not equal amount - delegator.Shares += stakeTx.Amount.Amount - store.SaveOrUpdate(delegator) - - candidate.Shares += stakeTx.Amount.Amount - candidate.VotingPower += uint64(stakeTx.Amount.Amount) - store.SaveOrUpdate(candidate) - break - case stake.TypeTxUnbond: - stakeTx, _ := tx.(document.StakeTx) - delegator, err := document.QueryDelegatorByAddressAndPubkey(stakeTx.From, stakeTx.PubKey) - // delegator is not exist - if err != nil { - logger.Info.Printf("delegator is not exist while unBond,add = %s,pub_key=%s\n", - stakeTx.From, stakeTx.PubKey) - return - } - delegator.Shares -= stakeTx.Amount.Amount - store.Update(delegator) - - candidate, err2 := document.QueryCandidateByPubkey(stakeTx.PubKey) - // candidate is not exist - if err2 != nil { - logger.Info.Printf("candidate is not exist while unBond,add = %s,pub_key=%s\n", - stakeTx.From, stakeTx.PubKey) - return - } - candidate.Shares -= stakeTx.Amount.Amount - candidate.VotingPower -= uint64(stakeTx.Amount.Amount) - store.Update(candidate) - break - } - - } - - } - - mutex.Unlock() -} - -func saveOrUpdateAccount(tx store.Docs) { - var ( - address string - updateTime time.Time - height int64 - ) - - fun := func(address string, updateTime time.Time, height int64) { - account := document.Account{ - Address: address, - Time: updateTime, - Height: height, - } - - if err := store.SaveOrUpdate(account); err != nil { - logger.Error.Printf("account:[%q] balance update failed,%s\n", account.Address, err) - } - } - - mutex.Lock() - - { - switch tx.Name() { - case document.CollectionNmCoinTx: - coinTx, _ := tx.(document.CoinTx) - updateTime = coinTx.Time - height = coinTx.Height - - fun(coinTx.From, updateTime, height) - fun(coinTx.To, updateTime, height) - break - case document.CollectionNmStakeTx: - if !reflect.ValueOf(tx).FieldByName("Type").IsValid() { - logger.Error.Println("type which is field name of stake tx is missed") - return - } - stakeType := constant.TxTypeStake + "/" + reflect.ValueOf(tx).FieldByName("Type").String() - - switch stakeType { - case stake.TypeTxDeclareCandidacy: - stakeTxDeclareCandidacy, _ := tx.(document.StakeTxDeclareCandidacy) - address = stakeTxDeclareCandidacy.From - updateTime = stakeTxDeclareCandidacy.Time - height = stakeTxDeclareCandidacy.Height - break - case stake.TypeTxEditCandidacy: - break - case stake.TypeTxDelegate, stake.TypeTxUnbond: - stakeTx, _ := tx.(document.StakeTx) - address = stakeTx.From - updateTime = stakeTx.Time - height = stakeTx.Height - break - } - fun(address, updateTime, height) - } - - } - - mutex.Unlock() - -} - -func updateAccountBalance(tx store.Docs) { - var ( - address string - ) - fun := func(address string) { - account, err := document.QueryAccount(address) - if err != nil { - return - } - // query balance of account - ac := helper.QueryAccountBalance(address, delay) - account.Amount = ac.Coins - if err := store.Update(account); err != nil { - logger.Error.Printf("account:[%q] balance update failed,%s\n", account.Address, err) - } - } - - mutex.Lock() - - { - switch tx.Name() { - case document.CollectionNmCoinTx: - coinTx, _ := tx.(document.CoinTx) - fun(coinTx.From) - fun(coinTx.To) - break - case document.CollectionNmStakeTx: - if !reflect.ValueOf(tx).FieldByName("Type").IsValid() { - logger.Error.Println("type which is field name of stake tx is missed") - return - } - stakeType := constant.TxTypeStake + "/" + reflect.ValueOf(tx).FieldByName("Type").String() - - switch stakeType { - case stake.TypeTxDeclareCandidacy: - stakeTxDeclareCandidacy, _ := tx.(document.StakeTxDeclareCandidacy) - address = stakeTxDeclareCandidacy.From - break - case stake.TypeTxEditCandidacy: - break - case stake.TypeTxDelegate, stake.TypeTxUnbond: - stakeTx, _ := tx.(document.StakeTx) - address = stakeTx.From - break - } - fun(address) - break - } - } - - mutex.Unlock() - - -} diff --git a/sync/handler_test.go b/sync/handler_test.go deleted file mode 100644 index a31456d..0000000 --- a/sync/handler_test.go +++ /dev/null @@ -1,193 +0,0 @@ -package sync - -import ( - "testing" - "github.com/irisnet/iris-sync-server/model/store" - "github.com/irisnet/iris-sync-server/util/helper" - "github.com/irisnet/iris-sync-server/module/logger" - "github.com/irisnet/iris-sync-server/util/constant" - "github.com/irisnet/iris-sync-server/model/store/document" - "github.com/irisnet/iris-sync-server/module/stake" -) - -func init() { - helper.InitClientPool() - store.Init() -} - -func buildDocData(blockHeight int64) store.Docs { - - client := helper.GetClient() - // release client - defer client.Release() - - block, err := client.Client.Block(&blockHeight) - - if err != nil { - logger.Error.Panic(err) - } - - if block.BlockMeta.Header.NumTxs > 0 { - txs := block.Block.Data.Txs - txByte := txs[0] - txType, tx := helper.ParseTx(txByte) - - switch txType { - case constant.TxTypeCoin: - coinTx, _ := tx.(document.CoinTx) - coinTx.Height = block.Block.Height - coinTx.Time = block.Block.Time - return coinTx - case stake.TypeTxDeclareCandidacy: - stakeTxDeclareCandidacy, _ := tx.(document.StakeTxDeclareCandidacy) - stakeTxDeclareCandidacy.Height = block.Block.Height - stakeTxDeclareCandidacy.Time = block.Block.Time - return stakeTxDeclareCandidacy - case stake.TypeTxEditCandidacy: - break - case stake.TypeTxDelegate, stake.TypeTxUnbond: - stakeTx, _ := tx.(document.StakeTx) - stakeTx.Height = block.Block.Height - stakeTx.Time = block.Block.Time - return stakeTx - } - - } - return nil -} - -func Test_saveTx(t *testing.T) { - - docTxCoin := buildDocData(12453) - docTxStakeDeclareCandidacy := buildDocData(19073) - docTxStakeDelegate := buildDocData(13725) - docTxStakeUnBond := buildDocData(14260) - - type args struct { - tx store.Docs - } - tests := []struct { - name string - args args - }{ - { - name:"save tx_coin", - args: struct{ tx store.Docs }{ - tx: docTxCoin,}, - }, - { - name:"save tx_stake_declareCandidacy", - args: struct{ tx store.Docs }{ - tx: docTxStakeDeclareCandidacy,}, - - }, - { - name:"save tx_stake_delegate", - args: struct{ tx store.Docs }{ - tx: docTxStakeDelegate,}, - - }, - { - name:"save tx_stake_unBond", - args: struct{ tx store.Docs }{ - tx: docTxStakeUnBond,}, - - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - saveTx(tt.args.tx) - }) - } -} - -func Test_saveOrUpdateAccount(t *testing.T) { - - docTxCoin := buildDocData(12453) - docTxStakeDeclareCandidacy := buildDocData(19073) - docTxStakeDelegate := buildDocData(13725) - docTxStakeUnBond := buildDocData(14260) - - type args struct { - tx store.Docs - } - tests := []struct { - name string - args args - }{ - { - name:"save tx_coin", - args: struct{ tx store.Docs }{ - tx: docTxCoin,}, - }, - { - name:"save tx_stake_declareCandidacy", - args: struct{ tx store.Docs }{ - tx: docTxStakeDeclareCandidacy,}, - - }, - { - name:"save tx_stake_delegate", - args: struct{ tx store.Docs }{ - tx: docTxStakeDelegate,}, - - }, - { - name:"save tx_stake_unBond", - args: struct{ tx store.Docs }{ - tx: docTxStakeUnBond,}, - - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - saveOrUpdateAccount(tt.args.tx) - }) - } -} - -func Test_updateAccountBalance(t *testing.T) { - - docTxCoin := buildDocData(12453) - docTxStakeDeclareCandidacy := buildDocData(19073) - docTxStakeDelegate := buildDocData(13725) - docTxStakeUnBond := buildDocData(14260) - - type args struct { - tx store.Docs - } - tests := []struct { - name string - args args - }{ - { - name:"tx_coin", - args: struct{ tx store.Docs }{ - tx: docTxCoin,}, - }, - { - name:"tx_stake_declareCandidacy", - args: struct{ tx store.Docs }{ - tx: docTxStakeDeclareCandidacy,}, - - }, - { - name:"tx_stake_delegate", - args: struct{ tx store.Docs }{ - tx: docTxStakeDelegate,}, - - }, - { - name:"tx_stake_unBond", - args: struct{ tx store.Docs }{ - tx: docTxStakeUnBond,}, - - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - updateAccountBalance(tt.args.tx) - }) - } -} diff --git a/sync/sync.go b/sync/sync.go deleted file mode 100644 index 0c398b5..0000000 --- a/sync/sync.go +++ /dev/null @@ -1,232 +0,0 @@ -package sync - -import ( - "strings" - "encoding/hex" - - conf "github.com/irisnet/iris-sync-server/conf/server" - "github.com/irisnet/iris-sync-server/model/store" - "github.com/irisnet/iris-sync-server/module/logger" - "github.com/irisnet/iris-sync-server/util/helper" - "github.com/irisnet/iris-sync-server/util/constant" - "github.com/irisnet/iris-sync-server/module/stake" - - "github.com/robfig/cron" - rpcClient "github.com/tendermint/tendermint/rpc/client" - "github.com/irisnet/iris-sync-server/model/store/document" - -) - -var ( - // how many block each goroutine need to sync when do fast sync - syncBlockNumFastSync = int64(conf.SyncBlockNumFastSync) - - // limit max goroutine - limitChan = make(chan int64, conf.SyncMaxGoroutine) -) - -// start sync server -func Start() { - InitServer() - c := helper.GetClient().Client - if err := fastSync(c); err != nil { - logger.Error.Fatalf("sync block failed,%v\n", err) - } - startCron(c) - //go watch(c) 监控的方式在启动同步过程中容易丢失区块 -} - -func InitServer() { - store.Init() - - chainId := conf.ChainId - syncTask, err := document.QuerySyncTask() - - if err != nil { - if chainId == "" { - logger.Error.Fatalln("sync process start failed,chainId is empty\n") - } - syncTask = document.SyncTask{ - Height: 0, - ChainID: chainId, - } - store.Save(syncTask) - } - - // init client pool - helper.InitClientPool() -} - -// start cron scheduler -func startCron(client rpcClient.Client) { - spec := conf.SyncCron - c := cron.New() - c.AddFunc(spec, func() { - watchBlock(client) - }) - go c.Start() -} - -func watchBlock(c rpcClient.Client) error { - b, _ := document.QuerySyncTask() - status, _ := c.Status() - latestBlockHeight := status.LatestBlockHeight - - funcChain := []func(tx store.Docs){ - saveTx, saveOrUpdateAccount, updateAccountBalance, - } - - ch := make(chan int64) - limitChan <- 1 - - go syncBlock(b.Height+1, latestBlockHeight, funcChain, ch, 0) - - select { - case <-ch: - //更新同步记录 - block, _ := c.Block(&latestBlockHeight) - b.Height = block.Block.Height - b.Time = block.Block.Time - return store.Update(b) - } -} - -// fast sync data from blockChain -func fastSync(c rpcClient.Client) error { - b, _ := document.QuerySyncTask() - status, _ := c.Status() - latestBlockHeight := status.LatestBlockHeight - funcChain := []func(tx store.Docs){ - saveTx, saveOrUpdateAccount, updateAccountBalance, - } - - ch := make(chan int64) - activeThreadNum := int64(0) - - goRoutineNum := (latestBlockHeight - b.Height) / syncBlockNumFastSync - - if goRoutineNum == 0 { - goRoutineNum = 10 - syncBlockNumFastSync = 100 - } - - for i := int64(1); i <= goRoutineNum; i++ { - activeThreadNum++ - limitChan <- i - var ( - start = b.Height + (i-1)*syncBlockNumFastSync + 1 - end = b.Height + i*syncBlockNumFastSync - ) - if i == goRoutineNum { - end = latestBlockHeight - } - go syncBlock(start, end, funcChain, ch, i) - } - - //threadNum := (latestBlockHeight - b.Height) / maxBatchNum - //// 单线程处理 - //if threadNum == 0 { - // go syncBlock(b.Height, latestBlockHeight, funcChain, ch, 0) - //} else { - // // 开启多线程处理 - // for i := int64(1); i <= threadNum; i++ { - // activeThreadNum ++ - // var start = b.Height + (i-1)*maxBatchNum + 1 - // var end = b.Height + i*maxBatchNum - // if i == threadNum { - // end = latestBlockHeight - // } - // - // go syncBlock(start, end, funcChain, ch, i) - // } - // - //} - - for { - select { - case threadNo := <-ch: - logger.Info.Printf("threadNo[%d] is over\n", threadNo) - activeThreadNum = activeThreadNum - 1 - if activeThreadNum == 0 { - goto end - } - } - } - -end: - { - //更新同步记录 - block, _ := c.Block(&latestBlockHeight) - b.Height = block.Block.Height - b.Time = block.Block.Time - store.Update(b) - return nil - } -} - -func syncBlock(start int64, end int64, funcChain []func(tx store.Docs), ch chan int64, threadNum int64) { - logger.Info.Printf("threadNo[%d] begin sync block from %d to %d\n", threadNum, start, end) - client := helper.GetClient() - // release client - defer client.Release() - // release buffer chain - defer func() { - <-limitChan - }() - - for j := start; j <= end; j++ { - logger.Info.Printf("===========threadNo[%d] sync block,height:%d===========\n", threadNum, j) - - // TODO 使用client.Client.BlockChainInfo - block, err := client.Client.Block(&j) - if err != nil { - // try again - client2 := helper.GetClient() - block, err = client2.Client.Block(&j) - if err != nil { - logger.Error.Fatalf("invalid block height %d\n", j) - } - } - if block.BlockMeta.Header.NumTxs > 0 { - txs := block.Block.Data.Txs - for _, txByte := range txs { - txType, tx := helper.ParseTx(txByte) - txHash := strings.ToUpper(hex.EncodeToString(txByte.Hash())) - logger.Info.Printf("===========threadNo[%d] find tx,txType=%s;txHash=%s\n", threadNum, txType, txHash) - - switch txType { - case constant.TxTypeCoin: - coinTx, _ := tx.(document.CoinTx) - coinTx.Height = block.Block.Height - coinTx.Time = block.Block.Time - handle(coinTx, funcChain) - break - case stake.TypeTxDeclareCandidacy: - stakeTxDeclareCandidacy, _ := tx.(document.StakeTxDeclareCandidacy) - stakeTxDeclareCandidacy.Height = block.Block.Height - stakeTxDeclareCandidacy.Time = block.Block.Time - handle(stakeTxDeclareCandidacy, funcChain) - break - case stake.TypeTxEditCandidacy: - break - case stake.TypeTxDelegate, stake.TypeTxUnbond: - stakeTx, _ := tx.(document.StakeTx) - stakeTx.Height = block.Block.Height - stakeTx.Time = block.Block.Time - handle(stakeTx, funcChain) - break - } - } - } - - // save block info - bk := document.Block{ - Height: block.Block.Height, - Time: block.Block.Time, - TxNum: block.BlockMeta.Header.NumTxs, - } - store.SaveOrUpdate(bk) - - } - ch <- threadNum -} diff --git a/tools/Makefile b/tools/Makefile new file mode 100644 index 0000000..08e0cbc --- /dev/null +++ b/tools/Makefile @@ -0,0 +1,33 @@ +######################################## +### DEP + +DEP = github.com/golang/dep/cmd/dep +DEP_CHECK := $(shell command -v dep 2> /dev/null) + +check_tools: +ifndef DEP_CHECK + @echo "No dep in path. Install with 'make get_tools'." +else + @echo "Found dep in path." +endif + +get_tools: +ifdef DEP_CHECK + @echo "Dep is already installed. Run 'make update_tools' to update." +else + @echo "$(ansi_grn)Installing dep$(ansi_end)" + go get -v $(DEP) +endif + +update_tools: + @echo "$(ansi_grn)Updating dep$(ansi_end)" + go get -u -v $(DEP) + + +######################################## +# ANSI colors + +ansi_red=\033[0;31m +ansi_grn=\033[0;32m +ansi_yel=\033[0;33m +ansi_end=\033[0m \ No newline at end of file diff --git a/util/constant/types.go b/util/constant/types.go index 2b95211..360e2c9 100644 --- a/util/constant/types.go +++ b/util/constant/types.go @@ -1,6 +1,25 @@ +// package for define constants + package constant const ( - TxTypeCoin = "coin" - TxTypeStake = "stake" + TxTypeBank = "coin" + TxTypeStakeCreate = "declareCandidacy" + TxTypeStakeEdit = "editCandidacy" + TxTypeStakeDelegate = "delegate" + TxTypeStakeUnbond = "unbond" + + TxStatusSuccess = "success" + + EnvNameDbHost = "DB_HOST" + EnvNameDbPort = "DB_PORT" + EnvNameDbUser = "DB_USER" + EnvNameDbPassWd = "DB_PASSWD" + EnvNameDbDataBase = "DB_DATABASE" + + EnvNameSerNetworkNodeUrl = "SER_BC_NODE_URL" + EnvNameSerNetworkChainId = "SER_BC_CHAIN_ID" + EnvNameSerNetworkToken = "SER_BC_TOKEN" + EnvNameSerMaxGoRoutine = "SER_MAX_GOROUTINE" + EnvNameSerSyncBlockNum = "SER_SYNC_BLOCK_NUM" ) diff --git a/util/helper/account.go b/util/helper/account.go index 88ae1d6..6913db8 100644 --- a/util/helper/account.go +++ b/util/helper/account.go @@ -1,98 +1,69 @@ +// This package is used for query balance of account + package helper import ( - "fmt" - "time" - - "github.com/irisnet/iris-sync-server/module/logger" + "github.com/irisnet/irishub-sync/module/codec" + "github.com/irisnet/irishub-sync/store" - wire "github.com/tendermint/go-wire" - "github.com/cosmos/cosmos-sdk/modules/coin" - "github.com/cosmos/cosmos-sdk/stack" - "github.com/cosmos/cosmos-sdk/client/commands" - "github.com/cosmos/cosmos-sdk/client/commands/query" - "github.com/tendermint/go-wire/data" - "github.com/tendermint/iavl" - "github.com/cosmos/cosmos-sdk/client" + "fmt" + sdk "github.com/cosmos/cosmos-sdk/types" + "github.com/cosmos/cosmos-sdk/x/auth" + authcmd "github.com/cosmos/cosmos-sdk/x/auth/client/cli" + "github.com/irisnet/irishub-sync/module/logger" + "github.com/pkg/errors" rpcclient "github.com/tendermint/tendermint/rpc/client" + cmn "github.com/tendermint/tmlibs/common" ) -var delay = false +func QueryAccountBalance(address string) store.Coins { + addr, err := sdk.GetValAddressHex(address) + if err != nil { + logger.Error.Printf("get addr from hex failed, %+v\n", err) + return nil + } -func setDelay(d bool) { - delay = d -} + res, err := query(auth.AddressStoreKey(addr), "acc", "key") -func QueryAccountBalance(address string, delay bool) *coin.Account { - account := new(coin.Account) - actor, err := commands.ParseActor(address) if err != nil { - return account + logger.Error.Printf("query balance from tendermint failed, %+v\n", err) + return nil } - actor = coin.ChainAddr(actor) - key := stack.PrefixedKey(coin.NameCoin, actor.Bytes()) - if delay { - time.Sleep(1 * time.Second) + // balance is empty + if len(res) <= 0 { + return nil } - _, err2 := GetParsed(key, account, query.GetHeight(), false) - if err2 != nil { - logger.Info.Printf("account bytes are empty for address: %q\n", address) - } - return account -} - -// argument (so pass in a pointer to the appropriate struct) -func GetParsed(key []byte, data interface{}, height int64, prove bool) (int64, error) { - bs, h, err := Get(key, height, prove) - if err != nil { - return 0, err - } - err = wire.ReadBinaryBytes(bs, data) + decoder := authcmd.GetAccountDecoder(codec.Cdc) + account, err := decoder(res) if err != nil { - return 0, err + logger.Error.Printf("decode account failed, %+v\n", err) + return nil } - return h, nil + + return BuildCoins(account.GetCoins()) } -// Get queries the given key and returns the value stored there and the -// height we checked at. -// -// If prove is true (and why shouldn't it be?), -// the data is fully verified before returning. If prove is false, -// we just repeat whatever any (potentially malicious) node gives us. -// Only use that if you are running the full node yourself, -// and it is localhost or you have a secure connection (not HTTP) -func Get(key []byte, height int64, prove bool) (data.Bytes, int64, error) { - if height < 0 { - return nil, 0, fmt.Errorf("Height cannot be negative\n") +// Query from Tendermint with the provided storename and path +func query(key cmn.HexBytes, storeName string, endPath string) (res []byte, err error) { + path := fmt.Sprintf("/store/%s/%s", storeName, endPath) + rpcClient := GetClient().Client + if err != nil { + return res, err } - if !prove { - tmClient := GetClient() - defer tmClient.Release() - resp, err := tmClient.Client.ABCIQueryWithOptions("/key", key, - rpcclient.ABCIQueryOptions{Trusted: true, Height: int64(height)}) - if resp == nil { - return nil, height, err - } - return data.Bytes(resp.Response.Value), resp.Response.Height, err + opts := rpcclient.ABCIQueryOptions{ + Height: 0, + Trusted: true, } - val, h, _, err := GetWithProof(key, height) - return val, h, err -} - -// GetWithProof returns the values stored under a given key at the named -// height as in Get. Additionally, it will return a validated merkle -// proof for the key-value pair if it exists, and all checks pass. -func GetWithProof(key []byte, height int64) (data.Bytes, int64, iavl.KeyProof, error) { - tmClient := GetClient() - defer tmClient.Release() - cert, err := commands.GetCertifier() + result, err := rpcClient.ABCIQueryWithOptions(path, key, opts) if err != nil { - return nil, 0, nil, err + return res, err } - return client.GetWithProof(key, height, tmClient.Client, cert) + resp := result.Response + if resp.Code != uint32(0) { + return res, errors.Errorf("Query failed: (%d) %s", resp.Code, resp.Log) + } + return resp.Value, nil } - diff --git a/util/helper/account_test.go b/util/helper/account_test.go index d0c9324..3cfadf7 100644 --- a/util/helper/account_test.go +++ b/util/helper/account_test.go @@ -1,50 +1,40 @@ +// This package is used for query balance of account + package helper import ( "testing" - "github.com/irisnet/iris-sync-server/module/logger" + "github.com/irisnet/irishub-sync/module/logger" ) -func setUp() { - InitClientPool() -} - func TestQueryAccountBalance(t *testing.T) { - setUp() + InitClientPool() type args struct { address string - delay bool } tests := []struct { name string args args }{ { - name:"the balance of address is not empty", - args: struct { - address string - delay bool - }{ - address: "ADBC4AAB3A089BDC8A8155AB97E64CD2CF4A0E9F", - delay: false}, + name: "test balance not nil", + args: args{ + address: "D770D45DEA7548076F8A27F9C9749B200934F1B4", + }, }, { - name:"the balance of address is empty", - args: struct { - address string - delay bool - }{ - address: "ADBC4AAB3A089BDC8A8155AB97E64CD2CF4A0E9E", - delay: false}, + name: "test balance is nil", + args: args{ + address: "D770D45DEA7548076F8A27F9C9749B200934F1B9", + }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - account := QueryAccountBalance(tt.args.address, tt.args.delay) - logger.Info.Printf("the balance of %s is %+v\n", tt.args.address, account) + got := QueryAccountBalance(tt.args.address) + logger.Info.Println(ToJson(got)) }) - } -} \ No newline at end of file +} diff --git a/util/helper/common.go b/util/helper/common.go new file mode 100644 index 0000000..5367301 --- /dev/null +++ b/util/helper/common.go @@ -0,0 +1,22 @@ +package helper + +import ( + "encoding/json" + "github.com/irisnet/irishub-sync/module/logger" + "encoding/binary" +) + + +// convert object to json +func ToJson(v interface{}) string { + data, err := json.Marshal(v) + if err != nil { + logger.Error.Println(err) + } + return string(data) +} + +// byte to int +func BytesToInt(bytes []byte) uint64 { + return binary.BigEndian.Uint64(bytes) +} diff --git a/util/helper/pool_client.go b/util/helper/pool_client.go index a52136b..633ed92 100644 --- a/util/helper/pool_client.go +++ b/util/helper/pool_client.go @@ -5,11 +5,9 @@ package helper import ( "errors" - "log" + conf "github.com/irisnet/irishub-sync/conf/server" - conf "github.com/irisnet/iris-sync-server/conf/server" - - rpcClient "github.com/cosmos/cosmos-sdk/client" + "github.com/irisnet/irishub-sync/module/logger" "github.com/tendermint/tendermint/rpc/client" ) @@ -47,7 +45,7 @@ func InitClientPool() { func GetClient() Client { c, err := getClient() if err != nil { - log.Fatal(err) + logger.Error.Fatalln(err.Error()) } return c } @@ -61,38 +59,47 @@ func (n Client) Release() { } func createConnection(id int64) Client { - client := Client{ - Client: rpcClient.GetNode(conf.BlockChainMonitorUrl), + tmClient := Client{ + Client: client.NewHTTP(conf.BlockChainMonitorUrl, "/websocket"), used: false, id: id, } - pool.clients[id] = client + + if id == int64(len(pool.clients)) { + newSlice := make([]Client, pool.maxConnection) + for i, v := range pool.clients { + newSlice[i] = v + } + pool.clients = newSlice + } + pool.clients[id] = tmClient pool.available++ - return client + return tmClient } func getClient() (Client, error) { if pool.available == 0 { maxConnNum := int64(conf.MaxConnectionNum) if pool.used < maxConnNum { - var client Client - for i := int64(len(pool.clients)); i < maxConnNum; i++ { - client = createConnection(i) + var tmClient Client + length := len(pool.clients) + for i := int64(length); i < maxConnNum; i++ { + tmClient = createConnection(i) } - return client, nil + return tmClient, nil } else { - log.Fatal("client pool has no available connection") + logger.Error.Fatalln("client pool has no available connection") } } - for _, client := range pool.clients { - if !client.used { - client.used = true - pool.clients[client.id] = client + for _, tmClient := range pool.clients { + if !tmClient.used { + tmClient.used = true + pool.clients[tmClient.id] = tmClient pool.available-- pool.used++ - log.Printf("current available coonection ：%d", pool.available) - return client, nil + logger.Info.Printf("current available coonection ：%d\n", pool.available) + return tmClient, nil } } return Client{}, errors.New("pool is empty") diff --git a/util/helper/pool_client_test.go b/util/helper/pool_client_test.go new file mode 100644 index 0000000..e2f8d9d --- /dev/null +++ b/util/helper/pool_client_test.go @@ -0,0 +1,57 @@ +// init client from clientPool. +// client is httpClient of tendermint + +package helper + +import ( + "testing" + + "github.com/tendermint/tendermint/rpc/client" + + conf "github.com/irisnet/irishub-sync/conf/server" + "github.com/irisnet/irishub-sync/module/logger" +) + +func TestInitClientPool(t *testing.T) { + a := []int{1, 2, 3} + b := make([]int, 6, 6) + for index, value := range a { + b[index] = value + } + b[3] = 4 + logger.Info.Println(b) +} + +func TestGetClient(t *testing.T) { + InitClientPool() + + for i := 0; i < conf.InitConnectionNum + 10; i++ { + client := GetClient() + logger.Info.Println(client) + } + +} + +func TestClient_Release(t *testing.T) { + type fields struct { + Client client.Client + used bool + id int64 + } + tests := []struct { + name string + fields fields + }{ + // TODO: Add test cases. + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + n := Client{ + Client: tt.fields.Client, + used: tt.fields.used, + id: tt.fields.id, + } + n.Release() + }) + } +} diff --git a/util/helper/tx.go b/util/helper/tx.go index fe3c5b7..4bdc00d 100644 --- a/util/helper/tx.go +++ b/util/helper/tx.go @@ -1,106 +1,183 @@ +// package for parse tx struct from binary data + package helper import ( - "encoding/hex" - "fmt" - "strings" + "github.com/irisnet/irishub-sync/store/document" - "github.com/irisnet/iris-sync-server/module/logger" - "github.com/irisnet/iris-sync-server/util/constant" - - sdk "github.com/cosmos/cosmos-sdk" - "github.com/tendermint/go-wire/data" + "encoding/hex" + sdktypes "github.com/cosmos/cosmos-sdk/types" + "github.com/cosmos/cosmos-sdk/wire" + "github.com/cosmos/cosmos-sdk/x/auth" + "github.com/cosmos/cosmos-sdk/x/bank" + "github.com/cosmos/cosmos-sdk/x/stake" + "github.com/irisnet/irishub-sync/module/logger" + "github.com/irisnet/irishub-sync/store" + "github.com/irisnet/irishub-sync/util/constant" "github.com/tendermint/tendermint/types" + "strconv" + "strings" +) - // 需要将 cosmos-sdk module 中的 txInner 注册到内存 - // 中，才能解析 tx 结构 - _ "github.com/cosmos/cosmos-sdk/modules/auth" - _ "github.com/cosmos/cosmos-sdk/modules/base" - "github.com/cosmos/cosmos-sdk/modules/coin" - "github.com/cosmos/cosmos-sdk/modules/nonce" - "github.com/irisnet/iris-sync-server/module/stake" - "github.com/irisnet/iris-sync-server/model/store/document" +type ( + msgBankSend = bank.MsgSend + msgStakeCreate = stake.MsgCreateValidator + msgStakeEdit = stake.MsgEditValidator + msgStakeDelegate = stake.MsgDelegate + msgStakeUnbond = stake.MsgUnbond ) -// parse tx struct from binary data -func ParseTx(txByte types.Tx) (string, interface{}) { - txb, err := sdk.LoadTx(txByte) +func ParseTx(cdc *wire.Codec, txBytes types.Tx, block *types.Block) store.Docs { + var ( + authTx auth.StdTx + ) + + err := cdc.UnmarshalBinary(txBytes, &authTx) if err != nil { logger.Error.Println(err) + return nil + } + + height := block.Height + time := block.Time + txHash := BuildHex(txBytes.Hash()) + fee := buildFee(authTx.Fee) + status := constant.TxStatusSuccess + + switch authTx.GetMsg().(type) { + case msgBankSend: + msg := authTx.Msg.(msgBankSend) + docTx := document.CommonTx{ + Height: height, + Time: time, + TxHash: txHash, + Fee: fee, + Status: status, + } + docTx.From = msg.Inputs[0].Address.String() + docTx.To = msg.Outputs[0].Address.String() + docTx.Amount = BuildCoins(msg.Inputs[0].Coins) + docTx.Type = constant.TxTypeBank + return docTx + case msgStakeCreate: + msg := authTx.Msg.(msgStakeCreate) + stakeTx := document.StakeTx{ + Height: height, + Time: time, + TxHash: txHash, + Fee: fee, + Status: status, + } + stakeTx.ValidatorAddr = msg.ValidatorAddr.String() + stakeTx.PubKey = BuildHex(msg.PubKey.Bytes()) + stakeTx.Amount = buildCoin(msg.Bond) + + description := document.Description{ + Moniker: msg.Moniker, + Identity: msg.Identity, + Website: msg.Website, + Details: msg.Details, + } + + docTx := document.StakeTxDeclareCandidacy{ + StakeTx: stakeTx, + Description: description, + } + docTx.Type = constant.TxTypeStakeCreate + return docTx + case msgStakeEdit: + msg := authTx.Msg.(msgStakeEdit) + stakeTx := document.StakeTx{ + Height: height, + Time: time, + TxHash: txHash, + Fee: fee, + Status: status, + } + stakeTx.ValidatorAddr = msg.ValidatorAddr.String() + + description := document.Description{ + Moniker: msg.Moniker, + Identity: msg.Identity, + Website: msg.Website, + Details: msg.Details, + } + + docTx := document.StakeTxEditCandidacy{ + StakeTx: stakeTx, + Description: description, + } + docTx.Type = constant.TxTypeStakeEdit + return docTx + case msgStakeDelegate: + msg := authTx.Msg.(msgStakeDelegate) + docTx := document.StakeTx{ + Height: height, + Time: time, + TxHash: txHash, + Fee: fee, + Status: status, + } + docTx.DelegatorAddr = msg.DelegatorAddr.String() + docTx.ValidatorAddr = msg.ValidatorAddr.String() + docTx.Amount = buildCoin(msg.Bond) + docTx.Type = constant.TxTypeStakeDelegate + return docTx + case msgStakeUnbond: + msg := authTx.Msg.(msgStakeUnbond) + shares, err := strconv.Atoi(msg.Shares) + if err != nil { + logger.Error.Println(err) + } + docTx := document.StakeTx{ + Height: height, + Time: time, + TxHash: txHash, + Fee: fee, + Status: status, + } + docTx.DelegatorAddr = msg.DelegatorAddr.String() + docTx.ValidatorAddr = msg.ValidatorAddr.String() + docTx.Amount = store.Coin{ + Amount: int64(shares), + } + docTx.Type = constant.TxTypeStakeUnbond + return docTx + default: + logger.Info.Println("unknown msg type") } - txl, ok := txb.Unwrap().(sdk.TxLayer) + return nil +} + +func BuildCoins(coins sdktypes.Coins) store.Coins { var ( - txi sdk.Tx - coinTx document.CoinTx - stakeTx document.StakeTx - StakeTxDeclareCandidacy document.StakeTxDeclareCandidacy - nonceAddr data.Bytes + localCoins store.Coins ) - for ok { - txi = txl.Next() - - switch txi.Unwrap().(type) { - - case coin.SendTx: - ctx, _ := txi.Unwrap().(coin.SendTx) - coinTx.From = fmt.Sprintf("%s", ctx.Inputs[0].Address.Address) - coinTx.To = fmt.Sprintf("%s", ctx.Outputs[0].Address.Address) - coinTx.Amount = ctx.Inputs[0].Coins - coinTx.TxHash = strings.ToUpper(hex.EncodeToString(txByte.Hash())) - return constant.TxTypeCoin, coinTx - case nonce.Tx: - ctx, _ := txi.Unwrap().(nonce.Tx) - nonceAddr = ctx.Signers[0].Address - fmt.Println(nonceAddr) - break - case stake.TxUnbond, stake.TxDelegate, stake.TxDeclareCandidacy: - kind, _ := txi.GetKind() - stakeTx.From = fmt.Sprintf("%s", nonceAddr) - stakeTx.Type = strings.Replace(kind, constant.TxTypeStake + "/", "", -1) - stakeTx.TxHash = strings.ToUpper(hex.EncodeToString(txByte.Hash())) - - switch kind { - case stake.TypeTxDeclareCandidacy: - ctx, _ := txi.Unwrap().(stake.TxDeclareCandidacy) - stakeTx.Amount.Denom = ctx.BondUpdate.Bond.Denom - stakeTx.Amount.Amount = ctx.BondUpdate.Bond.Amount - stakeTx.PubKey = fmt.Sprintf("%s", ctx.PubKey.KeyString()) - - description := document.Description{ - Moniker: ctx.Description.Moniker, - Identity: ctx.Description.Identity, - Website: ctx.Description.Website, - Details: ctx.Description.Details, - } - - StakeTxDeclareCandidacy.StakeTx = stakeTx - StakeTxDeclareCandidacy.Description = description - - return kind, StakeTxDeclareCandidacy - case stake.TypeTxEditCandidacy: - // TODO：record edit candidacy tx if necessary - //ctx, _ := txi.Unwrap().(stake.TxEditCandidacy) - break - case stake.TypeTxDelegate: - ctx, _ := txi.Unwrap().(stake.TxDelegate) - stakeTx.Amount.Denom = ctx.Bond.Denom - stakeTx.Amount.Amount = ctx.Bond.Amount - stakeTx.PubKey = fmt.Sprintf("%s", ctx.PubKey.KeyString()) - break - case stake.TypeTxUnbond: - ctx, _ := txi.Unwrap().(stake.TxUnbond) - stakeTx.Amount.Amount = int64(ctx.Shares) - stakeTx.PubKey = fmt.Sprintf("%s", ctx.PubKey.KeyString()) - break - } - return kind, stakeTx - default: - logger.Info.Printf("unsupported tx type, %+v\n", txi.Unwrap()) + if len(coins) > 0 { + for _, coin := range coins { + localCoins = append(localCoins, buildCoin(coin)) } + } - txl, ok = txi.Unwrap().(sdk.TxLayer) + return localCoins +} + +func buildCoin(coin sdktypes.Coin) store.Coin { + return store.Coin{ + Denom: coin.Denom, + Amount: coin.Amount, } - return "", nil +} + +func buildFee(fee auth.StdFee) store.Fee { + return store.Fee{ + Amount: BuildCoins(fee.Amount), + Gas: fee.Gas, + } +} + +func BuildHex(bytes []byte) string { + return strings.ToUpper(hex.EncodeToString(bytes)) } diff --git a/util/helper/tx_test.go b/util/helper/tx_test.go index 913596c..5aacb80 100644 --- a/util/helper/tx_test.go +++ b/util/helper/tx_test.go @@ -1,17 +1,16 @@ +// package for parse tx struct from binary data + package helper import ( "testing" - "github.com/irisnet/iris-sync-server/module/logger" - - _ "github.com/cosmos/cosmos-sdk/modules/auth" - _ "github.com/cosmos/cosmos-sdk/modules/base" - + "github.com/irisnet/irishub-sync/module/codec" "github.com/tendermint/tendermint/types" + "github.com/irisnet/irishub-sync/module/logger" ) -func buildTxByte(blockHeight int64) types.Tx { +func buildTxByte(blockHeight int64) (types.Tx, *types.Block) { InitClientPool() client := GetClient() @@ -26,57 +25,69 @@ func buildTxByte(blockHeight int64) types.Tx { if block.BlockMeta.Header.NumTxs > 0 { txs := block.Block.Data.Txs - return txs[0] + return txs[0], block.Block } - return nil + return nil, nil } func TestParseTx(t *testing.T) { - - txCoinByte := buildTxByte(12453) - txStakeDeclareCandidacyByte := buildTxByte(19073) - txStakeDelegateByte := buildTxByte(13725) - txStakeUnBondByte := buildTxByte(14260) - + coinByte, coinBlock := buildTxByte(1762) + scByte, scBlock := buildTxByte(46910) + seByte, seBlock := buildTxByte(49388) + sdByte, sdBlock := buildTxByte(47349) + suByte, suBlock := buildTxByte(34241) type args struct { txByte types.Tx + block *types.Block } tests := []struct { name string args args + want string + want1 interface{} }{ { - name: "tx_coin_send", - args: struct{ txByte types.Tx }{ - txByte: txCoinByte, - }, + name: "test tx coin", + args: args{ + txByte: coinByte, + block: coinBlock, + }, }, { - name: "tx_stake_declareCandidacy", - args: struct{ txByte types.Tx }{ - txByte: txStakeDeclareCandidacyByte, - }, + name: "test tx stake/create", + args: args{ + txByte: scByte, + block: scBlock, + }, }, { - name: "tx_stake_delegate", - args: struct{ txByte types.Tx }{ - txByte: txStakeDelegateByte, + name: "test tx stake/edit", + args: args{ + txByte: seByte, + block: seBlock, + }, + }, + { + name: "test tx stake/delegate", + args: args{ + txByte: sdByte, + block: sdBlock, }, }, - { - name: "tx_stake_unbond", - args: struct{ txByte types.Tx }{ - txByte: txStakeUnBondByte, + name: "test tx stake/unbond", + args: args{ + txByte: suByte, + block: suBlock, }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - txType, txContent := ParseTx(tt.args.txByte) - logger.Info.Printf("%s: tx type is %s, and struct is %+v\n", tt.name, txType, txContent) + res := ParseTx(codec.Cdc, tt.args.txByte, tt.args.block) + logger.Info.Printf("Tx is %v\n", ToJson(res)) }) } }