Dag inspector

.github/workflows/build.yml

@@ -20,11 +20,6 @@ jobs:
  uses: actions/setup-java@v1
  with:
  java-version: 11
- - name: maven-settings-xml-action
- uses: whelk-io/maven-settings-xml-action@v18
- with:
- repositories: '[{ "id": "github", "url": "https://maven.pkg.github.com/starcoinorg/*" }]'
- servers: '[{ "id": "github", "username": "${{ github.actor }}", "password": "${{ secrets.GIT_PACKAGE_TOKEN }}" }]'
  - name: Cache Maven packages
  uses: actions/cache@v1
  with:

.github/workflows/docker_build_indexer.yml

@@ -22,7 +22,7 @@ jobs:
  id: docker_meta
  uses: crazy-max/ghaction-docker-meta@v1
  with:
- images: starcoin/starcoin_indexer,ghcr.io/starcoinorg/starcoin_indexer
+ images: starcoin/starcoin_indexer
  tag-sha: true
  - name: Set up Docker Buildx
  uses: docker/[email protected]
@@ -33,12 +33,6 @@ jobs:
  with:
  username: ${{ secrets.DOCKERHUB_USERNAME }}
  password: ${{ secrets.DOCKERHUB_TOKEN }}
- - name: Login to GitHub Container Registry
- uses: docker/login-action@v1
- with:
- registry: ghcr.io
- username: ${{ github.actor }}
- password: ${{ secrets.GIT_PACKAGE_TOKEN }}
  - name: maven-settings-xml-action
  uses: whelk-io/maven-settings-xml-action@v18
  with:

.github/workflows/docker_build_scanapi.yml

@@ -22,7 +22,7 @@ jobs:
  id: docker_meta
  uses: crazy-max/ghaction-docker-meta@v1
  with:
- images: starcoin/starcoin_scan,ghcr.io/starcoinorg/starcoin_scan
+ images: starcoin/starcoin_scan
  tag-sha: true
  - name: Set up Docker Buildx
  uses: docker/[email protected]

README.md

@@ -1 +1,3 @@
-# starcoin-search
+# stc-scan
+Established some independent handles for indexing data with scan to customize statistics data for swap, used in conjunction with the [swap-stat-api](https://github.com/Elements-Studio/swap-stat-api) repository.
+

docker-compose/README.md

@@ -0,0 +1,63 @@
+# How to build local environment
+
+## 1. Install "Docker" and "Docker Compose"
+## 2. Run the command to start the database and components
+```bash
+docker-compose up
+```
+## 3. Start starcoin-index project
+
+### Config the startup environment variable
+```dotenv
+HOSTS=localhost 
+NETWORK=halley # select which network to scan
+BG_TASK_JOBS=dag_inspector
+TXN_OFFSET=0
+BULK_SIZE=100
+STARCOIN_ES_PWD=
+STARCOIN_ES_URL=localhost
+STARCOIN_ES_PROTOCOL=http
+STARCOIN_ES_PORT=9200
+STARCOIN_ES_USER=
+SWAP_API_URL=https://swap-api.starswap.xyz
+SWAP_CONTRACT_ADDR=0x8c109349c6bd91411d6bc962e080c4a3
+DS_URL=jdbc:postgresql://localhost/starcoin
+DB_SCHEMA=halley
+DB_USER_NAME=starcoin
+DB_PWD=starcoin
+PROGRAM_ARGS=
+# auto_repair 9411700
+```
+
+### Configuration Elasticsearch template
+[IMPORTANT!!] Make sure your template has added to Elastic search service before add data, including component template and index template to ES.
+Following file: [[es_pipeline.scripts](..%2Fkube%2Fmappings%2Fes_pipeline.scripts)]
+
+1. Open the 'Kibana' site has been started in the docker-compose environment, usually the url is http://localhost:5601 
+2. Navigate to 'Dev Tools'
+3. Follow the instructions in the file of giving above to add the template to ES
+
+### Add SQL tables for network
+[IMPORTANT!!] Add the [tables](../starcoin-indexer/deploy/create_table.sql) for the network you want to scan, including main, barnard, halley, etc.
+
+## 4. Start starcoin-scan-api project
+
+### Config the startup enviroment variable
+```dotenv
+STARCOIN_ES_URL=localhost
+STARCOIN_ES_PROTOCOL=http
+STARCOIN_ES_PORT=9200
+STARCOIN_ES_USER=
+STARCOIN_ES_INDEX_VERSION=
+STARCOIN_ES_PWD=
+MAIN_DS_URL=jdbc:postgresql://localhost/starcoin?currentSchema=main
+BARNARD_DS_URL=jdbc:postgresql://localhost/starcoin?currentSchema=barnard
+HALLEY_DS_URL=jdbc:postgresql://localhost/starcoin?currentSchema=halley
+DS_URL=jdbc:postgresql://localhost/starcoin
+STARCOIN_USER_DS_URL="jdbc:postgresql://localhost/starcoin?currentSchema=starcoin_user"
+DB_USER_NAME=starcoin
+DB_PWD=starcoin
+```
+
+### Add SQL tables for network
+[IMPORTANT!!] Add the [tables](../starcoin-scan-api/deploy/create_table.sql) for the network you want to scan, including main, barnard, halley, etc.

docker-compose/docker-compose.yaml

@@ -0,0 +1,67 @@
+# This composer file is used to configure the local environment for debugging
+version: '3.8'
+
+services:
+ elasticsearch:
+ image: docker.elastic.co/elasticsearch/elasticsearch:7.17.2
+ container_name: elasticsearch
+ environment:
+ - node.name=elasticsearch
+ - cluster.name=docker-cluster
+ - discovery.type=single-node
+ - bootstrap.memory_lock=true
+ - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+ ulimits:
+ memlock:
+ soft: -1
+ hard: -1
+ volumes:
+ - esdata:/usr/share/elasticsearch/data
+ ports:
+ - "9200:9200"
+
+ hazelcast:
+ image: hazelcast/hazelcast:latest
+ container_name: hazelcast
+ ports:
+ - "5701:5701"
+ environment:
+ - HZ_CLUSTERNAME=stcscan-hazelcast-cluster
+ - HZ_NETWORK_JOIN_MULTICAST_ENABLED=false
+ - HZ_NETWORK_JOIN_TCPIP_ENABLED=true
+ - HZ_NETWORK_JOIN_TCPIP_MEMBERS=hazelcast
+ - HZ_CACHE_CODE_CACHE_STATISTICS_ENABLED=true
+ - HZ_CACHE_CODE_CACHE_EVICTION_SIZE=10000
+ - HZ_CACHE_CODE_CACHE_EVICTION_MAX_SIZE_POLICY=ENTRY_COUNT
+ - HZ_CACHE_CODE_CACHE_EVICTION_EVICTION_POLICY=LFU
+ - HZ_CACHE_SESSION_STATISTICS_ENABLED=true
+ - HZ_CACHE_SESSION_EVICTION_SIZE=50000
+ - HZ_CACHE_SESSION_EVICTION_MAX_SIZE_POLICY=ENTRY_COUNT
+ - HZ_CACHE_SESSION_EVICTION_EVICTION_POLICY=LRU
+
+ kibana:
+ image: docker.elastic.co/kibana/kibana:7.17.2
+ container_name: kibana
+ environment:
+ ELASTICSEARCH_URL: http://elasticsearch:9200
+ ELASTICSEARCH_HOSTS: http://elasticsearch:9200
+ ports:
+ - 5601:5601
+ depends_on:
+ - elasticsearch
+
+ postgresql:
+ image: postgres:13.2
+ container_name: postgres_db
+ environment:
+ POSTGRES_USER: starcoin
+ POSTGRES_PASSWORD: starcoin
+ POSTGRES_DB: starcoin
+ volumes:
+ - db_data:/var/lib/postgresql/data
+ ports:
+ - 5432:5432
+
+volumes:
+ esdata:
+ driver: local

kube/base-components/allowaccess-network-policy.yaml

@@ -0,0 +1,92 @@
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+ name: allow-access-pg-from-vega
+ namespace: default
+spec:
+ podSelector:
+ matchLabels:
+ app: postgres-service
+ policyTypes:
+ - Ingress
+ ingress:
+ - from:
+ - namespaceSelector:
+ matchLabels:
+ name: starcoin-vega
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+ name: allow-access-es-from-vega
+ namespace: default
+spec:
+ podSelector:
+ matchLabels:
+ app: elasticsearch
+ policyTypes:
+ - Ingress
+ ingress:
+ - from:
+ - namespaceSelector:
+ matchLabels:
+ name: starcoin-vega
+
+
+---
+# Postgres service for starcoin-api
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+ name: allow-access-pg-from-api
+ namespace: default
+spec:
+ podSelector:
+ matchLabels:
+ app: postgres-service
+ policyTypes:
+ - Ingress
+ ingress:
+ - from:
+ - namespaceSelector:
+ matchLabels:
+ name: starcoin-api
+
+---
+# Elasticsearch service for starcoin-api
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+ name: allow-access-es-from-api
+ namespace: default
+spec:
+ podSelector:
+ matchLabels:
+ app: elasticsearch
+ policyTypes:
+ - Ingress
+ ingress:
+ - from:
+ - namespaceSelector:
+ matchLabels:
+ name: starcoin-api
+
+---
+# Default namespace access for elasticsearch
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+ name: allow-internal-elasticsearch
+ namespace: default
+spec:
+ podSelector:
+ matchLabels:
+ app: elasticsearch
+ policyTypes:
+ - Ingress
+ ingress:
+ - from:
+ - podSelector: {}
+ ports:
+ - protocol: TCP
+ port: 9200

kube/base-components/elasticsearch-deployment.yaml

@@ -0,0 +1,100 @@
+# elasticsearch-deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: elasticsearch
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app: elasticsearch
+ template:
+ metadata:
+ labels:
+ app: elasticsearch
+ spec:
+ initContainers:
+ - name: init-permissions
+ image: busybox
+ command: [ "sh", "-c", "chown -R 1000:1000 /usr/share/elasticsearch/data" ]
+ volumeMounts:
+ - name: es-data
+ mountPath: /usr/share/elasticsearch/data
+ containers:
+ - name: elasticsearch
+ image: docker.elastic.co/elasticsearch/elasticsearch:7.17.0
+ resources:
+ requests:
+ cpu: "1"
+ memory: "4Gi"
+ limits:
+ cpu: "2"
+ memory: "6Gi"
+ ports:
+ - containerPort: 9200
+ name: http
+ - containerPort: 9300
+ name: transport
+ volumeMounts:
+ - name: es-data
+ mountPath: /usr/share/elasticsearch/data
+ - name: elasticsearch-config
+ mountPath: /usr/share/elasticsearch/config/elasticsearch.yml
+ subPath: elasticsearch.yml
+ env:
+ - name: discovery.type
+ value: single-node
+ - name: ELASTIC_USERNAME
+ valueFrom:
+ secretKeyRef:
+ name: elasticsearch
+ key: username
+ - name: ELASTIC_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: elasticsearch
+ key: password
+ volumes:
+ - name: es-data
+ persistentVolumeClaim:
+ claimName: es-pvc
+ - name: elasticsearch-config
+ configMap:
+ name: elasticsearch-config
+---
+# Elasticsearch Configuration
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: elasticsearch-config
+data:
+ elasticsearch.yml: |
+ xpack.security.enabled: true
+ xpack.license.self_generated.type: basic
+ network.host: 0.0.0.0
+
+---
+# Elasticsearch Persistent Volume Claim
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+ name: es-pvc
+spec:
+ accessModes:
+ - ReadWriteOnce
+ resources:
+ requests:
+ storage: 100Gi
+ storageClassName: do-block-storage-retain
+
+---
+# Elasticsearch Service
+apiVersion: v1
+kind: Service
+metadata:
+ name: elasticsearch
+spec:
+ ports:
+ - port: 9200
+ selector:
+ app: elasticsearch