Merge pull request #909 from moonstream-to/zksync-era-mainnet-subscri…

…ption

Add zksync era smartcontract subscription type.

crawlers/deploy/deploy.bash

@@ -109,6 +109,16 @@ WYRM_HISTORICAL_CRAWL_TRANSACTIONS_TIMER_FILE="wyrm-historical-crawl-transaction
 WYRM_HISTORICAL_CRAWL_EVENTS_SERVICE_FILE="wyrm-historical-crawl-events.service"
 WYRM_HISTORICAL_CRAWL_EVENTS_TIMER_FILE="wyrm-historical-crawl-events.timer"
 
+# ZkSync Era
+ZKSYNC_ERA_SYNCHRONIZE_SERVICE="zksync-era-synchronize.service"
+ZKSYNC_ERA_MISSING_SERVICE_FILE="zksync-era-missing.service"
+ZKSYNC_ERA_MISSING_TIMER_FILE="zksync-era-missing.timer"
+ZKSYNC_ERA_MOONWORM_CRAWLER_SERVICE_FILE="zksync-era-moonworm-crawler.service"
+ZKSYNC_ERA_HISTORICAL_CRAWL_TRANSACTIONS_SERVICE_FILE="zksync-era-historical-crawl-transactions.service"
+ZKSYNC_ERA_HISTORICAL_CRAWL_TRANSACTIONS_TIMER_FILE="zksync-era-historical-crawl-transactions.timer"
+ZKSYNC_ERA_HISTORICAL_CRAWL_EVENTS_SERVICE_FILE="zksync-era-historical-crawl-events.service"
+ZKSYNC_ERA_HISTORICAL_CRAWL_EVENTS_TIMER_FILE="zksync-era-historical-crawl-events.timer"
+
 # ZkSync Era testnet
 ZKSYNC_ERA_TESTNET_SYNCHRONIZE_SERVICE="zksync-era-testnet-synchronize.service"
 ZKSYNC_ERA_TESTNET_MISSING_SERVICE_FILE="zksync-era-testnet-missing.service"
@@ -516,6 +526,51 @@ XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${WYRM_HIS
 # ZkSync Era
 echo
 echo
+echo -e "${PREFIX_INFO} Replacing existing ZkSync Era block with transactions syncronizer service definition with ${ZKSYNC_ERA_SYNCHRONIZE_SERVICE}"
+chmod 644 "${SCRIPT_DIR}/${ZKSYNC_ERA_SYNCHRONIZE_SERVICE}"
+cp "${SCRIPT_DIR}/${ZKSYNC_ERA_SYNCHRONIZE_SERVICE}" "/home/ubuntu/.config/systemd/user/${ZKSYNC_ERA_SYNCHRONIZE_SERVICE}"
+XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
+XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ZKSYNC_ERA_SYNCHRONIZE_SERVICE}"
+
+echo
+echo
+echo -e "${PREFIX_INFO} Replacing existing ZkSync Era missing service and timer with: ${ZKSYNC_ERA_MISSING_SERVICE_FILE}, ${ZKSYNC_ERA_MISSING_TIMER_FILE}"
+chmod 644 "${SCRIPT_DIR}/${ZKSYNC_ERA_MISSING_SERVICE_FILE}" "${SCRIPT_DIR}/${ZKSYNC_ERA_MISSING_TIMER_FILE}"
+cp "${SCRIPT_DIR}/${ZKSYNC_ERA_MISSING_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${ZKSYNC_ERA_MISSING_SERVICE_FILE}"
+cp "${SCRIPT_DIR}/${ZKSYNC_ERA_MISSING_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${ZKSYNC_ERA_MISSING_TIMER_FILE}"
+XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
+XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ZKSYNC_ERA_MISSING_TIMER_FILE}"
+
+echo
+echo
+echo -e "${PREFIX_INFO} Replacing existing ZkSync Era moonworm crawler service definition with ${ZKSYNC_ERA_MOONWORM_CRAWLER_SERVICE_FILE}"
+chmod 644 "${SCRIPT_DIR}/${ZKSYNC_ERA_MOONWORM_CRAWLER_SERVICE_FILE}"
+cp "${SCRIPT_DIR}/${ZKSYNC_ERA_MOONWORM_CRAWLER_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${ZKSYNC_ERA_MOONWORM_CRAWLER_SERVICE_FILE}"
+XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
+XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ZKSYNC_ERA_MOONWORM_CRAWLER_SERVICE_FILE}"
+
+echo
+echo
+echo -e "${PREFIX_INFO} Replacing existing ZkSync Era historical transactions crawler service and timer with: ${ZKSYNC_ERA_HISTORICAL_CRAWL_TRANSACTIONS_SERVICE_FILE}, ${ZKSYNC_ERA_HISTORICAL_CRAWL_TRANSACTIONS_TIMER_FILE}"
+chmod 644 "${SCRIPT_DIR}/${ZKSYNC_ERA_HISTORICAL_CRAWL_TRANSACTIONS_SERVICE_FILE}" "${SCRIPT_DIR}/${ZKSYNC_ERA_HISTORICAL_CRAWL_TRANSACTIONS_TIMER_FILE}"
+cp "${SCRIPT_DIR}/${ZKSYNC_ERA_HISTORICAL_CRAWL_TRANSACTIONS_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${ZKSYNC_ERA_HISTORICAL_CRAWL_TRANSACTIONS_SERVICE_FILE}"
+cp "${SCRIPT_DIR}/${ZKSYNC_ERA_HISTORICAL_CRAWL_TRANSACTIONS_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${ZKSYNC_ERA_HISTORICAL_CRAWL_TRANSACTIONS_TIMER_FILE}"
+XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
+XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ZKSYNC_ERA_HISTORICAL_CRAWL_TRANSACTIONS_TIMER_FILE}"
+
+echo
+echo
+echo -e "${PREFIX_INFO} Replacing existing ZkSync Era historical events crawler service and timer with: ${ZKSYNC_ERA_HISTORICAL_CRAWL_EVENTS_SERVICE_FILE}, ${ZKSYNC_ERA_HISTORICAL_CRAWL_EVENTS_TIMER_FILE}"
+chmod 644 "${SCRIPT_DIR}/${ZKSYNC_ERA_HISTORICAL_CRAWL_EVENTS_SERVICE_FILE}" "${SCRIPT_DIR}/${ZKSYNC_ERA_HISTORICAL_CRAWL_EVENTS_TIMER_FILE}"
+cp "${SCRIPT_DIR}/${ZKSYNC_ERA_HISTORICAL_CRAWL_EVENTS_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${ZKSYNC_ERA_HISTORICAL_CRAWL_EVENTS_SERVICE_FILE}"
+cp "${SCRIPT_DIR}/${ZKSYNC_ERA_HISTORICAL_CRAWL_EVENTS_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${ZKSYNC_ERA_HISTORICAL_CRAWL_EVENTS_TIMER_FILE}"
+XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
+XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ZKSYNC_ERA_HISTORICAL_CRAWL_EVENTS_TIMER_FILE}"
+
+
+# ZkSync Era testnet
+echo
+echo
 echo -e "${PREFIX_INFO} Replacing existing ZkSync Era testnet block with transactions syncronizer service definition with ${ZKSYNC_ERA_TESTNET_SYNCHRONIZE_SERVICE}"
 chmod 644 "${SCRIPT_DIR}/${ZKSYNC_ERA_TESTNET_SYNCHRONIZE_SERVICE}"
 cp "${SCRIPT_DIR}/${ZKSYNC_ERA_TESTNET_SYNCHRONIZE_SERVICE}" "/home/ubuntu/.config/systemd/user/${ZKSYNC_ERA_TESTNET_SYNCHRONIZE_SERVICE}"
@@ -558,7 +613,6 @@ XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
 XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ZKSYNC_ERA_TESTNET_HISTORICAL_CRAWL_EVENTS_TIMER_FILE}"
 
 
-
 echo
 echo
 echo -e "${PREFIX_INFO} Replacing existing Leaderboards worker service and timer with: ${LEADERBOARDS_WORKER_SERVICE_FILE}, ${LEADERBOARDS_WORKER_TIMER_FILE}"

crawlers/deploy/zksync-era-historical-crawl-events.service

@@ -0,0 +1,17 @@
+[Unit]
+Description=ZkSync Era historical crawler events
+After=network.target
+StartLimitIntervalSec=300
+StartLimitBurst=3
+
+[Service]
+WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
+EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
+Restart=on-failure
+RestartSec=15s
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" historical-crawl --blockchain-type zksync_era --find-deployed-blocks --end 0 --tasks-journal --only-events
+CPUWeight=70
+SyslogIdentifier=zksync-era-historical-crawl-events
+
+[Install]
+WantedBy=multi-user.target

crawlers/deploy/zksync-era-historical-crawl-events.timer

@@ -0,0 +1,9 @@
+[Unit]
+Description=Runs events historical crawler on ZkSync Era
+
+[Timer]
+OnBootSec=60s
+OnUnitActiveSec=10m
+
+[Install]
+WantedBy=timers.target

crawlers/deploy/zksync-era-historical-crawl-transactions.service

@@ -0,0 +1,17 @@
+[Unit]
+Description=ZkSync Era historical crawler transactions
+After=network.target
+StartLimitIntervalSec=300
+StartLimitBurst=3
+
+[Service]
+WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
+EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
+Restart=on-failure
+RestartSec=15s
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" historical-crawl --blockchain-type zksync_era --find-deployed-blocks --end 0 --tasks-journal --only-functions
+CPUWeight=70
+SyslogIdentifier=zksync-era-historical-crawl-transactions
+
+[Install]
+WantedBy=multi-user.target

crawlers/deploy/zksync-era-historical-crawl-transactions.timer

@@ -0,0 +1,9 @@
+[Unit]
+Description=Runs transactions historical crawler on ZkSync Era
+
+[Timer]
+OnBootSec=60s
+OnUnitActiveSec=10m
+
+[Install]
+WantedBy=timers.target

crawlers/deploy/zksync-era-missing.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=Fill missing blocks at ZkSync Era database
+After=network.target
+
+[Service]
+Type=oneshot
+WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
+EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks missing --blockchain zksync_era -n
+CPUWeight=50
+SyslogIdentifier=zksync-era-missing

crawlers/deploy/zksync-era-missing.timer

@@ -0,0 +1,9 @@
+[Unit]
+Description=Fill missing blocks at ZkSync Era database
+
+[Timer]
+OnBootSec=120s
+OnUnitActiveSec=15m
+
+[Install]
+WantedBy=timers.target

crawlers/deploy/zksync-era-moonworm-crawler.service

@@ -0,0 +1,17 @@
+[Unit]
+Description=ZkSync Era moonworm crawler
+After=network.target
+StartLimitIntervalSec=300
+StartLimitBurst=3
+
+[Service]
+WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
+EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
+Restart=on-failure
+RestartSec=15s
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl -b zksync_era  --confirmations 20 --min-blocks-batch 20
+CPUWeight=70
+SyslogIdentifier=zksync-era-moonworm-crawler
+
+[Install]
+WantedBy=multi-user.target

crawlers/deploy/zksync-era-synchronize.service

@@ -0,0 +1,17 @@
+[Unit]
+Description=ZkSync Era block with transactions synchronizer
+StartLimitIntervalSec=300
+StartLimitBurst=3
+After=network.target
+
+[Service]
+Restart=on-failure
+RestartSec=15s
+WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
+EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks synchronize --blockchain zksync_era -c 20 -j 2
+CPUWeight=90
+SyslogIdentifier=zksync-era-synchronize
+
+[Install]
+WantedBy=multi-user.target

crawlers/mooncrawl/mooncrawl/api.py

@@ -30,7 +30,6 @@
 from .middleware import MoonstreamHTTPException
 from .settings import (
     BUGOUT_RESOURCE_TYPE_ENTITY_SUBSCRIPTION,
-    BUGOUT_RESOURCE_TYPE_SUBSCRIPTION,
     DOCS_TARGET_PATH,
     LINKS_EXPIRATION_TIME,
     MOONSTREAM_ADMIN_ACCESS_TOKEN,
@@ -173,7 +172,7 @@ async def status_handler(
             dashboard_subscription_filters["subscription_id"]
         ]
 
-        for reqired_field in subscription.required_fields:
+        for reqired_field in subscription.required_fields:  # type: ignore
             if "subscription_type_id" in reqired_field:
                 subscriprions_type = reqired_field["subscription_type_id"]
 

crawlers/mooncrawl/mooncrawl/blockchain.py

@@ -28,6 +28,7 @@
     MOONSTREAM_WYRM_WEB3_PROVIDER_URI,
     MOONSTREAM_XDAI_WEB3_PROVIDER_URI,
     MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI,
+    MOONSTREAM_ZKSYNC_ERA_WEB3_PROVIDER_URI,
     NB_ACCESS_ID_HEADER,
     NB_DATA_SOURCE_HEADER,
     WEB3_CLIENT_REQUEST_TIMEOUT_SECONDS,
@@ -73,6 +74,8 @@ def connect(
             web3_uri = MOONSTREAM_WYRM_WEB3_PROVIDER_URI
         elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET:
             web3_uri = MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI
+        elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA:
+            web3_uri = MOONSTREAM_ZKSYNC_ERA_WEB3_PROVIDER_URI
         else:
             raise Exception("Wrong blockchain type provided for web3 URI")
 
@@ -126,7 +129,10 @@ def add_block(db_session, block: Any, blockchain_type: AvailableBlockchainType)
     )
     if blockchain_type == AvailableBlockchainType.XDAI:
         block_obj.author = block.author
-    if blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET:
+    if (
+        blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET
+        or blockchain_type == AvailableBlockchainType.ZKSYNC_ERA
+    ):
         block_obj.mix_hash = block.get("mixHash", "")
         block_obj.sha3_uncles = block.get("sha3Uncles", "")
         block_obj.l1_batch_number = (
@@ -168,7 +174,10 @@ def add_block_transactions(
             transaction_type=int(tx["type"], 0) if tx.get("type") is not None else None,
             value=tx.value,
         )
-        if blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET:
+        if (
+            blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET
+            or blockchain_type == AvailableBlockchainType.ZKSYNC_ERA
+        ):
             tx_obj.l1_batch_number = (
                 int(tx.get("l1BatchNumber"), 0)
                 if tx.get("l1BatchNumber") is not None

crawlers/mooncrawl/mooncrawl/leaderboards_generator/cli.py

@@ -1,7 +1,6 @@
 import argparse
 import json
 import logging
-import os
 from typing import cast, List
 import uuid
 
@@ -12,7 +11,6 @@
 from ..settings import (
     MOONSTREAM_ADMIN_ACCESS_TOKEN,
     MOONSTREAM_LEADERBOARD_GENERATOR_JOURNAL_ID,
-    BUGOUT_REQUEST_TIMEOUT_SECONDS,
     MOONSTREAM_API_URL,
     MOONSTREAM_ENGINE_URL,
 )

crawlers/mooncrawl/mooncrawl/metadata_crawler/cli.py

@@ -3,19 +3,16 @@
 import logging
 import random
 import urllib.request
-from contextlib import contextmanager
 from concurrent.futures import ThreadPoolExecutor
 from typing import Any, Dict, List, Optional
 from urllib.error import HTTPError
 
 from moonstreamdb.blockchain import AvailableBlockchainType
-from sqlalchemy.orm import sessionmaker
 
 from ..db import (
     yield_db_preping_session_ctx,
     yield_db_read_only_preping_session_ctx,
 )
-from ..settings import MOONSTREAM_CRAWLERS_DB_STATEMENT_TIMEOUT_MILLIS
 from .db import (
     clean_labels_from_db,
     get_current_metadata_for_address,

crawlers/mooncrawl/mooncrawl/moonworm_crawler/continuous_crawler.py

@@ -9,7 +9,7 @@
 from moonworm.crawler.moonstream_ethereum_state_provider import (  # type: ignore
     MoonstreamEthereumStateProvider,
 )
-from moonworm.crawler.networks import Network  # type: ignore
+from moonstreamdb.networks import Network
 from sqlalchemy.orm.session import Session
 from web3 import Web3
 
@@ -132,6 +132,8 @@ def continuous_crawler(
         network = Network.wyrm
     elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET:
         network = Network.zksync_era_testnet
+    elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA:
+        network = Network.zksync_era
     else:
         raise ValueError(f"Unknown blockchain type: {blockchain_type}")
 

crawlers/mooncrawl/mooncrawl/moonworm_crawler/crawler.py

@@ -36,6 +36,7 @@ class SubscriptionTypes(Enum):
     XDAI_BLOCKCHAIN = "xdai_smartcontract"
     WYRM_BLOCKCHAIN = "wyrm_smartcontract"
     ZKSYNC_ERA_TESTNET_BLOCKCHAIN = "zksync_era_testnet_smartcontract"
+    ZKSYNC_ERA_BLOCKCHAIN = "zksync_era_smartcontract"
 
 
 def abi_input_signature(input_abi: Dict[str, Any]) -> str:
@@ -142,6 +143,8 @@ def blockchain_type_to_subscription_type(
         return SubscriptionTypes.WYRM_BLOCKCHAIN
     elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET:
         return SubscriptionTypes.ZKSYNC_ERA_TESTNET_BLOCKCHAIN
+    elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA:
+        return SubscriptionTypes.ZKSYNC_ERA_BLOCKCHAIN
     else:
         raise ValueError(f"Unknown blockchain type: {blockchain_type}")
 

crawlers/mooncrawl/mooncrawl/moonworm_crawler/function_call_crawler.py

@@ -9,7 +9,7 @@
 from moonworm.crawler.moonstream_ethereum_state_provider import (  # type: ignore
     MoonstreamEthereumStateProvider,
 )
-from moonworm.crawler.networks import Network  # type: ignore
+from moonstreamdb.networks import Network  # type: ignore
 from moonworm.watch import MockState  # type: ignore
 from sqlalchemy.orm import Session
 from web3 import Web3
@@ -70,6 +70,8 @@ def function_call_crawler(
         network = Network.wyrm
     elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET:
         network = Network.zksync_era_testnet
+    elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA:
+        network = Network.zksync_era
     else:
         raise ValueError(f"Unknown blockchain type: {blockchain_type}")
 

crawlers/mooncrawl/mooncrawl/moonworm_crawler/historical_crawler.py

@@ -1,14 +1,14 @@
 import logging
 import time
-from typing import Dict, List, Optional, Tuple
+from typing import Dict, List, Optional
 from uuid import UUID
 
 from eth_typing.evm import ChecksumAddress
 from moonstreamdb.blockchain import AvailableBlockchainType
 from moonworm.crawler.moonstream_ethereum_state_provider import (  # type: ignore
     MoonstreamEthereumStateProvider,
 )
-from moonworm.crawler.networks import Network  # type: ignore
+from moonstreamdb.networks import Network  # type: ignore
 from sqlalchemy.orm.session import Session
 from web3 import Web3
 
@@ -51,11 +51,23 @@ def historical_crawler(
         web3.eth.block_number >= start_block
     ), "start_block must be less than current block"
 
-    network = (
-        Network.ethereum
-        if blockchain_type == AvailableBlockchainType.ETHEREUM
-        else Network.polygon
-    )
+    if blockchain_type == AvailableBlockchainType.ETHEREUM:
+        network = Network.ethereum
+    elif blockchain_type == AvailableBlockchainType.POLYGON:
+        network = Network.polygon
+    elif blockchain_type == AvailableBlockchainType.MUMBAI:
+        network = Network.mumbai
+    elif blockchain_type == AvailableBlockchainType.XDAI:
+        network = Network.xdai
+    elif blockchain_type == AvailableBlockchainType.WYRM:
+        network = Network.wyrm
+    elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET:
+        network = Network.zksync_era_testnet
+    elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA:
+        network = Network.zksync_era
+    else:
+        raise Exception("Unsupported blockchain type provided")
+
     ethereum_state_provider = MoonstreamEthereumStateProvider(
         web3,
         network,