From c196c9e0d22a24b7edc3d11f3f3e604d0f18fb76 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Thu, 2 Nov 2023 11:59:31 +0000 Subject: [PATCH 01/40] Crawlers systemd monitoring --- crawlers/deploy/deploy.bash | 85 ++++++++++++------- .../deploy/monitoring-crawlers-config.json | 7 ++ crawlers/deploy/monitoring-crawlers.service | 17 ++++ 3 files changed, 79 insertions(+), 30 deletions(-) create mode 100644 crawlers/deploy/monitoring-crawlers-config.json create mode 100644 crawlers/deploy/monitoring-crawlers.service diff --git a/crawlers/deploy/deploy.bash b/crawlers/deploy/deploy.bash index fc32785a3..d0b38ea0c 100755 --- a/crawlers/deploy/deploy.bash +++ b/crawlers/deploy/deploy.bash @@ -22,19 +22,19 @@ PYTHON="${PYTHON_ENV_DIR}/bin/python" PIP="${PYTHON_ENV_DIR}/bin/pip" SECRETS_DIR="${SECRETS_DIR:-/home/ubuntu/moonstream-secrets}" PARAMETERS_ENV_PATH="${SECRETS_DIR}/app.env" +PARAMETERS_ENV_MONITORING_PATH="${SECRETS_DIR}/monitoring.env" SCRIPT_DIR="$(realpath $(dirname $0))" # Service files MOONCRAWL_SERVICE_FILE="mooncrawl.service" +MONITORING_CRAWLERS_SERVICE_FILE="monitoring-crawlers.service" LEADERBOARDS_WORKER_SERVICE_FILE="leaderboards-worker.service" LEADERBOARDS_WORKER_TIMER_FILE="leaderboards-worker.timer" - # Ethereum service files ETHEREUM_SYNCHRONIZE_SERVICE_FILE="ethereum-synchronize.service" ETHEREUM_TRENDING_SERVICE_FILE="ethereum-trending.service" ETHEREUM_TRENDING_TIMER_FILE="ethereum-trending.timer" -ETHEREUM_TXPOOL_SERVICE_FILE="ethereum-txpool.service" ETHEREUM_MISSING_SERVICE_FILE="ethereum-missing.service" ETHEREUM_MISSING_TIMER_FILE="ethereum-missing.timer" ETHEREUM_MOONWORM_CRAWLER_SERVICE_FILE="ethereum-moonworm-crawler.service" @@ -51,7 +51,6 @@ POLYGON_MISSING_SERVICE_FILE="polygon-missing.service" POLYGON_MISSING_TIMER_FILE="polygon-missing.timer" POLYGON_STATISTICS_SERVICE_FILE="polygon-statistics.service" POLYGON_STATISTICS_TIMER_FILE="polygon-statistics.timer" -POLYGON_TXPOOL_SERVICE_FILE="polygon-txpool.service" POLYGON_MOONWORM_CRAWLER_SERVICE_FILE="polygon-moonworm-crawler.service" POLYGON_STATE_SERVICE_FILE="polygon-state.service" POLYGON_STATE_TIMER_FILE="polygon-state.timer" @@ -131,14 +130,6 @@ ZKSYNC_ERA_TESTNET_HISTORICAL_CRAWL_EVENTS_TIMER_FILE="zksync-era-testnet-histor set -eu -echo -echo -echo -e "${PREFIX_INFO} Building executable Ethereum transaction pool crawler script with Go" -EXEC_DIR=$(pwd) -cd "${APP_CRAWLERS_DIR}/txpool" -HOME=/home/ubuntu /usr/local/go/bin/go build -o "${APP_CRAWLERS_DIR}/txpool/txpool" "${APP_CRAWLERS_DIR}/txpool/main.go" -cd "${EXEC_DIR}" - echo echo echo -e "${PREFIX_INFO} Upgrading Python pip and setuptools" @@ -156,8 +147,11 @@ HOME=/home/ubuntu /usr/local/go/bin/go install github.com/bugout-dev/checkenv@la echo echo -echo -e "${PREFIX_INFO} Retrieving addition deployment parameters" -mkdir -p "${SECRETS_DIR}" +echo -e "${PREFIX_INFO} Retrieving deployment parameters" +if [ ! -d "${SECRETS_DIR}" ]; then + mkdir -p "${SECRETS_DIR}" + echo -e "${PREFIX_WARN} Created new secrets directory" +fi AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION}" /home/ubuntu/go/bin/checkenv show aws_ssm+moonstream:true > "${PARAMETERS_ENV_PATH}" chmod 0640 "${PARAMETERS_ENV_PATH}" @@ -166,6 +160,45 @@ echo echo -e "${PREFIX_INFO} Add instance local IP to parameters" echo "AWS_LOCAL_IPV4=$(ec2metadata --local-ipv4)" >> "${PARAMETERS_ENV_PATH}" +echo +echo +echo -e "${PREFIX_INFO} Copy monitoring binary from AWS S3" +aws s3 cp s3://bugout-binaries/prod/monitoring/monitoring "/home/ubuntu/monitoring" +chmod +x "/home/ubuntu/monitoring" +chown ubuntu:ubuntu "/home/ubuntu/monitoring" + +echo +echo +echo -e "${PREFIX_INFO} Retrieving monitoring deployment parameters" +AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION}" /home/ubuntu/go/bin/checkenv show aws_ssm+crawlers:true,monitoring:true > "${PARAMETERS_ENV_MONITORING_PATH}" +chmod 0640 "${PARAMETERS_ENV_MONITORING_PATH}" + +echo +echo +echo -e "${PREFIX_INFO} Add instance local IP to monitoring parameters" +echo "AWS_LOCAL_IPV4=$(ec2metadata --local-ipv4)" >> "${PARAMETERS_ENV_MONITORING_PATH}" + +echo +echo +echo -e "${PREFIX_INFO} Add AWS default region to monitring parameters" +echo "AWS_DEFAULT_REGION=${AWS_DEFAULT_REGION}" >> "${PARAMETERS_ENV_MONITORING_PATH}" + +echo +echo +echo -e "${PREFIX_INFO} Prepare monitoring configuration" +if [ ! -d "/home/ubuntu/.monitoring" ]; then + mkdir -p /home/ubuntu/.monitoring + echo -e "${PREFIX_WARN} Created monitoring configuration directory" +fi +cp "${SCRIPT_DIR}/monitoring-crawlers-config.json" /home/ubuntu/.monitoring/monitoring-crawlers-config.json + +echo +echo +if [ ! -d "/home/ubuntu/.config/systemd/user/" ]; then + mkdir -p /home/ubuntu/.config/systemd/user/ + echo -e "${PREFIX_WARN} Created user systemd directory" +fi + echo echo echo -e "${PREFIX_INFO} Replacing existing Moonstream crawlers HTTP API server service definition with ${MOONCRAWL_SERVICE_FILE}" @@ -191,14 +224,6 @@ cp "${SCRIPT_DIR}/${ETHEREUM_TRENDING_TIMER_FILE}" "/home/ubuntu/.config/systemd XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ETHEREUM_TRENDING_TIMER_FILE}" -# echo -# echo -# echo -e "${PREFIX_INFO} Replacing existing Ethereum transaction pool crawler service definition with ${ETHEREUM_TXPOOL_SERVICE_FILE}" -# chmod 644 "${SCRIPT_DIR}/${ETHEREUM_TXPOOL_SERVICE_FILE}" -# cp "${SCRIPT_DIR}/${ETHEREUM_TXPOOL_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${ETHEREUM_TXPOOL_SERVICE_FILE}" -# XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload -# XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ETHEREUM_TXPOOL_SERVICE_FILE}" - echo echo echo -e "${PREFIX_INFO} Replacing existing Ethereum missing service and timer with: ${ETHEREUM_MISSING_SERVICE_FILE}, ${ETHEREUM_MISSING_TIMER_FILE}" @@ -270,14 +295,6 @@ cp "${SCRIPT_DIR}/${POLYGON_STATISTICS_TIMER_FILE}" "/home/ubuntu/.config/system XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${POLYGON_STATISTICS_TIMER_FILE}" -# echo -# echo -# echo -e "${PREFIX_INFO} Replacing existing Polygon transaction pool crawler service definition with ${POLYGON_TXPOOL_SERVICE_FILE}" -# chmod 644 "${SCRIPT_DIR}/${POLYGON_TXPOOL_SERVICE_FILE}" -# cp "${SCRIPT_DIR}/${POLYGON_TXPOOL_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${POLYGON_TXPOOL_SERVICE_FILE}" -# XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload -# XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${POLYGON_TXPOOL_SERVICE_FILE}" - echo echo echo -e "${PREFIX_INFO} Replacing existing Polygon moonworm crawler service definition with ${POLYGON_MOONWORM_CRAWLER_SERVICE_FILE}" @@ -620,4 +637,12 @@ chmod 644 "${SCRIPT_DIR}/${LEADERBOARDS_WORKER_SERVICE_FILE}" "${SCRIPT_DIR}/${L cp "${SCRIPT_DIR}/${LEADERBOARDS_WORKER_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${LEADERBOARDS_WORKER_SERVICE_FILE}" cp "${SCRIPT_DIR}/${LEADERBOARDS_WORKER_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${LEADERBOARDS_WORKER_TIMER_FILE}" XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload -XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${LEADERBOARDS_WORKER_TIMER_FILE}" \ No newline at end of file +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${LEADERBOARDS_WORKER_TIMER_FILE}" + +echo +echo +echo -e "${PREFIX_INFO} Replacing existing systemd crawlers monitoring service definition with ${MONITORING_CRAWLERS_SERVICE_FILE}" +chmod 644 "${SCRIPT_DIR}/${MONITORING_CRAWLERS_SERVICE_FILE}" +cp "${SCRIPT_DIR}/${MONITORING_CRAWLERS_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${MONITORING_CRAWLERS_SERVICE_FILE}" +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart "${MONITORING_CRAWLERS_SERVICE_FILE}" diff --git a/crawlers/deploy/monitoring-crawlers-config.json b/crawlers/deploy/monitoring-crawlers-config.json new file mode 100644 index 000000000..654139c5a --- /dev/null +++ b/crawlers/deploy/monitoring-crawlers-config.json @@ -0,0 +1,7 @@ +{ + "notification_pagerduty": true, + "notification_telegram": true, + "notification_sendgrid": true, + "notification_humbug": true, + "silent": [] +} \ No newline at end of file diff --git a/crawlers/deploy/monitoring-crawlers.service b/crawlers/deploy/monitoring-crawlers.service new file mode 100644 index 000000000..5a5aa09f5 --- /dev/null +++ b/crawlers/deploy/monitoring-crawlers.service @@ -0,0 +1,17 @@ +[Unit] +Description=Monitor crawlers systemd state +StartLimitIntervalSec=300 +StartLimitBurst=3 +After=network.target + +[Service] +Restart=on-failure +RestartSec=15s +WorkingDirectory=/home/ubuntu/ +EnvironmentFile=/home/ubuntu/moonstream-secrets/monitoring.env +ExecStart=/home/ubuntu/monitoring -plugin systemd -host "${AWS_LOCAL_IPV4}" -port 7171 -healthcheck -server -threshold 3 -config /home/ubuntu/.monitoring/monitoring-crawlers-config.json -service ethereum-moonworm-crawler.service -service mumbai-moonworm-crawler.service -service polygon-moonworm-crawler.service -service zksync-era-moonworm-crawler.service +CPUWeight=90 +SyslogIdentifier=monitoring-crawlers + +[Install] +WantedBy=multi-user.target \ No newline at end of file From 51359548e46a2ec3dbc268d39179a7008c281886 Mon Sep 17 00:00:00 2001 From: Andrey Date: Mon, 6 Nov 2023 20:10:45 +0200 Subject: [PATCH 02/40] Add changes. --- crawlers/mooncrawl/mooncrawl/leaderboards_generator/cli.py | 7 +++++++ .../mooncrawl/mooncrawl/leaderboards_generator/utils.py | 5 +++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/crawlers/mooncrawl/mooncrawl/leaderboards_generator/cli.py b/crawlers/mooncrawl/mooncrawl/leaderboards_generator/cli.py index 2908305bd..39b8f0f74 100644 --- a/crawlers/mooncrawl/mooncrawl/leaderboards_generator/cli.py +++ b/crawlers/mooncrawl/mooncrawl/leaderboards_generator/cli.py @@ -97,6 +97,7 @@ def handle_leaderboards(args: argparse.Namespace) -> None: MOONSTREAM_API_URL, args.max_retries, args.interval, + args.query_api_retries, ) except Exception as e: logger.error(f"Could not get results for query {query_name}: error: {e}") @@ -188,6 +189,12 @@ def main(): default=12, help="Number of times to retry requests for Moonstream Query results", ) + leaderboard_generator_parser.add_argument( + "--query-api-retries", + type=int, + default=3, + help="Number of times to retry updating Moonstream Query data", + ) leaderboard_generator_parser.add_argument( "--interval", type=float, diff --git a/crawlers/mooncrawl/mooncrawl/leaderboards_generator/utils.py b/crawlers/mooncrawl/mooncrawl/leaderboards_generator/utils.py index fa235cca1..fc4d1e012 100644 --- a/crawlers/mooncrawl/mooncrawl/leaderboards_generator/utils.py +++ b/crawlers/mooncrawl/mooncrawl/leaderboards_generator/utils.py @@ -23,6 +23,7 @@ def get_results_for_moonstream_query( api_url: str = MOONSTREAM_API_URL, max_retries: int = 100, interval: float = 30.0, + query_api_retries: int = 3, ) -> Optional[Dict[str, Any]]: """ @@ -65,11 +66,11 @@ def get_results_for_moonstream_query( success = False attempts = 0 - while not success and attempts < max_retries: - attempts += 1 + while not success and attempts < query_api_retries: response = requests.post( request_url, json=request_body, headers=headers, timeout=10 ) + attempts += 1 response.raise_for_status() response_body = response.json() data_url = response_body["url"] From 560ccb678400af800a63b35706832ea9d6e87f35 Mon Sep 17 00:00:00 2001 From: Neeraj Kashyap Date: Wed, 8 Nov 2023 16:19:27 -0800 Subject: [PATCH 03/40] Made models.py changes and created alembic migration --- ...3_added_leaderboard_versions_table_and_.py | 119 ++++++++++++++++++ engineapi/engineapi/models.py | 44 ++++++- 2 files changed, 161 insertions(+), 2 deletions(-) create mode 100644 engineapi/alembic/versions/cc80e886e153_added_leaderboard_versions_table_and_.py diff --git a/engineapi/alembic/versions/cc80e886e153_added_leaderboard_versions_table_and_.py b/engineapi/alembic/versions/cc80e886e153_added_leaderboard_versions_table_and_.py new file mode 100644 index 000000000..bdc773f6f --- /dev/null +++ b/engineapi/alembic/versions/cc80e886e153_added_leaderboard_versions_table_and_.py @@ -0,0 +1,119 @@ +"""Added leaderboard_versions table and corresponding constraints + +Revision ID: cc80e886e153 +Revises: 040f2dfde5a5 +Create Date: 2023-11-08 16:16:39.265150 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = "cc80e886e153" +down_revision = "040f2dfde5a5" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "leaderboard_versions", + sa.Column("leaderboard_id", sa.UUID(), nullable=False), + sa.Column("version_number", sa.DECIMAL(), nullable=False), + sa.Column("published", sa.Boolean(), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("TIMEZONE('utc', statement_timestamp())"), + nullable=False, + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.text("TIMEZONE('utc', statement_timestamp())"), + nullable=False, + ), + sa.ForeignKeyConstraint( + ["leaderboard_id"], + ["leaderboards.id"], + name=op.f("fk_leaderboard_versions_leaderboard_id_leaderboards"), + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint( + "leaderboard_id", "version_number", name=op.f("pk_leaderboard_versions") + ), + sa.UniqueConstraint( + "leaderboard_id", + "version_number", + name=op.f("uq_leaderboard_versions_leaderboard_id"), + ), + ) + op.create_index( + op.f("ix_leaderboard_versions_created_at"), + "leaderboard_versions", + ["created_at"], + unique=False, + ) + op.add_column( + "leaderboard_scores", + sa.Column("leaderboard_version_number", sa.DECIMAL(), nullable=True), + ) + op.drop_constraint( + "fk_leaderboard_scores_leaderboard_id_leaderboards", + "leaderboard_scores", + type_="foreignkey", + ) + op.create_foreign_key( + op.f("fk_leaderboard_scores_leaderboard_id_leaderboard_versions"), + "leaderboard_scores", + "leaderboard_versions", + ["leaderboard_id", "leaderboard_version_number"], + ["leaderboard_id", "version_number"], + ondelete="CASCADE", + ) + # ### end Alembic commands ### + + # Insert version 0 for all existing leaderboards + op.execute( + """ + INSERT INTO leaderboard_versions (leaderboard_id, version_number, published) + SELECT id, 0, true FROM leaderboards + """ + ) + # Set the leaderboard_version_number for all existing scores to the version 0 + op.execute( + """ + UPDATE leaderboard_scores SET leaderboard_version_number = 0 + """ + ) + # Alter leaderboard_scores to make leaderboard_version_number non-nullable + op.alter_column( + "leaderboard_scores", + "leaderboard_version_number", + nullable=False, + ) + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_constraint( + op.f("fk_leaderboard_scores_leaderboard_id_leaderboard_versions"), + "leaderboard_scores", + type_="foreignkey", + ) + op.create_foreign_key( + "fk_leaderboard_scores_leaderboard_id_leaderboards", + "leaderboard_scores", + "leaderboards", + ["leaderboard_id"], + ["id"], + ondelete="CASCADE", + ) + op.drop_column("leaderboard_scores", "leaderboard_version_number") + op.drop_index( + op.f("ix_leaderboard_versions_created_at"), table_name="leaderboard_versions" + ) + op.drop_table("leaderboard_versions") + # ### end Alembic commands ### diff --git a/engineapi/engineapi/models.py b/engineapi/engineapi/models.py index 36cc975d1..c60ee0de1 100644 --- a/engineapi/engineapi/models.py +++ b/engineapi/engineapi/models.py @@ -13,6 +13,7 @@ MetaData, String, UniqueConstraint, + ForeignKeyConstraint, ) from sqlalchemy.dialects.postgresql import JSONB, UUID from sqlalchemy.ext.compiler import compiles @@ -357,9 +358,45 @@ class Leaderboard(Base): # type: ignore ) +class LeaderboardVersion(Base): # type: ignore + __tablename__ = "leaderboard_versions" + __table_args__ = (UniqueConstraint("leaderboard_id", "version_number"),) + + leaderboard_id = Column( + UUID(as_uuid=True), + ForeignKey("leaderboards.id", ondelete="CASCADE"), + primary_key=True, + nullable=False, + ) + version_number = Column(DECIMAL, primary_key=True, nullable=False) + published = Column(Boolean, default=False, nullable=False) + created_at = Column( + DateTime(timezone=True), + server_default=utcnow(), + nullable=False, + index=True, + ) + updated_at = Column( + DateTime(timezone=True), + server_default=utcnow(), + onupdate=utcnow(), + nullable=False, + ) + + class LeaderboardScores(Base): # type: ignore __tablename__ = "leaderboard_scores" - __table_args__ = (UniqueConstraint("leaderboard_id", "address"),) + __table_args__ = ( + UniqueConstraint("leaderboard_id", "address"), + ForeignKeyConstraint( + ["leaderboard_id", "leaderboard_version_number"], + [ + "leaderboard_versions.leaderboard_id", + "leaderboard_versions.version_number", + ], + ondelete="CASCADE", + ), + ) id = Column( UUID(as_uuid=True), @@ -370,7 +407,10 @@ class LeaderboardScores(Base): # type: ignore ) leaderboard_id = Column( UUID(as_uuid=True), - ForeignKey("leaderboards.id", ondelete="CASCADE"), + nullable=False, + ) + leaderboard_version_number = Column( + DECIMAL, nullable=False, ) address = Column(VARCHAR(256), nullable=False, index=True) From 7295b9c02359bc821e905a9f093cec23a3dfd323 Mon Sep 17 00:00:00 2001 From: Neeraj Kashyap Date: Fri, 10 Nov 2023 09:20:26 -0800 Subject: [PATCH 04/40] Fixed typo --- engineapi/engineapi/actions.py | 2 +- engineapi/engineapi/routes/leaderboard.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/engineapi/engineapi/actions.py b/engineapi/engineapi/actions.py index 23c3e20fe..5c38101a9 100644 --- a/engineapi/engineapi/actions.py +++ b/engineapi/engineapi/actions.py @@ -1436,7 +1436,7 @@ def add_scores( return leaderboard_scores -# leadrboard access actions +# leaderboard access actions def create_leaderboard_resource( diff --git a/engineapi/engineapi/routes/leaderboard.py b/engineapi/engineapi/routes/leaderboard.py index 15050f317..c0e08cb87 100644 --- a/engineapi/engineapi/routes/leaderboard.py +++ b/engineapi/engineapi/routes/leaderboard.py @@ -48,7 +48,7 @@ ) -leaderboad_whitelist = { +leaderboard_whitelist = { f"/leaderboard/{DOCS_TARGET_PATH}": "GET", "/leaderboard/openapi.json": "GET", "/leaderboard/info": "GET", @@ -76,7 +76,7 @@ ) -app.add_middleware(ExtractBearerTokenMiddleware, whitelist=leaderboad_whitelist) +app.add_middleware(ExtractBearerTokenMiddleware, whitelist=leaderboard_whitelist) app.add_middleware( CORSMiddleware, From 88f36f5d63c7149a5c2b2704bb1f15f00be56ecf Mon Sep 17 00:00:00 2001 From: Neeraj Kashyap Date: Fri, 10 Nov 2023 10:20:40 -0800 Subject: [PATCH 05/40] Jobs for Twilight Tactics Updated Mumbai job, and added Polygon mainnet job. --- .../state_crawler/jobs/mumbai-jobs.json | 295 +++++----- .../state_crawler/jobs/polygon-jobs.json | 507 ++++++++++-------- 2 files changed, 431 insertions(+), 371 deletions(-) diff --git a/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/mumbai-jobs.json b/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/mumbai-jobs.json index 2acd0beea..200896d7e 100644 --- a/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/mumbai-jobs.json +++ b/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/mumbai-jobs.json @@ -1,156 +1,145 @@ [ - { - "type": "function", - "stateMutability": "view", - "inputs": [ + { + "type": "function", + "stateMutability": "view", + "inputs": [ + { + "internalType": "uint256", + "name": "tokenId", + "type": "uint256", + "value": { + "type": "function", + "name": "totalSupply", + "outputs": [ { - "internalType": "uint256", - "name": "tokenId", - "type": "uint256", - "value": { - "type": "function", - "name": "totalSupply", - "outputs": [ - { - "internalType": "uint256", - "name": "", - "type": "uint256" - } - ], - "address": "0x39858b1A4e48CfFB1019F0A15ff54899213B3f8b", - "inputs": [] - } + "internalType": "uint256", + "name": "", + "type": "uint256" } - ], - "name": "tokenURI", - "outputs": [ - { - "internalType": "string", - "name": "", - "type": "string" - } - ], - "address": "0x39858b1A4e48CfFB1019F0A15ff54899213B3f8b" - }, - { - "type": "function", - "stateMutability": "view", - "inputs": [ - { - "internalType": "uint256", - "name": "tokenId", - "type": "uint256", - "value": { - "type": "queryAPI", - "query_url": "template_erc721_minting", - "blockchain": "mumbai", - "params": { - "address": "0x230E4e85d4549343A460F5dE0a7035130F62d74C" - }, - "keys": [ - "token_id" - ] - } - } - ], - "name": "tokenURI", - "outputs": [ - { - "internalType": "string", - "name": "", - "type": "string" - } - ], - "address": "0x230E4e85d4549343A460F5dE0a7035130F62d74C" - }, - { - "inputs": [ - { - "internalType": "uint16", - "name": "seasonId", - "type": "uint16", - "value": { - "type": "queryAPI", - "query_url": "twilight_seasons", - "blockchain": "mumbai", - "params": { - "address": "0x665B8Db5b9E3b396e2Ccb0Bd768dc74fC47Ec20D" - }, - "keys": [ - "season_id" - ] - } - }, - { - "internalType": "address", - "name": "user", - "type": "address", - "value": { - "type": "queryAPI", - "query_url": "twilight_tactics_players", - "blockchain": "mumbai", - "params": { - "season_id": "1" - }, - "keys": [ - "player" - ] - } - } - ], - "name": "twtGetSeasonalDominationPointsByAccount", - "address": "0x665B8Db5b9E3b396e2Ccb0Bd768dc74fC47Ec20D", - "outputs": [ - { - "internalType": "uint56[5]", - "name": "shadowcornDominationPoints", - "type": "uint56[5]" - }, - { - "internalType": "uint56[5]", - "name": "unicornDominationPoints", - "type": "uint56[5]" - } - ], - "stateMutability": "view", - "type": "function", - "selector": "0x0b4ef829" - }, - { - "inputs": [ - { - "internalType": "uint16", - "name": "seasonId", - "type": "uint16", - "value": { - "type": "queryAPI", - "query_url": "twilight_seasons", - "blockchain": "mumbai", - "params": { - "address": "0x665B8Db5b9E3b396e2Ccb0Bd768dc74fC47Ec20D" - }, - "keys": [ - "season_id" - ] - } - } - ], - "name": "twtGetSeasonalDominationPointsForAllRegions", - "address": "0x665B8Db5b9E3b396e2Ccb0Bd768dc74fC47Ec20D", - "outputs": [ - { - "internalType": "uint56[5]", - "name": "shadowcornDominationPoints", - "type": "uint56[5]" - }, - { - "internalType": "uint56[5]", - "name": "unicornDominationPoints", - "type": "uint56[5]" - } - ], - "stateMutability": "view", - "type": "function", - "selector": "0xbddb218c" - } -] \ No newline at end of file + ], + "address": "0x39858b1A4e48CfFB1019F0A15ff54899213B3f8b", + "inputs": [] + } + } + ], + "name": "tokenURI", + "outputs": [ + { + "internalType": "string", + "name": "", + "type": "string" + } + ], + "address": "0x39858b1A4e48CfFB1019F0A15ff54899213B3f8b" + }, + { + "type": "function", + "stateMutability": "view", + "inputs": [ + { + "internalType": "uint256", + "name": "tokenId", + "type": "uint256", + "value": { + "type": "queryAPI", + "query_url": "template_erc721_minting", + "blockchain": "mumbai", + "params": { + "address": "0x230E4e85d4549343A460F5dE0a7035130F62d74C" + }, + "keys": ["token_id"] + } + } + ], + "name": "tokenURI", + "outputs": [ + { + "internalType": "string", + "name": "", + "type": "string" + } + ], + "address": "0x230E4e85d4549343A460F5dE0a7035130F62d74C" + }, + { + "inputs": [ + { + "internalType": "uint16", + "name": "seasonId", + "type": "uint16", + "value": { + "type": "queryAPI", + "query_url": "twilight_seasons", + "blockchain": "mumbai", + "params": { + "address": "0x665B8Db5b9E3b396e2Ccb0Bd768dc74fC47Ec20D" + }, + "keys": ["season_id"] + } + }, + { + "internalType": "address", + "name": "user", + "type": "address", + "value": { + "type": "queryAPI", + "query_url": "twilight_tactics_players", + "blockchain": "mumbai", + "keys": ["player"] + } + } + ], + "name": "twtGetSeasonalDominationPointsByAccount", + "address": "0x665B8Db5b9E3b396e2Ccb0Bd768dc74fC47Ec20D", + "outputs": [ + { + "internalType": "uint56[5]", + "name": "shadowcornDominationPoints", + "type": "uint56[5]" + }, + { + "internalType": "uint56[5]", + "name": "unicornDominationPoints", + "type": "uint56[5]" + } + ], + "stateMutability": "view", + "type": "function", + "selector": "0x0b4ef829" + }, + { + "inputs": [ + { + "internalType": "uint16", + "name": "seasonId", + "type": "uint16", + "value": { + "type": "queryAPI", + "query_url": "twilight_seasons", + "blockchain": "mumbai", + "params": { + "address": "0x665B8Db5b9E3b396e2Ccb0Bd768dc74fC47Ec20D" + }, + "keys": ["season_id"] + } + } + ], + "name": "twtGetSeasonalDominationPointsForAllRegions", + "address": "0x665B8Db5b9E3b396e2Ccb0Bd768dc74fC47Ec20D", + "outputs": [ + { + "internalType": "uint56[5]", + "name": "shadowcornDominationPoints", + "type": "uint56[5]" + }, + { + "internalType": "uint56[5]", + "name": "unicornDominationPoints", + "type": "uint56[5]" + } + ], + "stateMutability": "view", + "type": "function", + "selector": "0xbddb218c" + } +] diff --git a/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/polygon-jobs.json b/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/polygon-jobs.json index 3f230278a..b5d6eb33d 100644 --- a/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/polygon-jobs.json +++ b/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/polygon-jobs.json @@ -1,223 +1,294 @@ [ - { - "type": "function", - "stateMutability": "view", - "inputs": [ + { + "type": "function", + "stateMutability": "view", + "inputs": [ + { + "internalType": "uint256", + "name": "tokenId", + "type": "uint256", + "value": { + "type": "function", + "name": "totalSupply", + "outputs": [ { - "internalType": "uint256", - "name": "tokenId", - "type": "uint256", - "value": { - "type": "function", - "name": "totalSupply", - "outputs": [ - { - "internalType": "uint256", - "name": "", - "type": "uint256" - } - ], - "address": "0xdC0479CC5BbA033B3e7De9F178607150B3AbCe1f", - "inputs": [] - } + "internalType": "uint256", + "name": "", + "type": "uint256" } - ], - "name": "tokenURI", - "outputs": [ + ], + "address": "0xdC0479CC5BbA033B3e7De9F178607150B3AbCe1f", + "inputs": [] + } + } + ], + "name": "tokenURI", + "outputs": [ + { + "internalType": "string", + "name": "", + "type": "string" + } + ], + "address": "0xdC0479CC5BbA033B3e7De9F178607150B3AbCe1f" + }, + { + "type": "function", + "stateMutability": "view", + "inputs": [ + { + "internalType": "uint256", + "name": "tokenId", + "type": "uint256", + "value": { + "type": "queryAPI", + "query_url": "template_erc721_minting", + "blockchain": "polygon", + "params": { + "address": "0xA2a13cE1824F3916fC84C65e559391fc6674e6e8" + }, + "keys": ["token_id"] + } + } + ], + "name": "tokenURI", + "outputs": [ + { + "internalType": "string", + "name": "", + "type": "string" + } + ], + "address": "0xA2a13cE1824F3916fC84C65e559391fc6674e6e8" + }, + { + "type": "function", + "stateMutability": "view", + "inputs": [ + { + "internalType": "uint256", + "name": "tokenId", + "type": "uint256", + "value": { + "type": "function", + "name": "totalSupply", + "outputs": [ { - "internalType": "string", - "name": "", - "type": "string" + "internalType": "uint256", + "name": "", + "type": "uint256" } - ], - "address": "0xdC0479CC5BbA033B3e7De9F178607150B3AbCe1f" - }, - { - "type": "function", - "stateMutability": "view", - "inputs": [ - { - "internalType": "uint256", - "name": "tokenId", - "type": "uint256", - "value": { - "type": "queryAPI", - "query_url": "template_erc721_minting", - "blockchain": "polygon", - "params": { - "address": "0xA2a13cE1824F3916fC84C65e559391fc6674e6e8" - }, - "keys": [ - "token_id" - ] - } - } - ], - "name": "tokenURI", - "outputs": [ - { - "internalType": "string", - "name": "", - "type": "string" - } - ], - "address": "0xA2a13cE1824F3916fC84C65e559391fc6674e6e8" - }, - { - "type": "function", - "stateMutability": "view", - "inputs": [ - { - "internalType": "uint256", - "name": "tokenId", - "type": "uint256", - "value": { - "type": "function", - "name": "totalSupply", - "outputs": [ - { - "internalType": "uint256", - "name": "", - "type": "uint256" - } - ], - "address": "0x20B807b9AF56977EF475C089A0e7977540743560", - "inputs": [] - } - } - ], - "name": "tokenURI", - "outputs": [ - { - "internalType": "string", - "name": "", - "type": "string" - } - ], - "address": "0x20B807b9AF56977EF475C089A0e7977540743560" - }, - { - "type": "function", - "stateMutability": "view", - "inputs": [ - { - "internalType": "uint256", - "name": "tokenId", - "type": "uint256", - "value": { - "type": "queryAPI", - "query_url": "template_erc1155_token_ids", - "blockchain": "polygon", - "params": { - "address": "0xd4D53d8D61adc3B8114C1cd17B89393640db9733" - }, - "keys": [ - "token_id" - ] - } - } - ], - "name": "uri", - "outputs": [ - { - "internalType": "string", - "name": "", - "type": "string" - } - ], - "address": "0xd4D53d8D61adc3B8114C1cd17B89393640db9733" - }, - { - "type": "function", - "stateMutability": "view", - "inputs": [ - { - "internalType": "uint256", - "name": "tokenId", - "type": "uint256", - "value": { - "type": "queryAPI", - "query_url": "template_erc1155_token_ids", - "blockchain": "polygon", - "params": { - "address": "0x74d4567fd8B0b873B61FA180618a82183012F369" - }, - "keys": [ - "token_id" - ] - } - } - ], - "name": "uri", - "outputs": [ - { - "internalType": "string", - "name": "", - "type": "string" - } - ], - "address": "0x74d4567fd8B0b873B61FA180618a82183012F369" - }, - { - "type": "function", - "stateMutability": "view", - "inputs": [ - { - "internalType": "uint256", - "name": "tokenId", - "type": "uint256", - "value": { - "type": "queryAPI", - "query_url": "template_erc1155_token_ids", - "blockchain": "polygon", - "params": { - "address": "0x44b3f42e2BF34F62868Ff9e9dAb7C2F807ba97Cb" - }, - "keys": [ - "token_id" - ] - } - } - ], - "name": "uri", - "outputs": [ - { - "internalType": "string", - "name": "", - "type": "string" - } - ], - "address": "0x44b3f42e2BF34F62868Ff9e9dAb7C2F807ba97Cb" - }, - { - "type": "function", - "stateMutability": "view", - "inputs": [ - { - "internalType": "uint256", - "name": "tokenId", - "type": "uint256", - "value": { - "type": "queryAPI", - "query_url": "template_erc721_minting", - "blockchain": "polygon", - "params": { - "address": "0xa7D50EE3D7485288107664cf758E877a0D351725" - }, - "keys": [ - "token_id" - ] - } - } - ], - "name": "tokenURI", - "outputs": [ - { - "internalType": "string", - "name": "", - "type": "string" - } - ], - "address": "0xa7D50EE3D7485288107664cf758E877a0D351725" - } -] \ No newline at end of file + ], + "address": "0x20B807b9AF56977EF475C089A0e7977540743560", + "inputs": [] + } + } + ], + "name": "tokenURI", + "outputs": [ + { + "internalType": "string", + "name": "", + "type": "string" + } + ], + "address": "0x20B807b9AF56977EF475C089A0e7977540743560" + }, + { + "type": "function", + "stateMutability": "view", + "inputs": [ + { + "internalType": "uint256", + "name": "tokenId", + "type": "uint256", + "value": { + "type": "queryAPI", + "query_url": "template_erc1155_token_ids", + "blockchain": "polygon", + "params": { + "address": "0xd4D53d8D61adc3B8114C1cd17B89393640db9733" + }, + "keys": ["token_id"] + } + } + ], + "name": "uri", + "outputs": [ + { + "internalType": "string", + "name": "", + "type": "string" + } + ], + "address": "0xd4D53d8D61adc3B8114C1cd17B89393640db9733" + }, + { + "type": "function", + "stateMutability": "view", + "inputs": [ + { + "internalType": "uint256", + "name": "tokenId", + "type": "uint256", + "value": { + "type": "queryAPI", + "query_url": "template_erc1155_token_ids", + "blockchain": "polygon", + "params": { + "address": "0x74d4567fd8B0b873B61FA180618a82183012F369" + }, + "keys": ["token_id"] + } + } + ], + "name": "uri", + "outputs": [ + { + "internalType": "string", + "name": "", + "type": "string" + } + ], + "address": "0x74d4567fd8B0b873B61FA180618a82183012F369" + }, + { + "type": "function", + "stateMutability": "view", + "inputs": [ + { + "internalType": "uint256", + "name": "tokenId", + "type": "uint256", + "value": { + "type": "queryAPI", + "query_url": "template_erc1155_token_ids", + "blockchain": "polygon", + "params": { + "address": "0x44b3f42e2BF34F62868Ff9e9dAb7C2F807ba97Cb" + }, + "keys": ["token_id"] + } + } + ], + "name": "uri", + "outputs": [ + { + "internalType": "string", + "name": "", + "type": "string" + } + ], + "address": "0x44b3f42e2BF34F62868Ff9e9dAb7C2F807ba97Cb" + }, + { + "type": "function", + "stateMutability": "view", + "inputs": [ + { + "internalType": "uint256", + "name": "tokenId", + "type": "uint256", + "value": { + "type": "queryAPI", + "query_url": "template_erc721_minting", + "blockchain": "polygon", + "params": { + "address": "0xa7D50EE3D7485288107664cf758E877a0D351725" + }, + "keys": ["token_id"] + } + } + ], + "name": "tokenURI", + "outputs": [ + { + "internalType": "string", + "name": "", + "type": "string" + } + ], + "address": "0xa7D50EE3D7485288107664cf758E877a0D351725" + }, + { + "inputs": [ + { + "internalType": "uint16", + "name": "seasonId", + "type": "uint16", + "value": { + "type": "queryAPI", + "query_url": "twilight_seasons", + "blockchain": "polygon", + "params": { + "address": "0xe570fAC6513A8018145aB398Ea71988C688F22C4" + }, + "keys": ["season_id"] + } + }, + { + "internalType": "address", + "name": "user", + "type": "address", + "value": { + "type": "queryAPI", + "query_url": "twilight_tactics_players", + "blockchain": "polygon", + "keys": ["player"] + } + } + ], + "name": "twtGetSeasonalDominationPointsByAccount", + "address": "0xe570fAC6513A8018145aB398Ea71988C688F22C4", + "outputs": [ + { + "internalType": "uint56[5]", + "name": "shadowcornDominationPoints", + "type": "uint56[5]" + }, + { + "internalType": "uint56[5]", + "name": "unicornDominationPoints", + "type": "uint56[5]" + } + ], + "stateMutability": "view", + "type": "function", + "selector": "0x0b4ef829" + }, + { + "inputs": [ + { + "internalType": "uint16", + "name": "seasonId", + "type": "uint16", + "value": { + "type": "queryAPI", + "query_url": "twilight_seasons", + "blockchain": "polygon", + "params": { + "address": "0xe570fAC6513A8018145aB398Ea71988C688F22C4" + }, + "keys": ["season_id"] + } + } + ], + "name": "twtGetSeasonalDominationPointsForAllRegions", + "address": "0xe570fAC6513A8018145aB398Ea71988C688F22C4", + "outputs": [ + { + "internalType": "uint56[5]", + "name": "shadowcornDominationPoints", + "type": "uint56[5]" + }, + { + "internalType": "uint56[5]", + "name": "unicornDominationPoints", + "type": "uint56[5]" + } + ], + "stateMutability": "view", + "type": "function", + "selector": "0xbddb218c" + } +] From 825b699b3f70251b667288c2e78068727984b792 Mon Sep 17 00:00:00 2001 From: Neeraj Kashyap Date: Fri, 10 Nov 2023 10:30:21 -0800 Subject: [PATCH 06/40] Fixed lacking "params" key --- crawlers/mooncrawl/mooncrawl/state_crawler/jobs/mumbai-jobs.json | 1 + .../mooncrawl/mooncrawl/state_crawler/jobs/polygon-jobs.json | 1 + 2 files changed, 2 insertions(+) diff --git a/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/mumbai-jobs.json b/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/mumbai-jobs.json index 200896d7e..0286508e0 100644 --- a/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/mumbai-jobs.json +++ b/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/mumbai-jobs.json @@ -85,6 +85,7 @@ "type": "queryAPI", "query_url": "twilight_tactics_players", "blockchain": "mumbai", + "params": {}, "keys": ["player"] } } diff --git a/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/polygon-jobs.json b/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/polygon-jobs.json index b5d6eb33d..44e66d8a7 100644 --- a/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/polygon-jobs.json +++ b/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/polygon-jobs.json @@ -234,6 +234,7 @@ "type": "queryAPI", "query_url": "twilight_tactics_players", "blockchain": "polygon", + "params": {}, "keys": ["player"] } } From 532e1bdae4cd22945eb78fcd87dbab98599f5f42 Mon Sep 17 00:00:00 2001 From: Andrey Date: Mon, 13 Nov 2023 17:34:36 +0200 Subject: [PATCH 07/40] Add changes. --- .../state_crawler/jobs/mumbai-jobs.json | 296 +++++++++--------- .../state_crawler/jobs/polygon-jobs.json | 38 ++- 2 files changed, 181 insertions(+), 153 deletions(-) diff --git a/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/mumbai-jobs.json b/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/mumbai-jobs.json index 0286508e0..e255672d0 100644 --- a/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/mumbai-jobs.json +++ b/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/mumbai-jobs.json @@ -1,146 +1,156 @@ [ - { - "type": "function", - "stateMutability": "view", - "inputs": [ - { - "internalType": "uint256", - "name": "tokenId", - "type": "uint256", - "value": { - "type": "function", - "name": "totalSupply", - "outputs": [ + { + "type": "function", + "stateMutability": "view", + "inputs": [ { - "internalType": "uint256", - "name": "", - "type": "uint256" + "internalType": "uint256", + "name": "tokenId", + "type": "uint256", + "value": { + "type": "function", + "name": "totalSupply", + "outputs": [ + { + "internalType": "uint256", + "name": "", + "type": "uint256" + } + ], + "address": "0x39858b1A4e48CfFB1019F0A15ff54899213B3f8b", + "inputs": [] + } } - ], - "address": "0x39858b1A4e48CfFB1019F0A15ff54899213B3f8b", - "inputs": [] - } - } - ], - "name": "tokenURI", - "outputs": [ - { - "internalType": "string", - "name": "", - "type": "string" - } - ], - "address": "0x39858b1A4e48CfFB1019F0A15ff54899213B3f8b" - }, - { - "type": "function", - "stateMutability": "view", - "inputs": [ - { - "internalType": "uint256", - "name": "tokenId", - "type": "uint256", - "value": { - "type": "queryAPI", - "query_url": "template_erc721_minting", - "blockchain": "mumbai", - "params": { - "address": "0x230E4e85d4549343A460F5dE0a7035130F62d74C" - }, - "keys": ["token_id"] - } - } - ], - "name": "tokenURI", - "outputs": [ - { - "internalType": "string", - "name": "", - "type": "string" - } - ], - "address": "0x230E4e85d4549343A460F5dE0a7035130F62d74C" - }, - { - "inputs": [ - { - "internalType": "uint16", - "name": "seasonId", - "type": "uint16", - "value": { - "type": "queryAPI", - "query_url": "twilight_seasons", - "blockchain": "mumbai", - "params": { - "address": "0x665B8Db5b9E3b396e2Ccb0Bd768dc74fC47Ec20D" - }, - "keys": ["season_id"] - } - }, - { - "internalType": "address", - "name": "user", - "type": "address", - "value": { - "type": "queryAPI", - "query_url": "twilight_tactics_players", - "blockchain": "mumbai", - "params": {}, - "keys": ["player"] - } - } - ], - "name": "twtGetSeasonalDominationPointsByAccount", - "address": "0x665B8Db5b9E3b396e2Ccb0Bd768dc74fC47Ec20D", - "outputs": [ - { - "internalType": "uint56[5]", - "name": "shadowcornDominationPoints", - "type": "uint56[5]" - }, - { - "internalType": "uint56[5]", - "name": "unicornDominationPoints", - "type": "uint56[5]" - } - ], - "stateMutability": "view", - "type": "function", - "selector": "0x0b4ef829" - }, - { - "inputs": [ - { - "internalType": "uint16", - "name": "seasonId", - "type": "uint16", - "value": { - "type": "queryAPI", - "query_url": "twilight_seasons", - "blockchain": "mumbai", - "params": { - "address": "0x665B8Db5b9E3b396e2Ccb0Bd768dc74fC47Ec20D" - }, - "keys": ["season_id"] - } - } - ], - "name": "twtGetSeasonalDominationPointsForAllRegions", - "address": "0x665B8Db5b9E3b396e2Ccb0Bd768dc74fC47Ec20D", - "outputs": [ - { - "internalType": "uint56[5]", - "name": "shadowcornDominationPoints", - "type": "uint56[5]" - }, - { - "internalType": "uint56[5]", - "name": "unicornDominationPoints", - "type": "uint56[5]" - } - ], - "stateMutability": "view", - "type": "function", - "selector": "0xbddb218c" - } -] + ], + "name": "tokenURI", + "outputs": [ + { + "internalType": "string", + "name": "", + "type": "string" + } + ], + "address": "0x39858b1A4e48CfFB1019F0A15ff54899213B3f8b" + }, + { + "type": "function", + "stateMutability": "view", + "inputs": [ + { + "internalType": "uint256", + "name": "tokenId", + "type": "uint256", + "value": { + "type": "queryAPI", + "query_url": "template_erc721_minting", + "blockchain": "mumbai", + "params": { + "address": "0x230E4e85d4549343A460F5dE0a7035130F62d74C" + }, + "keys": [ + "token_id" + ] + } + } + ], + "name": "tokenURI", + "outputs": [ + { + "internalType": "string", + "name": "", + "type": "string" + } + ], + "address": "0x230E4e85d4549343A460F5dE0a7035130F62d74C" + }, + { + "inputs": [ + { + "internalType": "uint16", + "name": "seasonId", + "type": "uint16", + "value": { + "type": "queryAPI", + "query_url": "twilight_seasons", + "blockchain": "mumbai", + "params": { + "address": "0x665B8Db5b9E3b396e2Ccb0Bd768dc74fC47Ec20D" + }, + "keys": [ + "season_id" + ] + } + }, + { + "internalType": "address", + "name": "user", + "type": "address", + "value": { + "type": "queryAPI", + "query_url": "twilight_tactics_players", + "blockchain": "mumbai", + "params": { + "address": "0x665B8Db5b9E3b396e2Ccb0Bd768dc74fC47Ec20D" + }, + "keys": [ + "player" + ] + } + } + ], + "name": "twtGetSeasonalDominationPointsByAccount", + "address": "0x665B8Db5b9E3b396e2Ccb0Bd768dc74fC47Ec20D", + "outputs": [ + { + "internalType": "uint56[5]", + "name": "shadowcornDominationPoints", + "type": "uint56[5]" + }, + { + "internalType": "uint56[5]", + "name": "unicornDominationPoints", + "type": "uint56[5]" + } + ], + "stateMutability": "view", + "type": "function", + "selector": "0x0b4ef829" + }, + { + "inputs": [ + { + "internalType": "uint16", + "name": "seasonId", + "type": "uint16", + "value": { + "type": "queryAPI", + "query_url": "twilight_seasons", + "blockchain": "mumbai", + "params": { + "address": "0x665B8Db5b9E3b396e2Ccb0Bd768dc74fC47Ec20D" + }, + "keys": [ + "season_id" + ] + } + } + ], + "name": "twtGetSeasonalDominationPointsForAllRegions", + "address": "0x665B8Db5b9E3b396e2Ccb0Bd768dc74fC47Ec20D", + "outputs": [ + { + "internalType": "uint56[5]", + "name": "shadowcornDominationPoints", + "type": "uint56[5]" + }, + { + "internalType": "uint56[5]", + "name": "unicornDominationPoints", + "type": "uint56[5]" + } + ], + "stateMutability": "view", + "type": "function", + "selector": "0xbddb218c" + } +] \ No newline at end of file diff --git a/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/polygon-jobs.json b/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/polygon-jobs.json index 44e66d8a7..9ed0986f0 100644 --- a/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/polygon-jobs.json +++ b/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/polygon-jobs.json @@ -47,7 +47,9 @@ "params": { "address": "0xA2a13cE1824F3916fC84C65e559391fc6674e6e8" }, - "keys": ["token_id"] + "keys": [ + "token_id" + ] } } ], @@ -109,7 +111,9 @@ "params": { "address": "0xd4D53d8D61adc3B8114C1cd17B89393640db9733" }, - "keys": ["token_id"] + "keys": [ + "token_id" + ] } } ], @@ -138,7 +142,9 @@ "params": { "address": "0x74d4567fd8B0b873B61FA180618a82183012F369" }, - "keys": ["token_id"] + "keys": [ + "token_id" + ] } } ], @@ -167,7 +173,9 @@ "params": { "address": "0x44b3f42e2BF34F62868Ff9e9dAb7C2F807ba97Cb" }, - "keys": ["token_id"] + "keys": [ + "token_id" + ] } } ], @@ -196,7 +204,9 @@ "params": { "address": "0xa7D50EE3D7485288107664cf758E877a0D351725" }, - "keys": ["token_id"] + "keys": [ + "token_id" + ] } } ], @@ -223,7 +233,9 @@ "params": { "address": "0xe570fAC6513A8018145aB398Ea71988C688F22C4" }, - "keys": ["season_id"] + "keys": [ + "season_id" + ] } }, { @@ -234,8 +246,12 @@ "type": "queryAPI", "query_url": "twilight_tactics_players", "blockchain": "polygon", - "params": {}, - "keys": ["player"] + "params": { + "address": "0x665B8Db5b9E3b396e2Ccb0Bd768dc74fC47Ec20D" + }, + "keys": [ + "player" + ] } } ], @@ -270,7 +286,9 @@ "params": { "address": "0xe570fAC6513A8018145aB398Ea71988C688F22C4" }, - "keys": ["season_id"] + "keys": [ + "season_id" + ] } } ], @@ -292,4 +310,4 @@ "type": "function", "selector": "0xbddb218c" } -] +] \ No newline at end of file From e217434e4e70c54cacc932fdf2cbed180fbceac3 Mon Sep 17 00:00:00 2001 From: Andrey Date: Mon, 13 Nov 2023 17:37:30 +0200 Subject: [PATCH 08/40] Add changes. --- .../mooncrawl/mooncrawl/state_crawler/jobs/polygon-jobs.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/polygon-jobs.json b/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/polygon-jobs.json index 9ed0986f0..c25a9a097 100644 --- a/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/polygon-jobs.json +++ b/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/polygon-jobs.json @@ -247,7 +247,7 @@ "query_url": "twilight_tactics_players", "blockchain": "polygon", "params": { - "address": "0x665B8Db5b9E3b396e2Ccb0Bd768dc74fC47Ec20D" + "address": "0xe570fAC6513A8018145aB398Ea71988C688F22C4" }, "keys": [ "player" From e6b75d59d918fba0177d65a922c67990f7864011 Mon Sep 17 00:00:00 2001 From: Andrey Date: Mon, 20 Nov 2023 10:51:31 +0200 Subject: [PATCH 09/40] Add endpoints for LeaderboardVersions. Change add_scores. Change get leaderboards positions. --- engineapi/engineapi/actions.py | 336 ++++++++++++++-- engineapi/engineapi/data.py | 8 + engineapi/engineapi/routes/leaderboard.py | 446 +++++++++++++++++++++- 3 files changed, 746 insertions(+), 44 deletions(-) diff --git a/engineapi/engineapi/actions.py b/engineapi/engineapi/actions.py index 5c38101a9..375e14688 100644 --- a/engineapi/engineapi/actions.py +++ b/engineapi/engineapi/actions.py @@ -11,7 +11,7 @@ import requests # type: ignore from sqlalchemy.dialects.postgresql import insert from sqlalchemy.orm import Session -from sqlalchemy import func, text, or_ +from sqlalchemy import func, text, or_, Subquery from sqlalchemy.engine import Row from web3 import Web3 from web3.types import ChecksumAddress @@ -24,6 +24,7 @@ DropperClaim, Leaderboard, LeaderboardScores, + LeaderboardVersion, ) from . import signatures from .settings import ( @@ -91,6 +92,10 @@ class LeaderboardConfigAlreadyInactive(Exception): pass +class LeaderboardVersionNotFound(Exception): + pass + + BATCH_SIGNATURE_PAGE_SIZE = 500 logger = logging.getLogger(__name__) @@ -959,6 +964,25 @@ def refetch_drop_signatures( return claimant_objects +def leaderboard_version_filter( + db_session: Session, + leaderboard_id: uuid.UUID, + version_number: Optional[int] = None, +) -> Union[Subquery, int]: + # Subquery to get the latest version number for the given leaderboard + if not version_number: + latest_version = ( + db_session.query(func.max(LeaderboardVersion.version_number)).filter( + LeaderboardVersion.leaderboard_id == leaderboard_id, + LeaderboardVersion.published == True, + ) + ).scalar_subquery() + else: + latest_version = version_number + + return latest_version + + def get_leaderboard_total_count(db_session: Session, leaderboard_id) -> int: """ Get the total number of claimants in the leaderboard @@ -971,12 +995,17 @@ def get_leaderboard_total_count(db_session: Session, leaderboard_id) -> int: def get_leaderboard_info( - db_session: Session, leaderboard_id: uuid.UUID + db_session: Session, leaderboard_id: uuid.UUID, version_number: Optional[int] = None ) -> Row[Tuple[uuid.UUID, str, str, int, Optional[datetime]]]: """ Get the leaderboard from the database with users count """ + latest_version = leaderboard_version_filter( + db_session=db_session, + leaderboard_id=leaderboard_id, + version_number=version_number, + ) leaderboard = ( db_session.query( Leaderboard.id, @@ -990,6 +1019,15 @@ def get_leaderboard_info( LeaderboardScores.leaderboard_id == Leaderboard.id, isouter=True, ) + .join( + LeaderboardVersion, + LeaderboardVersion.leaderboard_id == Leaderboard.id, + isouter=True, + ) + .filter( + LeaderboardVersion.published == True, + LeaderboardVersion.version_number == latest_version, + ) .filter(Leaderboard.id == leaderboard_id) .group_by(Leaderboard.id, Leaderboard.title, Leaderboard.description) .one() @@ -1078,19 +1116,44 @@ def get_leaderboards( def get_position( - db_session: Session, leaderboard_id, address, window_size, limit: int, offset: int + db_session: Session, + leaderboard_id, + address, + window_size, + limit: int, + offset: int, + version_number: Optional[int] = None, ) -> List[Row[Tuple[str, int, int, int, Any]]]: """ - Return position by address with window size """ - query = db_session.query( - LeaderboardScores.address, - LeaderboardScores.score, - LeaderboardScores.points_data.label("points_data"), - func.rank().over(order_by=LeaderboardScores.score.desc()).label("rank"), - func.row_number().over(order_by=LeaderboardScores.score.desc()).label("number"), - ).filter(LeaderboardScores.leaderboard_id == leaderboard_id) + + latest_version = leaderboard_version_filter( + db_session=db_session, + leaderboard_id=leaderboard_id, + version_number=version_number, + ) + + query = ( + db_session.query( + LeaderboardScores.address, + LeaderboardScores.score, + LeaderboardScores.points_data.label("points_data"), + func.rank().over(order_by=LeaderboardScores.score.desc()).label("rank"), + func.row_number() + .over(order_by=LeaderboardScores.score.desc()) + .label("number"), + ) + .join( + LeaderboardVersion, + LeaderboardVersion.leaderboard_id == LeaderboardScores.leaderboard_id, + ) + .filter( + LeaderboardVersion.published == True, + LeaderboardVersion.version_number == latest_version, + ) + .filter(LeaderboardScores.leaderboard_id == leaderboard_id) + ) ranked_leaderboard = query.cte(name="ranked_leaderboard") @@ -1130,11 +1193,25 @@ def get_position( def get_leaderboard_positions( - db_session: Session, leaderboard_id, limit: int, offset: int + db_session: Session, + leaderboard_id, + limit: int, + offset: int, + version_number: Optional[int] = None, ) -> List[Row[Tuple[uuid.UUID, str, int, str, int]]]: """ Get the leaderboard positions """ + + # get public leaderboard scores with max version + + latest_version = leaderboard_version_filter( + db_session=db_session, + leaderboard_id=leaderboard_id, + version_number=version_number, + ) + + # Main query query = ( db_session.query( LeaderboardScores.id, @@ -1143,8 +1220,13 @@ def get_leaderboard_positions( LeaderboardScores.points_data, func.rank().over(order_by=LeaderboardScores.score.desc()).label("rank"), ) + .join( + LeaderboardVersion, + LeaderboardVersion.leaderboard_id == LeaderboardScores.leaderboard_id, + ) .filter(LeaderboardScores.leaderboard_id == leaderboard_id) - .order_by(text("rank asc, id asc")) + .filter(LeaderboardVersion.published == True) + .filter(LeaderboardVersion.version_number == latest_version) ) if limit: @@ -1157,18 +1239,35 @@ def get_leaderboard_positions( def get_qurtiles( - db_session: Session, leaderboard_id + db_session: Session, leaderboard_id, version_number: Optional[int] = None ) -> Tuple[Row[Tuple[str, float, int]], ...]: """ Get the leaderboard qurtiles https://docs.sqlalchemy.org/en/14/core/functions.html#sqlalchemy.sql.functions.percentile_disc """ - query = db_session.query( - LeaderboardScores.address, - LeaderboardScores.score, - func.rank().over(order_by=LeaderboardScores.score.desc()).label("rank"), - ).filter(LeaderboardScores.leaderboard_id == leaderboard_id) + latest_version = leaderboard_version_filter( + db_session=db_session, + leaderboard_id=leaderboard_id, + version_number=version_number, + ) + + query = ( + db_session.query( + LeaderboardScores.address, + LeaderboardScores.score, + func.rank().over(order_by=LeaderboardScores.score.desc()).label("rank"), + ) + .join( + LeaderboardVersion, + LeaderboardVersion.leaderboard_id == LeaderboardScores.leaderboard_id, + ) + .filter( + LeaderboardVersion.published == True, + LeaderboardVersion.version_number == latest_version, + ) + .filter(LeaderboardScores.leaderboard_id == leaderboard_id) + ) ranked_leaderboard = query.cte(name="ranked_leaderboard") @@ -1192,17 +1291,37 @@ def get_qurtiles( return q1, q2, q3 -def get_ranks(db_session: Session, leaderboard_id) -> List[Row[Tuple[int, int, int]]]: +def get_ranks( + db_session: Session, leaderboard_id, version_number: Optional[int] = None +) -> List[Row[Tuple[int, int, int]]]: """ Get the leaderboard rank buckets(rank, size, score) """ - query = db_session.query( - LeaderboardScores.id, - LeaderboardScores.address, - LeaderboardScores.score, - LeaderboardScores.points_data, - func.rank().over(order_by=LeaderboardScores.score.desc()).label("rank"), - ).filter(LeaderboardScores.leaderboard_id == leaderboard_id) + + latest_version = leaderboard_version_filter( + db_session=db_session, + leaderboard_id=leaderboard_id, + version_number=version_number, + ) + + query = ( + db_session.query( + LeaderboardScores.id, + LeaderboardScores.address, + LeaderboardScores.score, + LeaderboardScores.points_data, + func.rank().over(order_by=LeaderboardScores.score.desc()).label("rank"), + ) + .join( + LeaderboardVersion, + LeaderboardVersion.leaderboard_id == LeaderboardScores.leaderboard_id, + ) + .filter( + LeaderboardVersion.published == True, + LeaderboardVersion.version_number == latest_version, + ) + .filter(LeaderboardScores.leaderboard_id == leaderboard_id) + ) ranked_leaderboard = query.cte(name="ranked_leaderboard") @@ -1220,10 +1339,18 @@ def get_rank( rank: int, limit: Optional[int] = None, offset: Optional[int] = None, + version_number: Optional[int] = None, ) -> List[Row[Tuple[uuid.UUID, str, int, str, int]]]: """ Get bucket in leaderboard by rank """ + + latest_version = leaderboard_version_filter( + db_session=db_session, + leaderboard_id=leaderboard_id, + version_number=version_number, + ) + query = ( db_session.query( LeaderboardScores.id, @@ -1232,6 +1359,14 @@ def get_rank( LeaderboardScores.points_data, func.rank().over(order_by=LeaderboardScores.score.desc()).label("rank"), ) + .join( + LeaderboardVersion, + LeaderboardVersion.leaderboard_id == LeaderboardScores.leaderboard_id, + ) + .filter( + LeaderboardVersion.published == True, + LeaderboardVersion.version_number == latest_version, + ) .filter(LeaderboardScores.leaderboard_id == leaderboard_id) .order_by(text("rank asc, id asc")) ) @@ -1377,7 +1512,7 @@ def add_scores( db_session: Session, leaderboard_id: uuid.UUID, scores: List[Score], - overwrite: bool = False, + version_number: int, normalize_addresses: bool = True, ): """ @@ -1397,16 +1532,6 @@ def add_scores( raise DuplicateLeaderboardAddressError("Dublicated addresses", duplicates) - if overwrite: - db_session.query(LeaderboardScores).filter( - LeaderboardScores.leaderboard_id == leaderboard_id - ).delete() - try: - db_session.commit() - except: - db_session.rollback() - raise LeaderboardDeleteScoresError("Error deleting leaderboard scores") - for score in scores: leaderboard_scores.append( { @@ -1414,6 +1539,7 @@ def add_scores( "address": normalizer_fn(score.address), "score": score.score, "points_data": score.points_data, + "leaderboard_version_number": version_number, } ) @@ -1675,3 +1801,137 @@ def check_leaderboard_resource_permissions( return True return False + + +def get_leaderboard_version( + db_session: Session, leaderboard_id: uuid.UUID, version_number: int +) -> LeaderboardVersion: + """ + Get the leaderboard version by id + """ + return ( + db_session.query(LeaderboardVersion) + .filter(LeaderboardVersion.leaderboard_id == leaderboard_id) + .filter(LeaderboardVersion.version_number == version_number) + .one() + ) + + +def create_leaderboard_version( + db_session: Session, + leaderboard_id: uuid.UUID, + version_number: Optional[int] = None, + publish: bool = False, +) -> LeaderboardVersion: + """ + Create a leaderboard version + """ + + if version_number is None: + latest_version_result = ( + db_session.query(func.max(LeaderboardVersion.version_number)) + .filter(LeaderboardVersion.leaderboard_id == leaderboard_id) + .one() + ) + + latest_version = latest_version_result[0] + + if latest_version is None: + version_number = 0 + else: + version_number = latest_version + 1 + + leaderboard_version = LeaderboardVersion( + leaderboard_id=leaderboard_id, + version_number=version_number, + published=publish, + ) + + db_session.add(leaderboard_version) + db_session.commit() + + return leaderboard_version + + +def change_publish_leaderboard_version_status( + db_session: Session, leaderboard_id: uuid.UUID, version_number: int, published: bool +) -> LeaderboardVersion: + """ + Publish a leaderboard version + """ + leaderboard_version = ( + db_session.query(LeaderboardVersion) + .filter(LeaderboardVersion.leaderboard_id == leaderboard_id) + .filter(LeaderboardVersion.version_number == version_number) + .one() + ) + + leaderboard_version.published = published + + db_session.commit() + + return leaderboard_version + + +def get_leaderboard_versions( + db_session: Session, leaderboard_id: uuid.UUID +) -> List[LeaderboardVersion]: + """ + Get all leaderboard versions + """ + return ( + db_session.query(LeaderboardVersion) + .filter(LeaderboardVersion.leaderboard_id == leaderboard_id) + .all() + ) + + +def delete_leaderboard_version( + db_session: Session, leaderboard_id: uuid.UUID, version_number: int +) -> LeaderboardVersion: + """ + Delete a leaderboard version + """ + leaderboard_version = ( + db_session.query(LeaderboardVersion) + .filter(LeaderboardVersion.leaderboard_id == leaderboard_id) + .filter(LeaderboardVersion.version_number == version_number) + .one() + ) + + db_session.delete(leaderboard_version) + db_session.commit() + + return leaderboard_version + + +def get_leaderboard_version_scores( + db_session: Session, + leaderboard_id: uuid.UUID, + version_number: int, + limit: int, + offset: int, +) -> List[LeaderboardScores]: + """ + Get the leaderboard scores by version number + """ + + query = ( + db_session.query( + LeaderboardScores.id, + LeaderboardScores.address.label("address"), + LeaderboardScores.score.label("score"), + LeaderboardScores.points_data.label("points_data"), + func.rank().over(order_by=LeaderboardScores.score.desc()).label("rank"), + ) + .filter(LeaderboardScores.leaderboard_id == leaderboard_id) + .filter(LeaderboardScores.leaderboard_version_number == version_number) + ) + + if limit: + query = query.limit(limit) + + if offset: + query = query.offset(offset) + + return query diff --git a/engineapi/engineapi/data.py b/engineapi/engineapi/data.py index 8a33d6197..acf1fa6e2 100644 --- a/engineapi/engineapi/data.py +++ b/engineapi/engineapi/data.py @@ -442,3 +442,11 @@ class LeaderboardConfigUpdate(BaseModel): query_name: Optional[str] = None params: Dict[str, int] normalize_addresses: Optional[bool] = None + + +class LeaderboardVersion(BaseModel): + leaderboard_id: UUID + version: int + published: bool + created_at: datetime + updated_at: datetime diff --git a/engineapi/engineapi/routes/leaderboard.py b/engineapi/engineapi/routes/leaderboard.py index c0e08cb87..f9bc4ef12 100644 --- a/engineapi/engineapi/routes/leaderboard.py +++ b/engineapi/engineapi/routes/leaderboard.py @@ -94,6 +94,7 @@ async def leaderboard( limit: int = Query(10), offset: int = Query(0), db_session: Session = Depends(db.yield_db_session), + version: Optional[str] = Query(None, description="Version of the leaderboard."), ) -> List[data.LeaderboardPosition]: """ Returns the leaderboard positions. @@ -112,7 +113,7 @@ async def leaderboard( raise EngineHTTPException(status_code=500, detail="Internal server error") leaderboard_positions = actions.get_leaderboard_positions( - db_session, leaderboard_id, limit, offset + db_session, leaderboard_id, limit, offset, version ) result = [ data.LeaderboardPosition( @@ -604,10 +605,6 @@ async def leaderboard_push_scores( scores: List[data.Score] = Body( ..., description="Scores to put to the leaderboard." ), - overwrite: bool = Query( - False, - description="If enabled, this will delete all current scores and replace them with the new scores provided.", - ), normalize_addresses: bool = Query( True, description="Normalize addresses to checksum." ), @@ -635,13 +632,22 @@ async def leaderboard_push_scores( status_code=403, detail="You don't have access to this leaderboard." ) + try: + new_version = actions.create_leaderboard_version( + db_session=db_session, + leaderboard_id=leaderboard_id, + ) + except Exception as e: + logger.error(f"Error while creating leaderboard version: {e}") + raise EngineHTTPException(status_code=500, detail="Internal server error") + try: leaderboard_points = actions.add_scores( db_session=db_session, leaderboard_id=leaderboard_id, scores=scores, - overwrite=overwrite, normalize_addresses=normalize_addresses, + version_number=new_version.version_number, ) except actions.DuplicateLeaderboardAddressError as e: raise EngineHTTPException( @@ -658,6 +664,17 @@ async def leaderboard_push_scores( logger.error(f"Score update failed with error: {e}") raise EngineHTTPException(status_code=500, detail="Score update failed.") + try: + actions.change_publish_leaderboard_version_status( + db_session=db_session, + leaderboard_id=leaderboard_id, + version_number=new_version.version_number, + published=True, + ) + except Exception as e: + logger.error(f"Error while updating leaderboard version: {e}") + raise EngineHTTPException(status_code=500, detail="Internal server error") + result = [ data.LeaderboardScore( leaderboard_id=score["leaderboard_id"], @@ -881,3 +898,420 @@ async def leaderboard_config_deactivate( raise EngineHTTPException(status_code=500, detail="Internal server error") return True + + +@app.get( + "/{leaderboard_id}/versions", + response_model=List[data.LeaderboardVersion], + tags=["Authorized Endpoints"], +) +async def leaderboard_versions_list( + request: Request, + leaderboard_id: UUID = Path(..., description="Leaderboard ID"), + db_session: Session = Depends(db.yield_db_session), + Authorization: str = AuthHeader, +) -> List[data.LeaderboardVersion]: + """ + Get leaderboard versions list. + """ + token = request.state.token + try: + access = actions.check_leaderboard_resource_permissions( + db_session=db_session, + leaderboard_id=leaderboard_id, + token=token, + ) + except NoResultFound as e: + raise EngineHTTPException( + status_code=404, + detail="Leaderboard version not found.", + ) + + if not access: + raise EngineHTTPException( + status_code=403, detail="You don't have access to this leaderboard version." + ) + + try: + leaderboard_versions = actions.get_leaderboard_versions( + db_session=db_session, + leaderboard_id=leaderboard_id, + ) + except Exception as e: + logger.error(f"Error while getting leaderboard versions list: {e}") + raise EngineHTTPException(status_code=500, detail="Internal server error") + + result = [ + data.LeaderboardVersion( + leaderboard_id=version.leaderboard_id, + version=version.version_number, + published=version.published, + created_at=version.created_at, + updated_at=version.updated_at, + ) + for version in leaderboard_versions + ] + + return result + + +@app.get( + "/{leaderboard_id}/versions/{version}", + response_model=data.LeaderboardVersion, + tags=["Authorized Endpoints"], +) +async def leaderboard_version_handler( + request: Request, + leaderboard_id: UUID = Path(..., description="Leaderboard ID"), + version: int = Path(..., description="Version of the leaderboard."), + db_session: Session = Depends(db.yield_db_session), + Authorization: str = AuthHeader, +) -> data.LeaderboardVersion: + """ + Get leaderboard version. + """ + token = request.state.token + try: + access = actions.check_leaderboard_resource_permissions( + db_session=db_session, + leaderboard_id=leaderboard_id, + token=token, + ) + except NoResultFound as e: + raise EngineHTTPException( + status_code=404, + detail="Leaderboard not found.", + ) + + if not access: + raise EngineHTTPException( + status_code=403, detail="You don't have access to this leaderboard." + ) + + try: + leaderboard_version = actions.get_leaderboard_version( + db_session=db_session, + leaderboard_id=leaderboard_id, + version_number=version, + ) + except NoResultFound as e: + raise EngineHTTPException( + status_code=404, + detail="Leaderboard version not found.", + ) + except Exception as e: + logger.error(f"Error while getting leaderboard version: {e}") + raise EngineHTTPException(status_code=500, detail="Internal server error") + + return data.LeaderboardVersion( + leaderboard_id=leaderboard_version.leaderboard_id, + version=leaderboard_version.version_number, + published=leaderboard_version.published, + created_at=leaderboard_version.created_at, + updated_at=leaderboard_version.updated_at, + ) + + +@app.post( + "/{leaderboard_id}/versions", + response_model=data.LeaderboardVersion, + tags=["Authorized Endpoints"], +) +async def create_leaderboard_version( + request: Request, + leaderboard_id: UUID = Path(..., description="Leaderboard ID"), + version: int = Query(..., description="Version of the leaderboard."), + db_session: Session = Depends(db.yield_db_session), + Authorization: str = AuthHeader, +) -> data.LeaderboardVersion: + """ + Create leaderboard version. + """ + token = request.state.token + try: + access = actions.check_leaderboard_resource_permissions( + db_session=db_session, + leaderboard_id=leaderboard_id, + token=token, + ) + except NoResultFound as e: + raise EngineHTTPException( + status_code=404, + detail="Leaderboard not found.", + ) + + if not access: + raise EngineHTTPException( + status_code=403, detail="You don't have access to this leaderboard." + ) + + try: + leaderboard_version = actions.create_leaderboard_version( + db_session=db_session, + leaderboard_id=leaderboard_id, + version=version, + ) + except BugoutResponseException as e: + raise EngineHTTPException(status_code=e.status_code, detail=e.detail) + except actions.LeaderboardConfigNotFound as e: + raise EngineHTTPException( + status_code=404, + detail="Leaderboard config not found.", + ) + except Exception as e: + logger.error(f"Error while creating leaderboard version: {e}") + raise EngineHTTPException(status_code=500, detail="Internal server error") + + return leaderboard_version + + +@app.put( + "/{leaderboard_id}/versions/{version}", + response_model=data.LeaderboardVersion, + tags=["Authorized Endpoints"], +) +async def update_leaderboard_version_handler( + request: Request, + leaderboard_id: UUID = Path(..., description="Leaderboard ID"), + version: int = Path(..., description="Version of the leaderboard."), + publish: bool = Query( + False, + description="If enabled, this will publish the leaderboard version.", + ), + db_session: Session = Depends(db.yield_db_session), + Authorization: str = AuthHeader, +) -> data.LeaderboardVersion: + """ + Update leaderboard version. + """ + token = request.state.token + try: + access = actions.check_leaderboard_resource_permissions( + db_session=db_session, + leaderboard_id=leaderboard_id, + token=token, + ) + except NoResultFound as e: + raise EngineHTTPException( + status_code=404, + detail="Leaderboard version not found.", + ) + + if not access: + raise EngineHTTPException( + status_code=403, detail="You don't have access to this leaderboard version." + ) + + try: + leaderboard_version = actions.change_publish_leaderboard_version_status( + db_session=db_session, + leaderboard_id=leaderboard_id, + version_number=version, + published=publish, + ) + except Exception as e: + logger.error(f"Error while updating leaderboard version: {e}") + raise EngineHTTPException(status_code=500, detail="Internal server error") + + return data.LeaderboardVersion( + leaderboard_id=leaderboard_version.leaderboard_id, + version=leaderboard_version.version_number, + published=leaderboard_version.published, + created_at=leaderboard_version.created_at, + updated_at=leaderboard_version.updated_at, + ) + + +@app.delete( + "/{leaderboard_id}/versions/{version}", + response_model=data.LeaderboardVersion, + tags=["Authorized Endpoints"], +) +async def delete_leaderboard_version_handler( + request: Request, + leaderboard_id: UUID = Path(..., description="Leaderboard ID"), + version: int = Path(..., description="Version of the leaderboard."), + db_session: Session = Depends(db.yield_db_session), + Authorization: str = AuthHeader, +) -> data.LeaderboardVersion: + """ + Delete leaderboard version. + """ + token = request.state.token + try: + access = actions.check_leaderboard_resource_permissions( + db_session=db_session, leaderboard_id=leaderboard_id, token=token + ) + except NoResultFound as e: + raise EngineHTTPException( + status_code=404, + detail="Leaderboard version not found.", + ) + + if not access: + raise EngineHTTPException( + status_code=403, detail="You don't have access to this leaderboard version." + ) + + try: + leaderboard_version = actions.delete_leaderboard_version( + db_session=db_session, + leaderboard_id=leaderboard_id, + version_number=version, + ) + except NoResultFound as e: + raise EngineHTTPException( + status_code=404, + detail="Leaderboard version not found.", + ) + except Exception as e: + logger.error(f"Error while deleting leaderboard version: {e}") + raise EngineHTTPException(status_code=500, detail="Internal server error") + + return data.LeaderboardVersion( + leaderboard_id=leaderboard_version.leaderboard_id, + version=leaderboard_version.version_number, + published=leaderboard_version.published, + created_at=leaderboard_version.created_at, + updated_at=leaderboard_version.updated_at, + ) + + +@app.get( + "/{leaderboard_id}/versions/{version}/scores", + response_model=List[data.LeaderboardPosition], + tags=["Authorized Endpoints"], +) +async def leaderboard_version_scores_handler( + request: Request, + leaderboard_id: UUID = Path(..., description="Leaderboard ID"), + version: int = Path(..., description="Version of the leaderboard."), + limit: int = Query(10), + offset: int = Query(0), + db_session: Session = Depends(db.yield_db_session), + Authorization: str = AuthHeader, +) -> List[data.LeaderboardPosition]: + """ + Get leaderboard version scores. + """ + token = request.state.token + try: + access = actions.check_leaderboard_resource_permissions( + db_session=db_session, leaderboard_id=leaderboard_id, token=token + ) + except NoResultFound as e: + raise EngineHTTPException( + status_code=404, + detail="Leaderboard version not found.", + ) + + if not access: + raise EngineHTTPException( + status_code=403, detail="You don't have access to this leaderboard version." + ) + + try: + leaderboard_version_scores = actions.get_leaderboard_version_scores( + db_session=db_session, + leaderboard_id=leaderboard_id, + version_number=version, + limit=limit, + offset=offset, + ) + except Exception as e: + logger.error(f"Error while getting leaderboard version scores: {e}") + raise EngineHTTPException(status_code=500, detail="Internal server error") + + result = [ + data.LeaderboardPosition( + address=score.address, + score=score.score, + rank=score.rank, + points_data=score.points_data, + ) + for score in leaderboard_version_scores + ] + + return result + + +@app.put( + "/{leaderboard_id}/versions/{version}/scores", + response_model=List[data.LeaderboardScore], + tags=["Authorized Endpoints"], +) +async def leaderboard_version_push_scores_handler( + request: Request, + leaderboard_id: UUID = Path(..., description="Leaderboard ID"), + version: int = Path(..., description="Version of the leaderboard."), + scores: List[data.Score] = Body( + ..., description="Scores to put to the leaderboard version." + ), + normalize_addresses: bool = Query( + True, description="Normalize addresses to checksum." + ), + db_session: Session = Depends(db.yield_db_session), + Authorization: str = AuthHeader, +) -> List[data.LeaderboardScore]: + """ + Put the leaderboard version to the database. + """ + token = request.state.token + try: + access = actions.check_leaderboard_resource_permissions( + db_session=db_session, leaderboard_id=leaderboard_id, token=token + ) + except NoResultFound as e: + raise EngineHTTPException( + status_code=404, + detail="Leaderboard version not found.", + ) + + if not access: + raise EngineHTTPException( + status_code=403, detail="You don't have access to this leaderboard version." + ) + + try: + leaderboard_version = actions.get_leaderboard_version( + db_session=db_session, + leaderboard_id=leaderboard_id, + version_number=version, + ) + except NoResultFound as e: + raise EngineHTTPException( + status_code=404, + detail="Leaderboard version not found.", + ) + except Exception as e: + logger.error(f"Error while getting leaderboard version: {e}") + raise EngineHTTPException(status_code=500, detail="Internal server error") + + try: + leaderboard_points = actions.add_scores( + db_session=db_session, + leaderboard_id=leaderboard_id, + scores=scores, + normalize_addresses=normalize_addresses, + version_number=leaderboard_version.version_number, + ) + except actions.DuplicateLeaderboardAddressError as e: + raise EngineHTTPException( + status_code=409, + detail=f"Duplicates in push to database is disallowed.\n List of duplicates:{e.duplicates}.\n Please handle duplicates manualy.", + ) + except Exception as e: + logger.error(f"Score update failed with error: {e}") + raise EngineHTTPException(status_code=500, detail="Score update failed.") + + result = [ + data.LeaderboardScore( + leaderboard_id=score["leaderboard_id"], + address=score["address"], + score=score["score"], + points_data=score["points_data"], + ) + for score in leaderboard_points + ] + + return result From 6a250fe5180e45da568c03ae4f62a503b542d465 Mon Sep 17 00:00:00 2001 From: Andrey Date: Mon, 20 Nov 2023 11:19:59 +0200 Subject: [PATCH 10/40] Deleting old versions on versionless add_scores endpoint. --- engineapi/engineapi/actions.py | 22 ++++++++++++++++++++++ engineapi/engineapi/routes/leaderboard.py | 10 ++++++++++ 2 files changed, 32 insertions(+) diff --git a/engineapi/engineapi/actions.py b/engineapi/engineapi/actions.py index 375e14688..c700b1726 100644 --- a/engineapi/engineapi/actions.py +++ b/engineapi/engineapi/actions.py @@ -1935,3 +1935,25 @@ def get_leaderboard_version_scores( query = query.offset(offset) return query + + +def delete_previous_versions( + db_session: Session, + leaderboard_id: uuid.UUID, + threshold_version_number: int, +) -> int: + """ + Delete old leaderboard versions + """ + + versions_to_delete = ( + db_session.query(LeaderboardVersion) + .filter(LeaderboardVersion.leaderboard_id == leaderboard_id) + .filter(LeaderboardVersion.version_number < threshold_version_number) + ) + + num_deleted = versions_to_delete.delete(synchronize_session=False) + + db_session.commit() + + return num_deleted diff --git a/engineapi/engineapi/routes/leaderboard.py b/engineapi/engineapi/routes/leaderboard.py index f9bc4ef12..b3ac25083 100644 --- a/engineapi/engineapi/routes/leaderboard.py +++ b/engineapi/engineapi/routes/leaderboard.py @@ -675,6 +675,16 @@ async def leaderboard_push_scores( logger.error(f"Error while updating leaderboard version: {e}") raise EngineHTTPException(status_code=500, detail="Internal server error") + try: + actions.delete_previous_versions( + db_session=db_session, + leaderboard_id=leaderboard_id, + threshold_version_number=new_version.version_number, + ) + except Exception as e: + logger.error(f"Error while deleting leaderboard versions: {e}") + raise EngineHTTPException(status_code=500, detail="Internal server error") + result = [ data.LeaderboardScore( leaderboard_id=score["leaderboard_id"], From aef32862c231576bbef85084d927bc019d419e55 Mon Sep 17 00:00:00 2001 From: Andrey Date: Mon, 20 Nov 2023 11:26:10 +0200 Subject: [PATCH 11/40] Remove endpoint. duplication from docs. --- engineapi/engineapi/routes/leaderboard.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/engineapi/engineapi/routes/leaderboard.py b/engineapi/engineapi/routes/leaderboard.py index b3ac25083..f0079a48b 100644 --- a/engineapi/engineapi/routes/leaderboard.py +++ b/engineapi/engineapi/routes/leaderboard.py @@ -87,7 +87,12 @@ ) -@app.get("", response_model=List[data.LeaderboardPosition], tags=["Public Endpoints"]) +@app.get( + "", + response_model=List[data.LeaderboardPosition], + tags=["Public Endpoints"], + include_in_schema=False, +) @app.get("/", response_model=List[data.LeaderboardPosition], tags=["Public Endpoints"]) async def leaderboard( leaderboard_id: UUID = Query(..., description="Leaderboard ID"), @@ -129,7 +134,10 @@ async def leaderboard( @app.post( - "", response_model=data.LeaderboardCreatedResponse, tags=["Authorized Endpoints"] + "", + response_model=data.LeaderboardCreatedResponse, + tags=["Authorized Endpoints"], + include_in_schema=False, ) @app.post( "/", response_model=data.LeaderboardCreatedResponse, tags=["Authorized Endpoints"] From 5f90fc4b56dc571dbb757fc674e8ee11ed8f2a38 Mon Sep 17 00:00:00 2001 From: Andrey Date: Mon, 20 Nov 2023 11:34:35 +0200 Subject: [PATCH 12/40] Add publish to request body. --- engineapi/engineapi/data.py | 4 ++++ engineapi/engineapi/routes/leaderboard.py | 8 ++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/engineapi/engineapi/data.py b/engineapi/engineapi/data.py index acf1fa6e2..6b7e3f826 100644 --- a/engineapi/engineapi/data.py +++ b/engineapi/engineapi/data.py @@ -450,3 +450,7 @@ class LeaderboardVersion(BaseModel): published: bool created_at: datetime updated_at: datetime + + +class LeaderboardVersionUpdateRequest(BaseModel): + publish: bool diff --git a/engineapi/engineapi/routes/leaderboard.py b/engineapi/engineapi/routes/leaderboard.py index f0079a48b..01c93d7eb 100644 --- a/engineapi/engineapi/routes/leaderboard.py +++ b/engineapi/engineapi/routes/leaderboard.py @@ -1092,9 +1092,9 @@ async def update_leaderboard_version_handler( request: Request, leaderboard_id: UUID = Path(..., description="Leaderboard ID"), version: int = Path(..., description="Version of the leaderboard."), - publish: bool = Query( - False, - description="If enabled, this will publish the leaderboard version.", + request_body: data.LeaderboardVersionUpdateRequest = Body( + ..., + description="JSON object specifying whether to publish or unpublish version.", ), db_session: Session = Depends(db.yield_db_session), Authorization: str = AuthHeader, @@ -1125,7 +1125,7 @@ async def update_leaderboard_version_handler( db_session=db_session, leaderboard_id=leaderboard_id, version_number=version, - published=publish, + published=request_body.publish, ) except Exception as e: logger.error(f"Error while updating leaderboard version: {e}") From 0eb6c3e2b9547ff1e806c142a419dc1fe6fb6f06 Mon Sep 17 00:00:00 2001 From: Andrey Date: Mon, 20 Nov 2023 12:18:55 +0200 Subject: [PATCH 13/40] Fix unique constrain on leaderboard scores to remove unique address and add unique version number. --- ...53_added_leaderboard_versions_table_and_.py | 18 ++++++++++++++++++ engineapi/engineapi/models.py | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/engineapi/alembic/versions/cc80e886e153_added_leaderboard_versions_table_and_.py b/engineapi/alembic/versions/cc80e886e153_added_leaderboard_versions_table_and_.py index bdc773f6f..515d638c7 100644 --- a/engineapi/alembic/versions/cc80e886e153_added_leaderboard_versions_table_and_.py +++ b/engineapi/alembic/versions/cc80e886e153_added_leaderboard_versions_table_and_.py @@ -60,6 +60,14 @@ def upgrade(): "leaderboard_scores", sa.Column("leaderboard_version_number", sa.DECIMAL(), nullable=True), ) + op.drop_constraint( + "uq_leaderboard_scores_leaderboard_id", "leaderboard_scores", type_="unique" + ) + op.create_unique_constraint( + op.f("uq_leaderboard_scores_leaderboard_id"), + "leaderboard_scores", + ["leaderboard_id", "address", "leaderboard_version_number"], + ) op.drop_constraint( "fk_leaderboard_scores_leaderboard_id_leaderboards", "leaderboard_scores", @@ -111,6 +119,16 @@ def downgrade(): ["id"], ondelete="CASCADE", ) + op.drop_constraint( + op.f("uq_leaderboard_scores_leaderboard_id"), + "leaderboard_scores", + type_="unique", + ) + op.create_unique_constraint( + "uq_leaderboard_scores_leaderboard_id", + "leaderboard_scores", + ["leaderboard_id", "address"], + ) op.drop_column("leaderboard_scores", "leaderboard_version_number") op.drop_index( op.f("ix_leaderboard_versions_created_at"), table_name="leaderboard_versions" diff --git a/engineapi/engineapi/models.py b/engineapi/engineapi/models.py index c60ee0de1..c516cde29 100644 --- a/engineapi/engineapi/models.py +++ b/engineapi/engineapi/models.py @@ -387,7 +387,7 @@ class LeaderboardVersion(Base): # type: ignore class LeaderboardScores(Base): # type: ignore __tablename__ = "leaderboard_scores" __table_args__ = ( - UniqueConstraint("leaderboard_id", "address"), + UniqueConstraint("leaderboard_id", "address", "leaderboard_version_number"), ForeignKeyConstraint( ["leaderboard_id", "leaderboard_version_number"], [ From de2afed55953a5ae3bf128f8fa995df21c635754 Mon Sep 17 00:00:00 2001 From: Andrey Date: Mon, 20 Nov 2023 15:34:47 +0200 Subject: [PATCH 14/40] Fix if in get latest version. Fix join. Add version parameter to all public endpoints. --- engineapi/engineapi/actions.py | 84 ++++++++++++++++++----- engineapi/engineapi/data.py | 2 +- engineapi/engineapi/routes/leaderboard.py | 55 ++++++++++----- 3 files changed, 105 insertions(+), 36 deletions(-) diff --git a/engineapi/engineapi/actions.py b/engineapi/engineapi/actions.py index c700b1726..ef7f79b61 100644 --- a/engineapi/engineapi/actions.py +++ b/engineapi/engineapi/actions.py @@ -11,7 +11,7 @@ import requests # type: ignore from sqlalchemy.dialects.postgresql import insert from sqlalchemy.orm import Session -from sqlalchemy import func, text, or_, Subquery +from sqlalchemy import func, text, or_, and_, Subquery from sqlalchemy.engine import Row from web3 import Web3 from web3.types import ChecksumAddress @@ -970,7 +970,7 @@ def leaderboard_version_filter( version_number: Optional[int] = None, ) -> Union[Subquery, int]: # Subquery to get the latest version number for the given leaderboard - if not version_number: + if version_number is None: latest_version = ( db_session.query(func.max(LeaderboardVersion.version_number)).filter( LeaderboardVersion.leaderboard_id == leaderboard_id, @@ -983,16 +983,38 @@ def leaderboard_version_filter( return latest_version -def get_leaderboard_total_count(db_session: Session, leaderboard_id) -> int: +def get_leaderboard_total_count( + db_session: Session, leaderboard_id, version_number: Optional[int] = None +) -> int: """ - Get the total number of claimants in the leaderboard + Get the total number of position in the leaderboard """ - return ( - db_session.query(LeaderboardScores) - .filter(LeaderboardScores.leaderboard_id == leaderboard_id) - .count() + + latest_version = leaderboard_version_filter( + db_session=db_session, + leaderboard_id=leaderboard_id, + version_number=version_number, ) + total_count = ( + db_session.query(func.count(LeaderboardScores.id)) + .join( + LeaderboardVersion, + and_( + LeaderboardVersion.leaderboard_id == LeaderboardScores.leaderboard_id, + LeaderboardVersion.version_number + == LeaderboardScores.leaderboard_version_number, + ), + ) + .filter( + LeaderboardVersion.published == True, + LeaderboardVersion.version_number == latest_version, + ) + .filter(LeaderboardScores.leaderboard_id == leaderboard_id) + ).scalar() + + return total_count + def get_leaderboard_info( db_session: Session, leaderboard_id: uuid.UUID, version_number: Optional[int] = None @@ -1006,6 +1028,7 @@ def get_leaderboard_info( leaderboard_id=leaderboard_id, version_number=version_number, ) + leaderboard = ( db_session.query( Leaderboard.id, @@ -1021,7 +1044,11 @@ def get_leaderboard_info( ) .join( LeaderboardVersion, - LeaderboardVersion.leaderboard_id == Leaderboard.id, + and_( + LeaderboardVersion.leaderboard_id == LeaderboardScores.leaderboard_id, + LeaderboardVersion.version_number + == LeaderboardScores.leaderboard_version_number, + ), isouter=True, ) .filter( @@ -1030,8 +1057,7 @@ def get_leaderboard_info( ) .filter(Leaderboard.id == leaderboard_id) .group_by(Leaderboard.id, Leaderboard.title, Leaderboard.description) - .one() - ) + ).one() return leaderboard @@ -1146,7 +1172,11 @@ def get_position( ) .join( LeaderboardVersion, - LeaderboardVersion.leaderboard_id == LeaderboardScores.leaderboard_id, + and_( + LeaderboardVersion.leaderboard_id == LeaderboardScores.leaderboard_id, + LeaderboardVersion.version_number + == LeaderboardScores.leaderboard_version_number, + ), ) .filter( LeaderboardVersion.published == True, @@ -1222,7 +1252,11 @@ def get_leaderboard_positions( ) .join( LeaderboardVersion, - LeaderboardVersion.leaderboard_id == LeaderboardScores.leaderboard_id, + and_( + LeaderboardVersion.leaderboard_id == LeaderboardScores.leaderboard_id, + LeaderboardVersion.version_number + == LeaderboardScores.leaderboard_version_number, + ), ) .filter(LeaderboardScores.leaderboard_id == leaderboard_id) .filter(LeaderboardVersion.published == True) @@ -1260,7 +1294,11 @@ def get_qurtiles( ) .join( LeaderboardVersion, - LeaderboardVersion.leaderboard_id == LeaderboardScores.leaderboard_id, + and_( + LeaderboardVersion.leaderboard_id == LeaderboardScores.leaderboard_id, + LeaderboardVersion.version_number + == LeaderboardScores.leaderboard_version_number, + ), ) .filter( LeaderboardVersion.published == True, @@ -1314,7 +1352,11 @@ def get_ranks( ) .join( LeaderboardVersion, - LeaderboardVersion.leaderboard_id == LeaderboardScores.leaderboard_id, + and_( + LeaderboardVersion.leaderboard_id == LeaderboardScores.leaderboard_id, + LeaderboardVersion.version_number + == LeaderboardScores.leaderboard_version_number, + ), ) .filter( LeaderboardVersion.published == True, @@ -1361,7 +1403,11 @@ def get_rank( ) .join( LeaderboardVersion, - LeaderboardVersion.leaderboard_id == LeaderboardScores.leaderboard_id, + and_( + LeaderboardVersion.leaderboard_id == LeaderboardScores.leaderboard_id, + LeaderboardVersion.version_number + == LeaderboardScores.leaderboard_version_number, + ), ) .filter( LeaderboardVersion.published == True, @@ -1546,7 +1592,11 @@ def add_scores( insert_statement = insert(LeaderboardScores).values(leaderboard_scores) result_stmt = insert_statement.on_conflict_do_update( - index_elements=[LeaderboardScores.address, LeaderboardScores.leaderboard_id], + index_elements=[ + LeaderboardScores.address, + LeaderboardScores.leaderboard_id, + LeaderboardScores.leaderboard_version_number, + ], set_=dict( score=insert_statement.excluded.score, points_data=insert_statement.excluded.points_data, diff --git a/engineapi/engineapi/data.py b/engineapi/engineapi/data.py index 6b7e3f826..395d913d6 100644 --- a/engineapi/engineapi/data.py +++ b/engineapi/engineapi/data.py @@ -452,5 +452,5 @@ class LeaderboardVersion(BaseModel): updated_at: datetime -class LeaderboardVersionUpdateRequest(BaseModel): +class LeaderboardVersionRequest(BaseModel): publish: bool diff --git a/engineapi/engineapi/routes/leaderboard.py b/engineapi/engineapi/routes/leaderboard.py index 01c93d7eb..20c6e0ddf 100644 --- a/engineapi/engineapi/routes/leaderboard.py +++ b/engineapi/engineapi/routes/leaderboard.py @@ -355,6 +355,7 @@ async def get_leaderboards( ) async def count_addresses( leaderboard_id: UUID = Query(..., description="Leaderboard ID"), + version: Optional[int] = Query(None, description="Version of the leaderboard."), db_session: Session = Depends(db.yield_db_session), ) -> data.CountAddressesResponse: """ @@ -373,7 +374,7 @@ async def count_addresses( logger.error(f"Error while getting leaderboard: {e}") raise EngineHTTPException(status_code=500, detail="Internal server error") - count = actions.get_leaderboard_total_count(db_session, leaderboard_id) + count = actions.get_leaderboard_total_count(db_session, leaderboard_id, version) return data.CountAddressesResponse(count=count) @@ -384,12 +385,13 @@ async def count_addresses( async def leadeboard_info( leaderboard_id: UUID = Query(..., description="Leaderboard ID"), db_session: Session = Depends(db.yield_db_session), + version: Optional[int] = Query(None, description="Version of the leaderboard."), ) -> data.LeaderboardInfoResponse: """ Returns leaderboard info. """ try: - leaderboard = actions.get_leaderboard_info(db_session, leaderboard_id) + leaderboard = actions.get_leaderboard_info(db_session, leaderboard_id, version) except NoResultFound as e: raise EngineHTTPException( status_code=404, @@ -443,6 +445,7 @@ async def get_scores_changes( async def quartiles( leaderboard_id: UUID = Query(..., description="Leaderboard ID"), db_session: Session = Depends(db.yield_db_session), + version: Optional[int] = Query(None, description="Version of the leaderboard."), ) -> data.QuartilesResponse: """ Returns the quartiles of the leaderboard. @@ -460,7 +463,7 @@ async def quartiles( raise EngineHTTPException(status_code=500, detail="Internal server error") try: - q1, q2, q3 = actions.get_qurtiles(db_session, leaderboard_id) + q1, q2, q3 = actions.get_qurtiles(db_session, leaderboard_id, version) except actions.LeaderboardIsEmpty: raise EngineHTTPException(status_code=204, detail="Leaderboard is empty.") @@ -489,6 +492,7 @@ async def position( normalize_addresses: bool = Query( True, description="Normalize addresses to checksum." ), + version: Optional[int] = Query(None, description="Version of the leaderboard."), db_session: Session = Depends(db.yield_db_session), ) -> List[data.LeaderboardPosition]: """ @@ -512,7 +516,13 @@ async def position( address = Web3.toChecksumAddress(address) positions = actions.get_position( - db_session, leaderboard_id, address, window_size, limit, offset + db_session, + leaderboard_id, + address, + window_size, + limit, + offset, + version, ) results = [ @@ -536,6 +546,7 @@ async def rank( rank: int = Query(1, description="Rank to get."), limit: Optional[int] = Query(None), offset: Optional[int] = Query(None), + version: Optional[int] = Query(None, description="Version of the leaderboard."), db_session: Session = Depends(db.yield_db_session), ) -> List[data.LeaderboardPosition]: """ @@ -555,7 +566,12 @@ async def rank( raise EngineHTTPException(status_code=500, detail="Internal server error") leaderboard_rank = actions.get_rank( - db_session, leaderboard_id, rank, limit=limit, offset=offset + db_session, + leaderboard_id, + rank, + limit=limit, + offset=offset, + version_number=version, ) results = [ data.LeaderboardPosition( @@ -572,6 +588,7 @@ async def rank( @app.get("/ranks", response_model=List[data.RanksResponse], tags=["Public Endpoints"]) async def ranks( leaderboard_id: UUID = Query(..., description="Leaderboard ID"), + version: Optional[int] = Query(None, description="Version of the leaderboard."), db_session: Session = Depends(db.yield_db_session), ) -> List[data.RanksResponse]: """ @@ -590,7 +607,7 @@ async def ranks( logger.error(f"Error while getting leaderboard: {e}") raise EngineHTTPException(status_code=500, detail="Internal server error") - ranks = actions.get_ranks(db_session, leaderboard_id) + ranks = actions.get_ranks(db_session, leaderboard_id, version) results = [ data.RanksResponse( score=rank.score, @@ -1038,8 +1055,11 @@ async def leaderboard_version_handler( async def create_leaderboard_version( request: Request, leaderboard_id: UUID = Path(..., description="Leaderboard ID"), - version: int = Query(..., description="Version of the leaderboard."), db_session: Session = Depends(db.yield_db_session), + request_body: data.LeaderboardVersionRequest = Body( + ..., + description="JSON object specifying whether to publish or unpublish version.", + ), Authorization: str = AuthHeader, ) -> data.LeaderboardVersion: """ @@ -1064,23 +1084,22 @@ async def create_leaderboard_version( ) try: - leaderboard_version = actions.create_leaderboard_version( + new_version = actions.create_leaderboard_version( db_session=db_session, leaderboard_id=leaderboard_id, - version=version, - ) - except BugoutResponseException as e: - raise EngineHTTPException(status_code=e.status_code, detail=e.detail) - except actions.LeaderboardConfigNotFound as e: - raise EngineHTTPException( - status_code=404, - detail="Leaderboard config not found.", + publish=request_body.publish, ) except Exception as e: logger.error(f"Error while creating leaderboard version: {e}") raise EngineHTTPException(status_code=500, detail="Internal server error") - return leaderboard_version + return data.LeaderboardVersion( + leaderboard_id=new_version.leaderboard_id, + version=new_version.version_number, + published=new_version.published, + created_at=new_version.created_at, + updated_at=new_version.updated_at, + ) @app.put( @@ -1092,7 +1111,7 @@ async def update_leaderboard_version_handler( request: Request, leaderboard_id: UUID = Path(..., description="Leaderboard ID"), version: int = Path(..., description="Version of the leaderboard."), - request_body: data.LeaderboardVersionUpdateRequest = Body( + request_body: data.LeaderboardVersionRequest = Body( ..., description="JSON object specifying whether to publish or unpublish version.", ), From 71e2ffbed322a260e162ad9bf9fe56ce638e7bbc Mon Sep 17 00:00:00 2001 From: Neeraj Kashyap Date: Tue, 21 Nov 2023 10:59:17 -0800 Subject: [PATCH 15/40] Modified alembic migration to downgrade properly Previously, the downgrade was hitting a violation of the old unique constraint on leaderboard_scores if multiple versions of scores were stored for the same address. The modified downgrade only retains the score for each address on the latest publich version of each leaderboard. --- ...153_added_leaderboard_versions_table_and_.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/engineapi/alembic/versions/cc80e886e153_added_leaderboard_versions_table_and_.py b/engineapi/alembic/versions/cc80e886e153_added_leaderboard_versions_table_and_.py index 515d638c7..b6dff83e0 100644 --- a/engineapi/alembic/versions/cc80e886e153_added_leaderboard_versions_table_and_.py +++ b/engineapi/alembic/versions/cc80e886e153_added_leaderboard_versions_table_and_.py @@ -105,6 +105,23 @@ def upgrade(): def downgrade(): + op.execute( + """ + WITH latest_version_for_leaderboard AS ( + SELECT leaderboard_id, MAX(version_number) AS latest_version + FROM leaderboard_versions WHERE published = true + GROUP BY leaderboard_id + ) + DELETE FROM leaderboard_scores WHERE + (leaderboard_id, leaderboard_version_number) NOT IN ( + SELECT + leaderboard_id, + latest_version AS leaderboard_version_number + FROM + latest_version_for_leaderboard + ) + """ + ) # ### commands auto generated by Alembic - please adjust! ### op.drop_constraint( op.f("fk_leaderboard_scores_leaderboard_id_leaderboard_versions"), From 9bb8c192836538cf1465d05828714772c495ae99 Mon Sep 17 00:00:00 2001 From: Neeraj Kashyap Date: Tue, 21 Nov 2023 11:00:38 -0800 Subject: [PATCH 16/40] Added the `client` directory It contains a lightweight Python client for the Engine API. --- engineapi/client/__init__.py | 0 engineapi/client/leaderboards.py | 234 +++++++++++++++++++++++++++++ engineapi/client/sample-score.json | 10 ++ 3 files changed, 244 insertions(+) create mode 100644 engineapi/client/__init__.py create mode 100644 engineapi/client/leaderboards.py create mode 100644 engineapi/client/sample-score.json diff --git a/engineapi/client/__init__.py b/engineapi/client/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/engineapi/client/leaderboards.py b/engineapi/client/leaderboards.py new file mode 100644 index 000000000..59e439ed8 --- /dev/null +++ b/engineapi/client/leaderboards.py @@ -0,0 +1,234 @@ +import argparse +import json +import os +import sys +from typing import Optional +import uuid + +import requests + +LEADERBOARD_API_URL = os.environ.get( + "LEADERBOARD_API_URL", "http://localhost:7191/leaderboard/" +) + + +def moonstream_access_token(value: Optional[str]) -> uuid.UUID: + if value is None: + value = os.environ.get("MOONSTREAM_ACCESS_TOKEN") + + if value is None: + raise ValueError( + "Moonstream access token is required: either via -A/--authorization, or via the MOONSTREAM_ACCESS_TOKEN environment variable" + ) + + try: + value_uuid = uuid.UUID(value) + except Exception: + raise ValueError("Moonstream access token must be a valid UUID") + + return value_uuid + + +def requires_authorization(parser: argparse.ArgumentParser) -> None: + parser.add_argument( + "-A", + "--authorization", + type=moonstream_access_token, + required=False, + default=os.environ.get("MOONSTREAM_ACCESS_TOKEN"), + help="Moonstream API access token (if not provided, must be specified using the MOONSTREAM_ACCESS_TOKEN environment variable)", + ) + + +def handle_get(args: argparse.Namespace) -> None: + url = LEADERBOARD_API_URL + params = { + "leaderboard_id": str(args.id), + "limit": str(args.limit), + "offset": str(args.offset), + } + if args.version is not None: + params["version"] = str(args.version) + + response = requests.get(url, params=params) + response.raise_for_status() + + print(json.dumps(response.json())) + + +def handle_create(args: argparse.Namespace) -> None: + url = LEADERBOARD_API_URL + + headers = { + "Content-Type": "application/json", + "Authorization": f"Bearer {str(args.authorization)}", + } + + body = { + "title": args.title, + "description": args.description, + } + + response = requests.post(url, headers=headers, json=body) + response.raise_for_status() + print(json.dumps(response.json())) + + +def handle_versions(args: argparse.Namespace) -> None: + url = f"{LEADERBOARD_API_URL}{args.id}/versions" + + headers = { + "Authorization": f"Bearer {str(args.authorization)}", + } + + response = requests.get(url, headers=headers) + response.raise_for_status() + print(json.dumps(response.json())) + + +def handle_create_version(args: argparse.Namespace) -> None: + url = f"{LEADERBOARD_API_URL}{args.id}/versions" + + headers = { + "Authorization": f"Bearer {str(args.authorization)}", + "Content-Type": "application/json", + } + + body = { + "publish": args.publish, + } + + response = requests.post(url, headers=headers, json=body) + response.raise_for_status() + print(json.dumps(response.json())) + + +def handle_publish(args: argparse.Namespace) -> None: + url = f"{LEADERBOARD_API_URL}{args.id}/versions/{args.version}" + + headers = { + "Authorization": f"Bearer {str(args.authorization)}", + "Content-Type": "application/json", + } + + body = { + "publish": args.publish, + } + + response = requests.put(url, headers=headers, json=body) + response.raise_for_status() + print(json.dumps(response.json())) + + +def handle_upload_scores(args: argparse.Namespace) -> None: + url = f"{LEADERBOARD_API_URL}{args.id}/scores" + if args.version is not None: + url = f"{LEADERBOARD_API_URL}{args.id}/versions/{args.version}/scores" + + params = { + "overwrite": "true", + "normalize_addresses": "false", + } + + headers = { + "Content-Type": "application/json", + "Authorization": f"Bearer {str(args.authorization)}", + } + + if args.scores is None: + args.scores = sys.stdin + + with args.scores as ifp: + body = json.load(ifp) + + response = requests.put(url, headers=headers, params=params, json=body) + response.raise_for_status() + print(json.dumps(response.json())) + + +def generate_cli() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="HTTP client for Leaderboard API") + parser.set_defaults(func=lambda _: parser.print_help()) + + subparsers = parser.add_subparsers() + + # GET /leaderboard/?leaderboard_id=&limit=&offset=&version= + get_parser = subparsers.add_parser("get") + get_parser.add_argument("-i", "--id", type=uuid.UUID, required=True) + get_parser.add_argument("-l", "--limit", type=int, default=10) + get_parser.add_argument("-o", "--offset", type=int, default=0) + get_parser.add_argument("-v", "--version", type=int, default=None) + get_parser.set_defaults(func=handle_get) + + # POST /leaderboard/ + create_parser = subparsers.add_parser("create") + create_parser.add_argument( + "-t", "--title", type=str, required=True, help="Title for leaderboard" + ) + create_parser.add_argument( + "-d", + "--description", + type=str, + required=False, + default="", + help="Description for leaderboard", + ) + requires_authorization(create_parser) + create_parser.set_defaults(func=handle_create) + + # GET /leaderboard//versions + versions_parser = subparsers.add_parser("versions") + versions_parser.add_argument("-i", "--id", type=uuid.UUID, required=True) + requires_authorization(versions_parser) + versions_parser.set_defaults(func=handle_versions) + + # POST /leaderboard//versions + create_version_parser = subparsers.add_parser("create-version") + create_version_parser.add_argument("-i", "--id", type=uuid.UUID, required=True) + create_version_parser.add_argument( + "--publish", + action="store_true", + help="Set this flag to publish the version immediately upon creation", + ) + requires_authorization(create_version_parser) + create_version_parser.set_defaults(func=handle_create_version) + + # PUT /leaderboard//versions/ + publish_parser = subparsers.add_parser("publish") + publish_parser.add_argument("-i", "--id", type=uuid.UUID, required=True) + publish_parser.add_argument("-v", "--version", type=int, required=True) + publish_parser.add_argument( + "--publish", action="store_true", help="Set to publish, leave to unpublish" + ) + requires_authorization(publish_parser) + publish_parser.set_defaults(func=handle_publish) + + # PUT /leaderboard//scores and PUT /leaderboard//versions//scores + upload_scores_parser = subparsers.add_parser("upload-scores") + upload_scores_parser.add_argument("-i", "--id", type=uuid.UUID, required=True) + upload_scores_parser.add_argument( + "-v", + "--version", + type=int, + required=False, + default=None, + help="Specify a version to upload scores to (if not specified a new version is created)", + ) + upload_scores_parser.add_argument( + "-s", + "--scores", + type=argparse.FileType("r"), + required=False, + default=None, + help="Path to scores file. If not provided, reads from stdin.", + ) + upload_scores_parser.set_defaults(func=handle_upload_scores) + requires_authorization(upload_scores_parser) + + return parser + + +if __name__ == "__main__": + parser = generate_cli() + args = parser.parse_args() + args.func(args) diff --git a/engineapi/client/sample-score.json b/engineapi/client/sample-score.json new file mode 100644 index 000000000..3523a3240 --- /dev/null +++ b/engineapi/client/sample-score.json @@ -0,0 +1,10 @@ +[ + { + "address": "0x0000000000000000000000000000000000000000", + "score": 19, + "points_data": { + "secondary_score_1": 7, + "secondary_score_2": 29 + } + } +] From 354ae7ddc7023c53d41c7e38eda2b4cd26322584 Mon Sep 17 00:00:00 2001 From: Neeraj Kashyap Date: Tue, 21 Nov 2023 11:01:46 -0800 Subject: [PATCH 17/40] Updated README --- engineapi/README.md | 42 ++++-------------------------------------- 1 file changed, 4 insertions(+), 38 deletions(-) diff --git a/engineapi/README.md b/engineapi/README.md index 77e6257c8..5c9c0c374 100644 --- a/engineapi/README.md +++ b/engineapi/README.md @@ -1,43 +1,9 @@ -# lootbox +## `client` -Use lootboxes in your game economy with ready to use contracts +This repository contains a lightweight Python client for the Engine API. -## Deployment - -Deployment with local signer server +To use, for example, with Leaderboard API: ```bash -MOONSTREAM_SIGNING_SERVER_IP=127.0.0.1 ./dev.sh -``` - -## Run frontend - -Do from root directory workspace directory: - -Engine: - -Run dev - -``` -yarn workspace engine run dev -``` - -Build - -``` -yarn workspace engine run build -``` - -Player: - -Run dev - -``` -yarn workspace player run dev -``` - -Build - -``` -yarn workspace player run build +python -m client.leaderboards -h ``` From 65f253145c13f3d786c52cc2de86d87368fdaab2 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Thu, 30 Nov 2023 13:11:20 +0000 Subject: [PATCH 18/40] Removed divide char and added comment restriction --- crawlers/mooncrawl/mooncrawl/stats_worker/queries.py | 2 +- crawlers/mooncrawl/mooncrawl/stats_worker/test_queries.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/crawlers/mooncrawl/mooncrawl/stats_worker/queries.py b/crawlers/mooncrawl/mooncrawl/stats_worker/queries.py index a5d954c20..4c9b94530 100644 --- a/crawlers/mooncrawl/mooncrawl/stats_worker/queries.py +++ b/crawlers/mooncrawl/mooncrawl/stats_worker/queries.py @@ -23,7 +23,7 @@ logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) -QUERY_REGEX = re.compile("[\[\]@#$%^&?;`/]") +QUERY_REGEX = re.compile(r"[\[\]@#$%^&?;`]|/\*|\*/") class QueryNotValid(Exception): diff --git a/crawlers/mooncrawl/mooncrawl/stats_worker/test_queries.py b/crawlers/mooncrawl/mooncrawl/stats_worker/test_queries.py index a71911d7e..3b6473417 100644 --- a/crawlers/mooncrawl/mooncrawl/stats_worker/test_queries.py +++ b/crawlers/mooncrawl/mooncrawl/stats_worker/test_queries.py @@ -60,6 +60,6 @@ def test_query_validation(self): with self.assertRaises(queries.QueryNotValid): queries.query_validation("OR(1=1)#") - + with self.assertRaises(queries.QueryNotValid): - queries.query_validation("/etc/hosts") + queries.query_validation("0/**/or/**/1") From ea04e61f60d3971015085173d72f7079528c96f3 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Thu, 30 Nov 2023 13:15:26 +0000 Subject: [PATCH 19/40] Black check --- crawlers/mooncrawl/mooncrawl/stats_worker/test_queries.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crawlers/mooncrawl/mooncrawl/stats_worker/test_queries.py b/crawlers/mooncrawl/mooncrawl/stats_worker/test_queries.py index 3b6473417..c3c76d349 100644 --- a/crawlers/mooncrawl/mooncrawl/stats_worker/test_queries.py +++ b/crawlers/mooncrawl/mooncrawl/stats_worker/test_queries.py @@ -60,6 +60,6 @@ def test_query_validation(self): with self.assertRaises(queries.QueryNotValid): queries.query_validation("OR(1=1)#") - + with self.assertRaises(queries.QueryNotValid): queries.query_validation("0/**/or/**/1") From ede0093414958512af1cf2afc71342b99c6a8a51 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Tue, 5 Dec 2023 10:57:40 +0000 Subject: [PATCH 20/40] Split monitoring deployment --- crawlers/deploy/deploy-monitoring.bash | 77 ++++++++++++++++++++++++++ crawlers/deploy/deploy.bash | 42 -------------- 2 files changed, 77 insertions(+), 42 deletions(-) create mode 100755 crawlers/deploy/deploy-monitoring.bash diff --git a/crawlers/deploy/deploy-monitoring.bash b/crawlers/deploy/deploy-monitoring.bash new file mode 100755 index 000000000..12600b482 --- /dev/null +++ b/crawlers/deploy/deploy-monitoring.bash @@ -0,0 +1,77 @@ +#!/usr/bin/env bash + +# Deployment script of monitoring services - intended to run on Moonstream crawlers server + +# Colors +C_RESET='\033[0m' +C_RED='\033[1;31m' +C_GREEN='\033[1;32m' +C_YELLOW='\033[1;33m' + +# Logs +PREFIX_INFO="${C_GREEN}[INFO]${C_RESET} [$(date +%d-%m\ %T)]" +PREFIX_WARN="${C_YELLOW}[WARN]${C_RESET} [$(date +%d-%m\ %T)]" +PREFIX_CRIT="${C_RED}[CRIT]${C_RESET} [$(date +%d-%m\ %T)]" + +# Main +AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION:-us-east-1}" +SECRETS_DIR="${SECRETS_DIR:-/home/ubuntu/moonstream-secrets}" +PARAMETERS_ENV_MONITORING_PATH="${SECRETS_DIR}/monitoring.env" +SCRIPT_DIR="$(realpath $(dirname $0))" + +# Service files +MONITORING_CRAWLERS_SERVICE_FILE="monitoring-crawlers.service" + +set -eu + +echo +echo +echo -e "${PREFIX_INFO} Install checkenv" +HOME=/home/ubuntu /usr/local/go/bin/go install github.com/bugout-dev/checkenv@latest + +echo +echo +echo -e "${PREFIX_INFO} Copy monitoring binary from AWS S3" +aws s3 cp s3://bugout-binaries/prod/monitoring/monitoring "/home/ubuntu/monitoring" +chmod +x "/home/ubuntu/monitoring" +chown ubuntu:ubuntu "/home/ubuntu/monitoring" + +echo +echo +echo -e "${PREFIX_INFO} Retrieving monitoring deployment parameters" +AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION}" /home/ubuntu/go/bin/checkenv show aws_ssm+crawlers:true,monitoring:true > "${PARAMETERS_ENV_MONITORING_PATH}" +chmod 0640 "${PARAMETERS_ENV_MONITORING_PATH}" + +echo +echo +echo -e "${PREFIX_INFO} Add instance local IP to monitoring parameters" +echo "AWS_LOCAL_IPV4=$(ec2metadata --local-ipv4)" >> "${PARAMETERS_ENV_MONITORING_PATH}" + +echo +echo +echo -e "${PREFIX_INFO} Add AWS default region to monitring parameters" +echo "AWS_DEFAULT_REGION=${AWS_DEFAULT_REGION}" >> "${PARAMETERS_ENV_MONITORING_PATH}" + +echo +echo +echo -e "${PREFIX_INFO} Prepare monitoring configuration" +if [ ! -d "/home/ubuntu/.monitoring" ]; then + mkdir -p /home/ubuntu/.monitoring + echo -e "${PREFIX_WARN} Created monitoring configuration directory" +fi +cp "${SCRIPT_DIR}/monitoring-crawlers-config.json" /home/ubuntu/.monitoring/monitoring-crawlers-config.json + +echo +echo +if [ ! -d "/home/ubuntu/.config/systemd/user/" ]; then + mkdir -p /home/ubuntu/.config/systemd/user/ + echo -e "${PREFIX_WARN} Created user systemd directory" +fi + +echo +echo +echo -e "${PREFIX_INFO} Replacing existing systemd crawlers monitoring service definition with ${MONITORING_CRAWLERS_SERVICE_FILE}" +chmod 644 "${SCRIPT_DIR}/${MONITORING_CRAWLERS_SERVICE_FILE}" +cp "${SCRIPT_DIR}/${MONITORING_CRAWLERS_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${MONITORING_CRAWLERS_SERVICE_FILE}" +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart "${MONITORING_CRAWLERS_SERVICE_FILE}" diff --git a/crawlers/deploy/deploy.bash b/crawlers/deploy/deploy.bash index d0b38ea0c..99e8b8893 100755 --- a/crawlers/deploy/deploy.bash +++ b/crawlers/deploy/deploy.bash @@ -22,12 +22,10 @@ PYTHON="${PYTHON_ENV_DIR}/bin/python" PIP="${PYTHON_ENV_DIR}/bin/pip" SECRETS_DIR="${SECRETS_DIR:-/home/ubuntu/moonstream-secrets}" PARAMETERS_ENV_PATH="${SECRETS_DIR}/app.env" -PARAMETERS_ENV_MONITORING_PATH="${SECRETS_DIR}/monitoring.env" SCRIPT_DIR="$(realpath $(dirname $0))" # Service files MOONCRAWL_SERVICE_FILE="mooncrawl.service" -MONITORING_CRAWLERS_SERVICE_FILE="monitoring-crawlers.service" LEADERBOARDS_WORKER_SERVICE_FILE="leaderboards-worker.service" LEADERBOARDS_WORKER_TIMER_FILE="leaderboards-worker.timer" @@ -160,38 +158,6 @@ echo echo -e "${PREFIX_INFO} Add instance local IP to parameters" echo "AWS_LOCAL_IPV4=$(ec2metadata --local-ipv4)" >> "${PARAMETERS_ENV_PATH}" -echo -echo -echo -e "${PREFIX_INFO} Copy monitoring binary from AWS S3" -aws s3 cp s3://bugout-binaries/prod/monitoring/monitoring "/home/ubuntu/monitoring" -chmod +x "/home/ubuntu/monitoring" -chown ubuntu:ubuntu "/home/ubuntu/monitoring" - -echo -echo -echo -e "${PREFIX_INFO} Retrieving monitoring deployment parameters" -AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION}" /home/ubuntu/go/bin/checkenv show aws_ssm+crawlers:true,monitoring:true > "${PARAMETERS_ENV_MONITORING_PATH}" -chmod 0640 "${PARAMETERS_ENV_MONITORING_PATH}" - -echo -echo -echo -e "${PREFIX_INFO} Add instance local IP to monitoring parameters" -echo "AWS_LOCAL_IPV4=$(ec2metadata --local-ipv4)" >> "${PARAMETERS_ENV_MONITORING_PATH}" - -echo -echo -echo -e "${PREFIX_INFO} Add AWS default region to monitring parameters" -echo "AWS_DEFAULT_REGION=${AWS_DEFAULT_REGION}" >> "${PARAMETERS_ENV_MONITORING_PATH}" - -echo -echo -echo -e "${PREFIX_INFO} Prepare monitoring configuration" -if [ ! -d "/home/ubuntu/.monitoring" ]; then - mkdir -p /home/ubuntu/.monitoring - echo -e "${PREFIX_WARN} Created monitoring configuration directory" -fi -cp "${SCRIPT_DIR}/monitoring-crawlers-config.json" /home/ubuntu/.monitoring/monitoring-crawlers-config.json - echo echo if [ ! -d "/home/ubuntu/.config/systemd/user/" ]; then @@ -638,11 +604,3 @@ cp "${SCRIPT_DIR}/${LEADERBOARDS_WORKER_SERVICE_FILE}" "/home/ubuntu/.config/sys cp "${SCRIPT_DIR}/${LEADERBOARDS_WORKER_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${LEADERBOARDS_WORKER_TIMER_FILE}" XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${LEADERBOARDS_WORKER_TIMER_FILE}" - -echo -echo -echo -e "${PREFIX_INFO} Replacing existing systemd crawlers monitoring service definition with ${MONITORING_CRAWLERS_SERVICE_FILE}" -chmod 644 "${SCRIPT_DIR}/${MONITORING_CRAWLERS_SERVICE_FILE}" -cp "${SCRIPT_DIR}/${MONITORING_CRAWLERS_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${MONITORING_CRAWLERS_SERVICE_FILE}" -XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload -XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart "${MONITORING_CRAWLERS_SERVICE_FILE}" From 6554b1632dfb72e9c338aa3e15121608337c5807 Mon Sep 17 00:00:00 2001 From: Andrey Date: Sat, 9 Dec 2023 05:24:59 +0200 Subject: [PATCH 21/40] Add WETH and MUTE. --- crawlers/mooncrawl/mooncrawl/settings.py | 1 + .../state_crawler/jobs/zksync-era-jobs.json | 75 +++++++++++++++++++ 2 files changed, 76 insertions(+) create mode 100644 crawlers/mooncrawl/mooncrawl/state_crawler/jobs/zksync-era-jobs.json diff --git a/crawlers/mooncrawl/mooncrawl/settings.py b/crawlers/mooncrawl/mooncrawl/settings.py index 79a3a8a5c..fe14e66c8 100644 --- a/crawlers/mooncrawl/mooncrawl/settings.py +++ b/crawlers/mooncrawl/mooncrawl/settings.py @@ -241,6 +241,7 @@ AvailableBlockchainType.POLYGON: "0xc8E51042792d7405184DfCa245F2d27B94D013b6", AvailableBlockchainType.MUMBAI: "0xe9939e7Ea7D7fb619Ac57f648Da7B1D425832631", AvailableBlockchainType.ETHEREUM: "0x5BA1e12693Dc8F9c48aAD8770482f4739bEeD696", + AvailableBlockchainType.ZKSYNC_ERA: "0xF9cda624FBC7e059355ce98a31693d299FACd963", } diff --git a/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/zksync-era-jobs.json b/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/zksync-era-jobs.json new file mode 100644 index 000000000..6301e9039 --- /dev/null +++ b/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/zksync-era-jobs.json @@ -0,0 +1,75 @@ +[ + { + "constant": true, + "inputs": [], + "name": "getReserves", + "outputs": [ + { + "internalType": "uint", + "name": "", + "type": "uint256" + }, + { + "internalType": "uint", + "name": "", + "type": "uint256" + } + ], + "payable": false, + "address": "0x80115c708E12eDd42E504c1cD52Aea96C547c05c", + "stateMutability": "view", + "type": "function" + }, + { + "constant": true, + "inputs": [], + "name": "getReserves", + "outputs": [ + { + "internalType": "uint112", + "name": "_reserve0", + "type": "uint112" + }, + { + "internalType": "uint112", + "name": "_reserve1", + "type": "uint112" + }, + { + "internalType": "uint32", + "name": "_blockTimestampLast", + "type": "uint32" + } + ], + "payable": false, + "address": "0xb85feb6aF3412d690DFDA280b73EaED73a2315bC", + "stateMutability": "view", + "type": "function" + }, + { + "constant": true, + "inputs": [], + "name": "getReserves", + "outputs": [ + { + "internalType": "uint112", + "name": "_reserve0", + "type": "uint112" + }, + { + "internalType": "uint112", + "name": "_reserve1", + "type": "uint112" + }, + { + "internalType": "uint32", + "name": "_blockTimestampLast", + "type": "uint32" + } + ], + "payable": false, + "address": "0xDFAaB828f5F515E104BaaBa4d8D554DA9096f0e4", + "stateMutability": "view", + "type": "function" + } +] \ No newline at end of file From 21fb0ca0d7c94a29eeef335967739d7ab2424aed Mon Sep 17 00:00:00 2001 From: Andrey Date: Sat, 9 Dec 2023 06:42:36 +0200 Subject: [PATCH 22/40] Add batching leaderboard generator. --- .../mooncrawl/leaderboards_generator/cli.py | 66 ++++++--- .../mooncrawl/leaderboards_generator/utils.py | 126 +++++++++++++++++- crawlers/mooncrawl/mooncrawl/settings.py | 4 + 3 files changed, 174 insertions(+), 22 deletions(-) diff --git a/crawlers/mooncrawl/mooncrawl/leaderboards_generator/cli.py b/crawlers/mooncrawl/mooncrawl/leaderboards_generator/cli.py index 39b8f0f74..7e3821094 100644 --- a/crawlers/mooncrawl/mooncrawl/leaderboards_generator/cli.py +++ b/crawlers/mooncrawl/mooncrawl/leaderboards_generator/cli.py @@ -7,12 +7,14 @@ import requests # type: ignore from bugout.data import BugoutSearchResult -from .utils import get_results_for_moonstream_query +from .utils import get_results_for_moonstream_query, leaderboard_push_batch from ..settings import ( MOONSTREAM_ADMIN_ACCESS_TOKEN, MOONSTREAM_LEADERBOARD_GENERATOR_JOURNAL_ID, MOONSTREAM_API_URL, MOONSTREAM_ENGINE_URL, + MOONSTREAM_LEADERBOARD_GENERATOR_BATCH_SIZE, + MOONSTREAM_LEADERBOARD_GENERATOR_PUSH_TIMEOUT_SECONDS, ) from ..settings import bugout_client as bc @@ -35,10 +37,15 @@ def handle_leaderboards(args: argparse.Namespace) -> None: ### get leaderboard journal + leaderboard_push_batch_size = args.leaderboard_push_batch_size + + leaderboard_push_timeout_seconds = args.leaderboard_push_timeout_seconds + query = "#leaderboard #status:active" - if args.leaderboard_id: # way to run only one leaderboard - query += f" #leaderboard_id:{args.leaderboard_id}" + if args.leaderboard_id: # way to run only one leaderboard without status:active + query = f"#leaderboard #leaderboard_id:{args.leaderboard_id}" + try: leaderboards = bc.search( token=MOONSTREAM_ADMIN_ACCESS_TOKEN, @@ -116,26 +123,33 @@ def handle_leaderboards(args: argparse.Namespace) -> None: "Content-Type": "application/json", } - try: - leaderboard_api_response = requests.put( - leaderboard_push_api_url, - json=query_results["data"], - headers=leaderboard_api_headers, - timeout=10, + if len(query_results["data"]) > leaderboard_push_batch_size: + logger.info( + f"Pushing {len(query_results['data'])} scores to leaderboard {leaderboard_id} in batches of {leaderboard_push_batch_size}" ) - except Exception as e: - logger.error( - f"Could not push results to leaderboard API: {e} for leaderboard {leaderboard_id}" + leaderboard_push_batch( + leaderboard_id, + leaderboard_data, + query_results["data"], + leaderboard_api_headers, + leaderboard_push_batch_size, + timeout=leaderboard_push_timeout_seconds, ) - continue - try: - leaderboard_api_response.raise_for_status() - except requests.exceptions.HTTPError as http_error: - logger.error( - f"Could not push results to leaderboard API: {http_error.response.text} with status code {http_error.response.status_code}" - ) - continue + else: + try: + leaderboard_api_response = requests.put( + leaderboard_push_api_url, + json=query_results["data"], + headers=leaderboard_api_headers, + timeout=leaderboard_push_timeout_seconds, + ) + leaderboard_api_response.raise_for_status() + except requests.exceptions.HTTPError as http_error: + logger.error( + f"Could not push results to leaderboard API: {http_error.response.text} with status code {http_error.response.status_code}" + ) + continue ### get leaderboard from leaderboard API @@ -213,6 +227,18 @@ def main(): required=True, help="Moonstream Access Token to use for Moonstream Query API requests", ) + leaderboard_generator_parser.add_argument( + "--leaderboard-push-batch-size", + type=int, + default=MOONSTREAM_LEADERBOARD_GENERATOR_BATCH_SIZE, + help="Number of scores to push to leaderboard API at once", + ) + leaderboard_generator_parser.add_argument( + "--leaderboard-push-timeout-seconds", + type=int, + default=MOONSTREAM_LEADERBOARD_GENERATOR_PUSH_TIMEOUT_SECONDS, + help="Timeout for leaderboard API requests", + ) leaderboard_generator_parser.set_defaults(func=handle_leaderboards) diff --git a/crawlers/mooncrawl/mooncrawl/leaderboards_generator/utils.py b/crawlers/mooncrawl/mooncrawl/leaderboards_generator/utils.py index fc4d1e012..1e08f6680 100644 --- a/crawlers/mooncrawl/mooncrawl/leaderboards_generator/utils.py +++ b/crawlers/mooncrawl/mooncrawl/leaderboards_generator/utils.py @@ -3,12 +3,17 @@ import logging import os import time -from typing import Any, Dict, Optional +from typing import Any, Dict, Optional, List import requests # type: ignore -from ..settings import MOONSTREAM_API_URL +from ..settings import ( + MOONSTREAM_API_URL, + MOONSTREAM_ENGINE_URL, + MOONSTREAM_LEADERBOARD_GENERATOR_BATCH_SIZE, + MOONSTREAM_LEADERBOARD_GENERATOR_PUSH_TIMEOUT_SECONDS, +) logging.basicConfig() @@ -101,3 +106,120 @@ def get_results_for_moonstream_query( keep_going = num_retries <= max_retries return result + + +def get_data_from_url(url): + response = requests.get(url) + if response.status_code == 200: + return response.json() + else: + raise Exception(f"Failed to get data: HTTP {response.status_code}") + + +def chunk_data(data, chunk_size=100000): + for i in range(0, len(data), chunk_size): + yield data[i : i + chunk_size] + + +def send_data_to_endpoint(chunks, endpoint_url, headers, timeout=10): + for index, chunk in enumerate(chunks): + try: + logger.info(f"Pushing chunk {index} to leaderboard API") + response = requests.put( + endpoint_url, headers=headers, json=chunk, timeout=timeout + ) + + response.raise_for_status() + except requests.exceptions.HTTPError as http_error: + logger.error( + f"Could not push results to leaderboard API: {http_error.response.text} with status code {http_error.response.status_code}" + ) + continue + + +def leaderboard_push_batch( + leaderboard_id: str, + leaderboard_config: Dict[str, Any], + data: List[Dict[str, Any]], + headers: Dict[str, str], + batch_size: int = MOONSTREAM_LEADERBOARD_GENERATOR_BATCH_SIZE, + timeout: int = 10, +) -> None: + """ + Push leaderboard data to the leaderboard API in batches. + """ + + ## first step create leaderboard version + + leaderboard_version_api_url = ( + f"{MOONSTREAM_ENGINE_URL}/leaderboard/{leaderboard_id}/versions" + ) + + json_data = { + "publish": False, + } + + leaderboard_api_response = requests.post( + leaderboard_version_api_url, json=json_data, headers=headers, timeout=5 + ) + + try: + leaderboard_api_response.raise_for_status() + except requests.exceptions.HTTPError as http_error: + logger.error( + f"Could not create leaderboard version: {http_error.response.text} with status code {http_error.response.status_code}" + ) + return + + leaderboard_version_id = leaderboard_api_response.json()["version"] + + ## second step push data to leaderboard version + + leaderboard_version_push_api_url = f"{MOONSTREAM_ENGINE_URL}/leaderboard/{leaderboard_id}/versions/{leaderboard_version_id}/scores?normalize_addresses={leaderboard_config['normalize_addresses']}&overwrite=false" + + chunks = chunk_data(data, chunk_size=batch_size) + + send_data_to_endpoint( + chunks, leaderboard_version_push_api_url, headers, timeout=timeout + ) + + ## third step publish leaderboard version + + leaderboard_version_publish_api_url = f"{MOONSTREAM_ENGINE_URL}/leaderboard/{leaderboard_id}/versions/{leaderboard_version_id}" + + json_data = { + "publish": True, + } + + try: + leaderboard_api_response = requests.put( + leaderboard_version_publish_api_url, + json=json_data, + headers=headers, + timeout=5, + ) + + leaderboard_api_response.raise_for_status() + except requests.exceptions.HTTPError as http_error: + logger.error( + f"Could not publish leaderboard version: {http_error.response.text} with status code {http_error.response.status_code}" + ) + return + + ## delete leaderboard version -1 + + try: + leaderboard_version_delete_api_url = f"{MOONSTREAM_ENGINE_URL}/leaderboard/{leaderboard_id}/versions/{leaderboard_version_id - 1}" + + leaderboard_api_response = requests.delete( + leaderboard_version_delete_api_url, + headers=headers, + timeout=5, + ) + + leaderboard_api_response.raise_for_status() + except requests.exceptions.HTTPError as http_error: + logger.error( + f"Could not delete leaderboard version: {http_error.response.text} with status code {http_error.response.status_code}" + ) + return diff --git a/crawlers/mooncrawl/mooncrawl/settings.py b/crawlers/mooncrawl/mooncrawl/settings.py index 79a3a8a5c..2faafec26 100644 --- a/crawlers/mooncrawl/mooncrawl/settings.py +++ b/crawlers/mooncrawl/mooncrawl/settings.py @@ -321,3 +321,7 @@ raise ValueError( "MOONSTREAM_LEADERBOARD_GENERATOR_JOURNAL_ID environment variable must be set" ) + + +MOONSTREAM_LEADERBOARD_GENERATOR_BATCH_SIZE = 20000 +MOONSTREAM_LEADERBOARD_GENERATOR_PUSH_TIMEOUT_SECONDS = 60 From 5a72358e994c8f60b7ce230724ad5ab5cb503e0b Mon Sep 17 00:00:00 2001 From: Andrey Date: Sat, 9 Dec 2023 10:41:51 +0200 Subject: [PATCH 23/40] Add deploy files. --- crawlers/deploy/deploy.bash | 21 +++++++++++++++++++ .../deploy/zksync-era-state-clean.service | 11 ++++++++++ crawlers/deploy/zksync-era-state-clean.timer | 9 ++++++++ crawlers/deploy/zksync-era-state.service | 11 ++++++++++ crawlers/deploy/zksync-era-state.timer | 9 ++++++++ 5 files changed, 61 insertions(+) create mode 100644 crawlers/deploy/zksync-era-state-clean.service create mode 100644 crawlers/deploy/zksync-era-state-clean.timer create mode 100644 crawlers/deploy/zksync-era-state.service create mode 100644 crawlers/deploy/zksync-era-state.timer diff --git a/crawlers/deploy/deploy.bash b/crawlers/deploy/deploy.bash index 99e8b8893..269a7cc8a 100755 --- a/crawlers/deploy/deploy.bash +++ b/crawlers/deploy/deploy.bash @@ -115,6 +115,10 @@ ZKSYNC_ERA_HISTORICAL_CRAWL_TRANSACTIONS_SERVICE_FILE="zksync-era-historical-cra ZKSYNC_ERA_HISTORICAL_CRAWL_TRANSACTIONS_TIMER_FILE="zksync-era-historical-crawl-transactions.timer" ZKSYNC_ERA_HISTORICAL_CRAWL_EVENTS_SERVICE_FILE="zksync-era-historical-crawl-events.service" ZKSYNC_ERA_HISTORICAL_CRAWL_EVENTS_TIMER_FILE="zksync-era-historical-crawl-events.timer" +ZKSYNC_ERA_STATE_SERVICE_FILE="zksync-era-state.service" +ZKSYNC_ERA_STATE_TIMER_FILE="zksync-era-state.timer" +ZKSYNC_ERA_STATE_CLEAN_SERVICE_FILE="zksync-era-state-clean.service" +ZKSYNC_ERA_STATE_CLEAN_TIMER_FILE="zksync-era-state-clean.timer" # ZkSync Era testnet ZKSYNC_ERA_TESTNET_SYNCHRONIZE_SERVICE="zksync-era-testnet-synchronize.service" @@ -550,6 +554,23 @@ cp "${SCRIPT_DIR}/${ZKSYNC_ERA_HISTORICAL_CRAWL_EVENTS_TIMER_FILE}" "/home/ubunt XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ZKSYNC_ERA_HISTORICAL_CRAWL_EVENTS_TIMER_FILE}" +echo +echo +echo -e "${PREFIX_INFO} Replacing existing ZkSync Era state service and timer with: ${ZKSYNC_ERA_STATE_SERVICE_FILE}, ${ZKSYNC_ERA_STATE_TIMER_FILE}" +chmod 644 "${SCRIPT_DIR}/${ZKSYNC_ERA_STATE_SERVICE_FILE}" "${SCRIPT_DIR}/${ZKSYNC_ERA_STATE_TIMER_FILE}" +cp "${SCRIPT_DIR}/${ZKSYNC_ERA_STATE_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${ZKSYNC_ERA_STATE_SERVICE_FILE}" +cp "${SCRIPT_DIR}/${ZKSYNC_ERA_STATE_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${ZKSYNC_ERA_STATE_TIMER_FILE}" +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ZKSYNC_ERA_STATE_TIMER_FILE}" + +echo +echo +echo -e "${PREFIX_INFO} Replacing existing ZkSync Era state clean service and timer with: ${ZKSYNC_ERA_STATE_CLEAN_SERVICE_FILE}, ${ZKSYNC_ERA_STATE_CLEAN_TIMER_FILE}" +chmod 644 "${SCRIPT_DIR}/${ZKSYNC_ERA_STATE_CLEAN_SERVICE_FILE}" "${SCRIPT_DIR}/${ZKSYNC_ERA_STATE_CLEAN_TIMER_FILE}" +cp "${SCRIPT_DIR}/${ZKSYNC_ERA_STATE_CLEAN_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${ZKSYNC_ERA_STATE_CLEAN_SERVICE_FILE}" +cp "${SCRIPT_DIR}/${ZKSYNC_ERA_STATE_CLEAN_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${ZKSYNC_ERA_STATE_CLEAN_TIMER_FILE}" +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ZKSYNC_ERA_STATE_CLEAN_TIMER_FILE}" # ZkSync Era testnet echo diff --git a/crawlers/deploy/zksync-era-state-clean.service b/crawlers/deploy/zksync-era-state-clean.service new file mode 100644 index 000000000..6b650d647 --- /dev/null +++ b/crawlers/deploy/zksync-era-state-clean.service @@ -0,0 +1,11 @@ +[Unit] +Description=Execute state clean labels crawler +After=network.target + +[Service] +Type=oneshot +WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl +EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" clean-state-labels --blockchain zksync-era -N 10000 +CPUWeight=60 +SyslogIdentifier=zksync-era-state-clean \ No newline at end of file diff --git a/crawlers/deploy/zksync-era-state-clean.timer b/crawlers/deploy/zksync-era-state-clean.timer new file mode 100644 index 000000000..c6e6ae99a --- /dev/null +++ b/crawlers/deploy/zksync-era-state-clean.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Execute Zksync Era state clean labels crawler each 25m + +[Timer] +OnBootSec=50s +OnUnitActiveSec=25m + +[Install] +WantedBy=timers.target diff --git a/crawlers/deploy/zksync-era-state.service b/crawlers/deploy/zksync-era-state.service new file mode 100644 index 000000000..8da611452 --- /dev/null +++ b/crawlers/deploy/zksync-era-state.service @@ -0,0 +1,11 @@ +[Unit] +Description=Execute state crawler +After=network.target + +[Service] +Type=oneshot +WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl +EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl-jobs --moonstream-token "${MOONSTREAM_PUBLIC_QUERIES_DATA_ACCESS_TOKEN}" --blockchain polygon --infura --jobs-file /home/ubuntu/moonstream/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/zksync-era-jobs.json +CPUWeight=60 +SyslogIdentifier=zksync-era-state \ No newline at end of file diff --git a/crawlers/deploy/zksync-era-state.timer b/crawlers/deploy/zksync-era-state.timer new file mode 100644 index 000000000..b67cc1fe9 --- /dev/null +++ b/crawlers/deploy/zksync-era-state.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Execute Zksync Era state crawler each 10m + +[Timer] +OnBootSec=15s +OnUnitActiveSec=10m + +[Install] +WantedBy=timers.target From f04ac73865e8fca3704f8d42f604aa5ef4afaebe Mon Sep 17 00:00:00 2001 From: Andrey Date: Sat, 9 Dec 2023 11:38:50 +0200 Subject: [PATCH 24/40] Fix blockchain in deploy script. --- crawlers/deploy/zksync-era-state.service | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crawlers/deploy/zksync-era-state.service b/crawlers/deploy/zksync-era-state.service index 8da611452..91ab6f4b8 100644 --- a/crawlers/deploy/zksync-era-state.service +++ b/crawlers/deploy/zksync-era-state.service @@ -6,6 +6,6 @@ After=network.target Type=oneshot WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env -ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl-jobs --moonstream-token "${MOONSTREAM_PUBLIC_QUERIES_DATA_ACCESS_TOKEN}" --blockchain polygon --infura --jobs-file /home/ubuntu/moonstream/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/zksync-era-jobs.json +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl-jobs --moonstream-token "${MOONSTREAM_PUBLIC_QUERIES_DATA_ACCESS_TOKEN}" --blockchain zksync-era --infura --jobs-file /home/ubuntu/moonstream/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/zksync-era-jobs.json CPUWeight=60 SyslogIdentifier=zksync-era-state \ No newline at end of file From a8a0b165a226c0295381f0f0057aac4148b975af Mon Sep 17 00:00:00 2001 From: Andrey Date: Sat, 9 Dec 2023 11:47:09 +0200 Subject: [PATCH 25/40] Add fix. --- crawlers/deploy/zksync-era-state-clean.service | 2 +- crawlers/deploy/zksync-era-state.service | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crawlers/deploy/zksync-era-state-clean.service b/crawlers/deploy/zksync-era-state-clean.service index 6b650d647..4cd68f253 100644 --- a/crawlers/deploy/zksync-era-state-clean.service +++ b/crawlers/deploy/zksync-era-state-clean.service @@ -6,6 +6,6 @@ After=network.target Type=oneshot WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env -ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" clean-state-labels --blockchain zksync-era -N 10000 +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" clean-state-labels --blockchain zksync_era -N 10000 CPUWeight=60 SyslogIdentifier=zksync-era-state-clean \ No newline at end of file diff --git a/crawlers/deploy/zksync-era-state.service b/crawlers/deploy/zksync-era-state.service index 91ab6f4b8..573ae5d70 100644 --- a/crawlers/deploy/zksync-era-state.service +++ b/crawlers/deploy/zksync-era-state.service @@ -6,6 +6,6 @@ After=network.target Type=oneshot WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env -ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl-jobs --moonstream-token "${MOONSTREAM_PUBLIC_QUERIES_DATA_ACCESS_TOKEN}" --blockchain zksync-era --infura --jobs-file /home/ubuntu/moonstream/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/zksync-era-jobs.json +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl-jobs --moonstream-token "${MOONSTREAM_PUBLIC_QUERIES_DATA_ACCESS_TOKEN}" --blockchain zksync_era --infura --jobs-file /home/ubuntu/moonstream/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/zksync-era-jobs.json CPUWeight=60 SyslogIdentifier=zksync-era-state \ No newline at end of file From 691bdbf50fe1e60c65953a2368b7892ea1cbae0d Mon Sep 17 00:00:00 2001 From: Andrey Date: Sat, 9 Dec 2023 12:17:32 +0200 Subject: [PATCH 26/40] Remove infura for zksync. --- crawlers/deploy/zksync-era-state.service | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crawlers/deploy/zksync-era-state.service b/crawlers/deploy/zksync-era-state.service index 573ae5d70..e92a6cccd 100644 --- a/crawlers/deploy/zksync-era-state.service +++ b/crawlers/deploy/zksync-era-state.service @@ -6,6 +6,6 @@ After=network.target Type=oneshot WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env -ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl-jobs --moonstream-token "${MOONSTREAM_PUBLIC_QUERIES_DATA_ACCESS_TOKEN}" --blockchain zksync_era --infura --jobs-file /home/ubuntu/moonstream/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/zksync-era-jobs.json +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl-jobs --moonstream-token "${MOONSTREAM_PUBLIC_QUERIES_DATA_ACCESS_TOKEN}" --blockchain zksync_era --jobs-file /home/ubuntu/moonstream/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/zksync-era-jobs.json CPUWeight=60 SyslogIdentifier=zksync-era-state \ No newline at end of file From 8a2e624e097a002a9b0b1b92de50a9032ddcb546 Mon Sep 17 00:00:00 2001 From: Andrey Date: Wed, 13 Dec 2023 21:39:52 +0200 Subject: [PATCH 27/40] Add changes. --- .../mooncrawl/leaderboards_generator/utils.py | 15 +++++---------- crawlers/mooncrawl/mooncrawl/settings.py | 2 +- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/crawlers/mooncrawl/mooncrawl/leaderboards_generator/utils.py b/crawlers/mooncrawl/mooncrawl/leaderboards_generator/utils.py index 1e08f6680..981353707 100644 --- a/crawlers/mooncrawl/mooncrawl/leaderboards_generator/utils.py +++ b/crawlers/mooncrawl/mooncrawl/leaderboards_generator/utils.py @@ -116,11 +116,6 @@ def get_data_from_url(url): raise Exception(f"Failed to get data: HTTP {response.status_code}") -def chunk_data(data, chunk_size=100000): - for i in range(0, len(data), chunk_size): - yield data[i : i + chunk_size] - - def send_data_to_endpoint(chunks, endpoint_url, headers, timeout=10): for index, chunk in enumerate(chunks): try: @@ -134,7 +129,7 @@ def send_data_to_endpoint(chunks, endpoint_url, headers, timeout=10): logger.error( f"Could not push results to leaderboard API: {http_error.response.text} with status code {http_error.response.status_code}" ) - continue + raise http_error def leaderboard_push_batch( @@ -160,7 +155,7 @@ def leaderboard_push_batch( } leaderboard_api_response = requests.post( - leaderboard_version_api_url, json=json_data, headers=headers, timeout=5 + leaderboard_version_api_url, json=json_data, headers=headers, timeout=10 ) try: @@ -177,7 +172,7 @@ def leaderboard_push_batch( leaderboard_version_push_api_url = f"{MOONSTREAM_ENGINE_URL}/leaderboard/{leaderboard_id}/versions/{leaderboard_version_id}/scores?normalize_addresses={leaderboard_config['normalize_addresses']}&overwrite=false" - chunks = chunk_data(data, chunk_size=batch_size) + chunks = [data[x : x + batch_size] for x in range(0, len(data), batch_size)] send_data_to_endpoint( chunks, leaderboard_version_push_api_url, headers, timeout=timeout @@ -196,7 +191,7 @@ def leaderboard_push_batch( leaderboard_version_publish_api_url, json=json_data, headers=headers, - timeout=5, + timeout=10, ) leaderboard_api_response.raise_for_status() @@ -214,7 +209,7 @@ def leaderboard_push_batch( leaderboard_api_response = requests.delete( leaderboard_version_delete_api_url, headers=headers, - timeout=5, + timeout=timeout, ) leaderboard_api_response.raise_for_status() diff --git a/crawlers/mooncrawl/mooncrawl/settings.py b/crawlers/mooncrawl/mooncrawl/settings.py index 2faafec26..ff8433736 100644 --- a/crawlers/mooncrawl/mooncrawl/settings.py +++ b/crawlers/mooncrawl/mooncrawl/settings.py @@ -323,5 +323,5 @@ ) -MOONSTREAM_LEADERBOARD_GENERATOR_BATCH_SIZE = 20000 +MOONSTREAM_LEADERBOARD_GENERATOR_BATCH_SIZE = 12000 MOONSTREAM_LEADERBOARD_GENERATOR_PUSH_TIMEOUT_SECONDS = 60 From 859f2ef2785f695eb5e3ab5a240e9139ddfbb124 Mon Sep 17 00:00:00 2001 From: Andrey Date: Wed, 13 Dec 2023 23:07:58 +0200 Subject: [PATCH 28/40] Add task. --- .../state_crawler/jobs/zksync-era-jobs.json | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/zksync-era-jobs.json b/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/zksync-era-jobs.json index 6301e9039..1247343db 100644 --- a/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/zksync-era-jobs.json +++ b/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/zksync-era-jobs.json @@ -71,5 +71,20 @@ "address": "0xDFAaB828f5F515E104BaaBa4d8D554DA9096f0e4", "stateMutability": "view", "type": "function" + }, + { + "constant": true, + "inputs": [], + "name": "totalSupply", + "outputs": [ + { + "name": "", + "type": "uint256" + } + ], + "payable": false, + "stateMutability": "view", + "type": "function", + "address": "0xDFAaB828f5F515E104BaaBa4d8D554DA9096f0e4" } ] \ No newline at end of file From 73df559c3e9e8400c499f070ff5af64de20e8b3d Mon Sep 17 00:00:00 2001 From: Andrey Date: Thu, 14 Dec 2023 03:17:50 +0200 Subject: [PATCH 29/40] Add internal timeout. --- moonstreamapi/moonstreamapi/routes/queries.py | 3 ++- moonstreamapi/moonstreamapi/settings.py | 8 ++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/moonstreamapi/moonstreamapi/routes/queries.py b/moonstreamapi/moonstreamapi/routes/queries.py index 5a1a01898..26e8b1236 100644 --- a/moonstreamapi/moonstreamapi/routes/queries.py +++ b/moonstreamapi/moonstreamapi/routes/queries.py @@ -32,6 +32,7 @@ MOONSTREAM_APPLICATION_ID, MOONSTREAM_CRAWLERS_SERVER_PORT, MOONSTREAM_CRAWLERS_SERVER_URL, + MOONSTREAM_INTERNAL_REQUEST_TIMEOUT_SECONDS, MOONSTREAM_QUERIES_JOURNAL_ID, MOONSTREAM_QUERY_TEMPLATE_CONTEXT_TYPE, MOONSTREAM_S3_QUERIES_BUCKET, @@ -473,7 +474,7 @@ async def update_query_data_handler( if request_update.blockchain else None, }, - timeout=5, + timeout=MOONSTREAM_INTERNAL_REQUEST_TIMEOUT_SECONDS, ) except Exception as e: logger.error(f"Error interaction with crawlers: {str(e)}") diff --git a/moonstreamapi/moonstreamapi/settings.py b/moonstreamapi/moonstreamapi/settings.py index 95f7491af..1ae46003e 100644 --- a/moonstreamapi/moonstreamapi/settings.py +++ b/moonstreamapi/moonstreamapi/settings.py @@ -259,3 +259,11 @@ "type": "function", } ] + +MOONSTREAM_INTERNAL_REQUEST_TIMEOUT_SECONDS = os.environ.get( + "MOONSTREAM_INTERNAL_REQUEST_TIMEOUT_SECONDS", "10" +) + +MOONSTREAM_INTERNAL_REQUEST_TIMEOUT_SECONDS = int( + MOONSTREAM_INTERNAL_REQUEST_TIMEOUT_SECONDS +) From 8bf0e723f0a14e2a927d4591f86c9899943ebd69 Mon Sep 17 00:00:00 2001 From: Andrey Date: Thu, 14 Dec 2023 03:35:55 +0200 Subject: [PATCH 30/40] Add suggested changes. --- moonstreamapi/moonstreamapi/settings.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/moonstreamapi/moonstreamapi/settings.py b/moonstreamapi/moonstreamapi/settings.py index 1ae46003e..cf1db9a0c 100644 --- a/moonstreamapi/moonstreamapi/settings.py +++ b/moonstreamapi/moonstreamapi/settings.py @@ -260,10 +260,16 @@ } ] -MOONSTREAM_INTERNAL_REQUEST_TIMEOUT_SECONDS = os.environ.get( - "MOONSTREAM_INTERNAL_REQUEST_TIMEOUT_SECONDS", "10" -) - -MOONSTREAM_INTERNAL_REQUEST_TIMEOUT_SECONDS = int( - MOONSTREAM_INTERNAL_REQUEST_TIMEOUT_SECONDS +MOONSTREAM_INTERNAL_REQUEST_TIMEOUT_SECONDS_RAW = os.environ.get( + "MOONSTREAM_INTERNAL_REQUEST_TIMEOUT_SECONDS" ) +MOONSTREAM_INTERNAL_REQUEST_TIMEOUT_SECONDS = 10 +try: + if MOONSTREAM_INTERNAL_REQUEST_TIMEOUT_SECONDS_RAW is not None: + MOONSTREAM_INTERNAL_REQUEST_TIMEOUT_SECONDS = int( + MOONSTREAM_INTERNAL_REQUEST_TIMEOUT_SECONDS_RAW + ) +except: + raise Exception( + f"Could not parse MOONSTREAM_INTERNAL_REQUEST_TIMEOUT_SECONDS as int: {MOONSTREAM_INTERNAL_REQUEST_TIMEOUT_SECONDS_RAW}" + ) From b8b6d0c8c70d5c8c03fa109e1b3ed728ca2bf834 Mon Sep 17 00:00:00 2001 From: Andrey Date: Wed, 20 Dec 2023 01:29:25 +0200 Subject: [PATCH 31/40] Clean deprecated workers. --- crawlers/deploy/deploy.bash | 42 +------------------ .../polygon-cu-reports-tokenonomics.service | 11 ----- .../polygon-cu-reports-tokenonomics.timer | 9 ---- crawlers/deploy/polygon-statistics.service | 11 ----- crawlers/deploy/polygon-statistics.timer | 9 ---- crawlers/deploy/wyrm-statistics.service | 11 ----- crawlers/deploy/wyrm-statistics.timer | 9 ---- crawlers/deploy/xdai-statistics.service | 11 ----- crawlers/deploy/xdai-statistics.timer | 9 ---- 9 files changed, 1 insertion(+), 121 deletions(-) delete mode 100644 crawlers/deploy/polygon-cu-reports-tokenonomics.service delete mode 100644 crawlers/deploy/polygon-cu-reports-tokenonomics.timer delete mode 100644 crawlers/deploy/polygon-statistics.service delete mode 100644 crawlers/deploy/polygon-statistics.timer delete mode 100644 crawlers/deploy/wyrm-statistics.service delete mode 100644 crawlers/deploy/wyrm-statistics.timer delete mode 100644 crawlers/deploy/xdai-statistics.service delete mode 100644 crawlers/deploy/xdai-statistics.timer diff --git a/crawlers/deploy/deploy.bash b/crawlers/deploy/deploy.bash index 269a7cc8a..b481dfd6c 100755 --- a/crawlers/deploy/deploy.bash +++ b/crawlers/deploy/deploy.bash @@ -47,8 +47,6 @@ ETHEREUM_HISTORICAL_CRAWL_EVENTS_TIMER_FILE="ethereum-historical-crawl-events.ti POLYGON_SYNCHRONIZE_SERVICE="polygon-synchronize.service" POLYGON_MISSING_SERVICE_FILE="polygon-missing.service" POLYGON_MISSING_TIMER_FILE="polygon-missing.timer" -POLYGON_STATISTICS_SERVICE_FILE="polygon-statistics.service" -POLYGON_STATISTICS_TIMER_FILE="polygon-statistics.timer" POLYGON_MOONWORM_CRAWLER_SERVICE_FILE="polygon-moonworm-crawler.service" POLYGON_STATE_SERVICE_FILE="polygon-state.service" POLYGON_STATE_TIMER_FILE="polygon-state.timer" @@ -56,8 +54,6 @@ POLYGON_STATE_CLEAN_SERVICE_FILE="polygon-state-clean.service" POLYGON_STATE_CLEAN_TIMER_FILE="polygon-state-clean.timer" POLYGON_METADATA_SERVICE_FILE="polygon-metadata.service" POLYGON_METADATA_TIMER_FILE="polygon-metadata.timer" -POLYGON_CU_REPORTS_TOKENONOMICS_SERVICE_FILE="polygon-cu-reports-tokenonomics.service" -POLYGON_CU_REPORTS_TOKENONOMICS_TIMER_FILE="polygon-cu-reports-tokenonomics.timer" POLYGON_CU_NFT_DASHBOARD_SERVICE_FILE="polygon-cu-nft-dashboard.service" POLYGON_CU_NFT_DASHBOARD_TIMER_FILE="polygon-cu-nft-dashboard.timer" POLYGON_HISTORICAL_CRAWL_TRANSACTIONS_SERVICE_FILE="polygon-historical-crawl-transactions.service" @@ -86,8 +82,7 @@ MUMBAI_HISTORICAL_CRAWL_EVENTS_TIMER_FILE="mumbai-historical-crawl-events.timer" XDAI_SYNCHRONIZE_SERVICE="xdai-synchronize.service" XDAI_MISSING_SERVICE_FILE="xdai-missing.service" XDAI_MISSING_TIMER_FILE="xdai-missing.timer" -XDAI_STATISTICS_SERVICE_FILE="xdai-statistics.service" -XDAI_STATISTICS_TIMER_FILE="xdai-statistics.timer" + XDAI_MOONWORM_CRAWLER_SERVICE_FILE="xdai-moonworm-crawler.service" XDai_HISTORICAL_CRAWL_TRANSACTIONS_SERVICE_FILE="xdai-historical-crawl-transactions.service" XDai_HISTORICAL_CRAWL_TRANSACTIONS_TIMER_FILE="xdai-historical-crawl-transactions.timer" @@ -98,8 +93,6 @@ XDai_HISTORICAL_CRAWL_EVENTS_TIMER_FILE="xdai-historical-crawl-events.timer" WYRM_SYNCHRONIZE_SERVICE="wyrm-synchronize.service" WYRM_MISSING_SERVICE_FILE="wyrm-missing.service" WYRM_MISSING_TIMER_FILE="wyrm-missing.timer" -WYRM_STATISTICS_SERVICE_FILE="wyrm-statistics.service" -WYRM_STATISTICS_TIMER_FILE="wyrm-statistics.timer" WYRM_MOONWORM_CRAWLER_SERVICE_FILE="wyrm-moonworm-crawler.service" WYRM_HISTORICAL_CRAWL_TRANSACTIONS_SERVICE_FILE="wyrm-historical-crawl-transactions.service" WYRM_HISTORICAL_CRAWL_TRANSACTIONS_TIMER_FILE="wyrm-historical-crawl-transactions.timer" @@ -256,14 +249,6 @@ cp "${SCRIPT_DIR}/${POLYGON_MISSING_TIMER_FILE}" "/home/ubuntu/.config/systemd/u XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${POLYGON_MISSING_TIMER_FILE}" -echo -echo -echo -e "${PREFIX_INFO} Replacing existing Polygon statistics dashbord service and timer with: ${POLYGON_STATISTICS_SERVICE_FILE}, ${POLYGON_STATISTICS_TIMER_FILE}" -chmod 644 "${SCRIPT_DIR}/${POLYGON_STATISTICS_SERVICE_FILE}" "${SCRIPT_DIR}/${POLYGON_STATISTICS_TIMER_FILE}" -cp "${SCRIPT_DIR}/${POLYGON_STATISTICS_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${POLYGON_STATISTICS_SERVICE_FILE}" -cp "${SCRIPT_DIR}/${POLYGON_STATISTICS_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${POLYGON_STATISTICS_TIMER_FILE}" -XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload -XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${POLYGON_STATISTICS_TIMER_FILE}" echo echo @@ -300,14 +285,6 @@ cp "${SCRIPT_DIR}/${POLYGON_METADATA_TIMER_FILE}" "/home/ubuntu/.config/systemd/ XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${POLYGON_METADATA_TIMER_FILE}" -echo -echo -echo -e "${PREFIX_INFO} Replacing existing Polygon CU reports tokenonomics service and timer with: ${POLYGON_CU_REPORTS_TOKENONOMICS_SERVICE_FILE}, ${POLYGON_CU_REPORTS_TOKENONOMICS_TIMER_FILE}" -chmod 644 "${SCRIPT_DIR}/${POLYGON_CU_REPORTS_TOKENONOMICS_SERVICE_FILE}" "${SCRIPT_DIR}/${POLYGON_CU_REPORTS_TOKENONOMICS_TIMER_FILE}" -cp "${SCRIPT_DIR}/${POLYGON_CU_REPORTS_TOKENONOMICS_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${POLYGON_CU_REPORTS_TOKENONOMICS_SERVICE_FILE}" -cp "${SCRIPT_DIR}/${POLYGON_CU_REPORTS_TOKENONOMICS_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${POLYGON_CU_REPORTS_TOKENONOMICS_TIMER_FILE}" -XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload -XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${POLYGON_CU_REPORTS_TOKENONOMICS_TIMER_FILE}" echo echo @@ -423,14 +400,6 @@ cp "${SCRIPT_DIR}/${XDAI_MISSING_TIMER_FILE}" "/home/ubuntu/.config/systemd/user XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${XDAI_MISSING_TIMER_FILE}" -echo -echo -echo -e "${PREFIX_INFO} Replacing existing XDai statistics dashbord service and timer with: ${XDAI_STATISTICS_SERVICE_FILE}, ${XDAI_STATISTICS_TIMER_FILE}" -chmod 644 "${SCRIPT_DIR}/${XDAI_STATISTICS_SERVICE_FILE}" "${SCRIPT_DIR}/${XDAI_STATISTICS_TIMER_FILE}" -cp "${SCRIPT_DIR}/${XDAI_STATISTICS_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${XDAI_STATISTICS_SERVICE_FILE}" -cp "${SCRIPT_DIR}/${XDAI_STATISTICS_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${XDAI_STATISTICS_TIMER_FILE}" -XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload -XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${XDAI_STATISTICS_TIMER_FILE}" echo echo @@ -475,15 +444,6 @@ cp "${SCRIPT_DIR}/${WYRM_MISSING_TIMER_FILE}" "/home/ubuntu/.config/systemd/user XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${WYRM_MISSING_TIMER_FILE}" -echo -echo -echo -e "${PREFIX_INFO} Replacing existing Wyrm statistics dashbord service and timer with: ${WYRM_STATISTICS_SERVICE_FILE}, ${WYRM_STATISTICS_TIMER_FILE}" -chmod 644 "${SCRIPT_DIR}/${WYRM_STATISTICS_SERVICE_FILE}" "${SCRIPT_DIR}/${WYRM_STATISTICS_TIMER_FILE}" -cp "${SCRIPT_DIR}/${WYRM_STATISTICS_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${WYRM_STATISTICS_SERVICE_FILE}" -cp "${SCRIPT_DIR}/${WYRM_STATISTICS_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${WYRM_STATISTICS_TIMER_FILE}" -XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload -XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${WYRM_STATISTICS_TIMER_FILE}" - echo echo echo -e "${PREFIX_INFO} Replacing existing Wyrm moonworm crawler service definition with ${WYRM_MOONWORM_CRAWLER_SERVICE_FILE}" diff --git a/crawlers/deploy/polygon-cu-reports-tokenonomics.service b/crawlers/deploy/polygon-cu-reports-tokenonomics.service deleted file mode 100644 index dededd71f..000000000 --- a/crawlers/deploy/polygon-cu-reports-tokenonomics.service +++ /dev/null @@ -1,11 +0,0 @@ -[Unit] -Description=Runs custom crawler for CU tokenonomics -After=network.target - -[Service] -Type=oneshot -WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl -EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env -ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.reports_crawler.cli cu-reports --moonstream-token "${MOONSTREAM_PUBLIC_QUERIES_DATA_ACCESS_TOKEN}" queries run-tokenonomics -CPUWeight=60 -SyslogIdentifier=polygon-cu-reports-tokenonomics \ No newline at end of file diff --git a/crawlers/deploy/polygon-cu-reports-tokenonomics.timer b/crawlers/deploy/polygon-cu-reports-tokenonomics.timer deleted file mode 100644 index 3a56d85de..000000000 --- a/crawlers/deploy/polygon-cu-reports-tokenonomics.timer +++ /dev/null @@ -1,9 +0,0 @@ -[Unit] -Description=Runs custom crawler for CU tokenonomics - -[Timer] -OnBootSec=60s -OnUnitActiveSec=60m - -[Install] -WantedBy=timers.target diff --git a/crawlers/deploy/polygon-statistics.service b/crawlers/deploy/polygon-statistics.service deleted file mode 100644 index ea51aa850..000000000 --- a/crawlers/deploy/polygon-statistics.service +++ /dev/null @@ -1,11 +0,0 @@ -[Unit] -Description=Update Polygon statistics dashboards -After=network.target - -[Service] -Type=oneshot -WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl -EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env -ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.stats_worker.dashboard --access-id "${NB_CONTROLLER_ACCESS_ID}" generate --blockchain polygon -CPUWeight=60 -SyslogIdentifier=polygon-statistics \ No newline at end of file diff --git a/crawlers/deploy/polygon-statistics.timer b/crawlers/deploy/polygon-statistics.timer deleted file mode 100644 index 0cab38da7..000000000 --- a/crawlers/deploy/polygon-statistics.timer +++ /dev/null @@ -1,9 +0,0 @@ -[Unit] -Description=Update Polygon statistics dashboards each 6 hours - -[Timer] -OnBootSec=20s -OnUnitActiveSec=6h - -[Install] -WantedBy=timers.target diff --git a/crawlers/deploy/wyrm-statistics.service b/crawlers/deploy/wyrm-statistics.service deleted file mode 100644 index e8a7add25..000000000 --- a/crawlers/deploy/wyrm-statistics.service +++ /dev/null @@ -1,11 +0,0 @@ -[Unit] -Description=Update Wyrm statistics dashboards -After=network.target - -[Service] -Type=oneshot -WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl -EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env -ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.stats_worker.dashboard --access-id "${NB_CONTROLLER_ACCESS_ID}" generate --blockchain wyrm -CPUWeight=60 -SyslogIdentifier=wyrm-statistics \ No newline at end of file diff --git a/crawlers/deploy/wyrm-statistics.timer b/crawlers/deploy/wyrm-statistics.timer deleted file mode 100644 index 885c5191b..000000000 --- a/crawlers/deploy/wyrm-statistics.timer +++ /dev/null @@ -1,9 +0,0 @@ -[Unit] -Description=Update Wyrm statistics dashboards each 6 hours - -[Timer] -OnBootSec=25s -OnUnitActiveSec=6h - -[Install] -WantedBy=timers.target \ No newline at end of file diff --git a/crawlers/deploy/xdai-statistics.service b/crawlers/deploy/xdai-statistics.service deleted file mode 100644 index 4a1b12cb6..000000000 --- a/crawlers/deploy/xdai-statistics.service +++ /dev/null @@ -1,11 +0,0 @@ -[Unit] -Description=Update XDai statistics dashboards -After=network.target - -[Service] -Type=oneshot -WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl -EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env -ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.stats_worker.dashboard --access-id "${NB_CONTROLLER_ACCESS_ID}" generate --blockchain xdai -CPUWeight=60 -SyslogIdentifier=xdai-statistics \ No newline at end of file diff --git a/crawlers/deploy/xdai-statistics.timer b/crawlers/deploy/xdai-statistics.timer deleted file mode 100644 index 8ef61b518..000000000 --- a/crawlers/deploy/xdai-statistics.timer +++ /dev/null @@ -1,9 +0,0 @@ -[Unit] -Description=Update XDai statistics dashboards each 6 hours - -[Timer] -OnBootSec=25s -OnUnitActiveSec=6h - -[Install] -WantedBy=timers.target \ No newline at end of file From ad9e332d15ca7d16fc1dae1f79d7e7a14b72148a Mon Sep 17 00:00:00 2001 From: Andrey Date: Wed, 20 Dec 2023 04:06:46 +0200 Subject: [PATCH 32/40] Fix leaderboard info endpoint. --- engineapi/engineapi/actions.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/engineapi/engineapi/actions.py b/engineapi/engineapi/actions.py index ef7f79b61..c85e8059b 100644 --- a/engineapi/engineapi/actions.py +++ b/engineapi/engineapi/actions.py @@ -1029,7 +1029,7 @@ def get_leaderboard_info( version_number=version_number, ) - leaderboard = ( + query = ( db_session.query( Leaderboard.id, Leaderboard.title, @@ -1052,12 +1052,19 @@ def get_leaderboard_info( isouter=True, ) .filter( - LeaderboardVersion.published == True, - LeaderboardVersion.version_number == latest_version, + or_( + LeaderboardVersion.published == None, + and_( + LeaderboardVersion.published == True, + LeaderboardVersion.version_number == latest_version, + ), + ) ) .filter(Leaderboard.id == leaderboard_id) .group_by(Leaderboard.id, Leaderboard.title, Leaderboard.description) - ).one() + ) + + leaderboard = query.one() return leaderboard From 919c7c6a79c404da25ca54ff00cd836be32dc7aa Mon Sep 17 00:00:00 2001 From: Andrey Date: Wed, 20 Dec 2023 04:08:34 +0200 Subject: [PATCH 33/40] revert changes. --- engineapi/engineapi/actions.py | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/engineapi/engineapi/actions.py b/engineapi/engineapi/actions.py index c85e8059b..ef7f79b61 100644 --- a/engineapi/engineapi/actions.py +++ b/engineapi/engineapi/actions.py @@ -1029,7 +1029,7 @@ def get_leaderboard_info( version_number=version_number, ) - query = ( + leaderboard = ( db_session.query( Leaderboard.id, Leaderboard.title, @@ -1052,19 +1052,12 @@ def get_leaderboard_info( isouter=True, ) .filter( - or_( - LeaderboardVersion.published == None, - and_( - LeaderboardVersion.published == True, - LeaderboardVersion.version_number == latest_version, - ), - ) + LeaderboardVersion.published == True, + LeaderboardVersion.version_number == latest_version, ) .filter(Leaderboard.id == leaderboard_id) .group_by(Leaderboard.id, Leaderboard.title, Leaderboard.description) - ) - - leaderboard = query.one() + ).one() return leaderboard From 4d90055813abe38c908baa3640670dd26f49326b Mon Sep 17 00:00:00 2001 From: Andrey Date: Wed, 20 Dec 2023 04:51:20 +0200 Subject: [PATCH 34/40] Fix leaderboard info endpoint. --- engineapi/engineapi/actions.py | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/engineapi/engineapi/actions.py b/engineapi/engineapi/actions.py index ef7f79b61..5d8071af2 100644 --- a/engineapi/engineapi/actions.py +++ b/engineapi/engineapi/actions.py @@ -1029,7 +1029,7 @@ def get_leaderboard_info( version_number=version_number, ) - leaderboard = ( + query = ( db_session.query( Leaderboard.id, Leaderboard.title, @@ -1038,26 +1038,33 @@ def get_leaderboard_info( func.max(LeaderboardScores.updated_at).label("last_update"), ) .join( - LeaderboardScores, - LeaderboardScores.leaderboard_id == Leaderboard.id, + LeaderboardVersion, + and_( + LeaderboardVersion.leaderboard_id == Leaderboard.id, + LeaderboardVersion.published == True, + ), isouter=True, ) .join( - LeaderboardVersion, + LeaderboardScores, and_( - LeaderboardVersion.leaderboard_id == LeaderboardScores.leaderboard_id, - LeaderboardVersion.version_number - == LeaderboardScores.leaderboard_version_number, + LeaderboardScores.leaderboard_id == Leaderboard.id, + LeaderboardScores.leaderboard_version_number + == LeaderboardVersion.version_number, ), isouter=True, ) .filter( - LeaderboardVersion.published == True, - LeaderboardVersion.version_number == latest_version, + or_( + LeaderboardVersion.published == None, + LeaderboardVersion.version_number == latest_version, + ) ) .filter(Leaderboard.id == leaderboard_id) .group_by(Leaderboard.id, Leaderboard.title, Leaderboard.description) - ).one() + ) + + leaderboard = query.one() return leaderboard From 7a24e2fb85c3417d11851fb8268eab0b23306d31 Mon Sep 17 00:00:00 2001 From: Andrey Date: Wed, 20 Dec 2023 05:01:57 +0200 Subject: [PATCH 35/40] remove new line. --- crawlers/deploy/deploy.bash | 1 - 1 file changed, 1 deletion(-) diff --git a/crawlers/deploy/deploy.bash b/crawlers/deploy/deploy.bash index b481dfd6c..9f883381d 100755 --- a/crawlers/deploy/deploy.bash +++ b/crawlers/deploy/deploy.bash @@ -82,7 +82,6 @@ MUMBAI_HISTORICAL_CRAWL_EVENTS_TIMER_FILE="mumbai-historical-crawl-events.timer" XDAI_SYNCHRONIZE_SERVICE="xdai-synchronize.service" XDAI_MISSING_SERVICE_FILE="xdai-missing.service" XDAI_MISSING_TIMER_FILE="xdai-missing.timer" - XDAI_MOONWORM_CRAWLER_SERVICE_FILE="xdai-moonworm-crawler.service" XDai_HISTORICAL_CRAWL_TRANSACTIONS_SERVICE_FILE="xdai-historical-crawl-transactions.service" XDai_HISTORICAL_CRAWL_TRANSACTIONS_TIMER_FILE="xdai-historical-crawl-transactions.timer" From cfeae159d4e34616e1e6996a94f8875a0a9b247e Mon Sep 17 00:00:00 2001 From: Andrey Date: Wed, 10 Jan 2024 17:25:30 +0200 Subject: [PATCH 36/40] Add get score endpoint. --- engineapi/engineapi/actions.py | 37 ++++++++++++++++ engineapi/engineapi/routes/leaderboard.py | 52 +++++++++++++++++++++++ 2 files changed, 89 insertions(+) diff --git a/engineapi/engineapi/actions.py b/engineapi/engineapi/actions.py index ef7f79b61..28077dd7e 100644 --- a/engineapi/engineapi/actions.py +++ b/engineapi/engineapi/actions.py @@ -1222,6 +1222,43 @@ def get_position( return query.all() +def get_leaderboard_score( + db_session: Session, + leaderboard_id, + address, + version_number: Optional[int] = None, +) -> Optional[LeaderboardScores]: + """ + Return address score + """ + + latest_version = leaderboard_version_filter( + db_session=db_session, + leaderboard_id=leaderboard_id, + version_number=version_number, + ) + + query = ( + db_session.query(LeaderboardScores) + .join( + LeaderboardVersion, + and_( + LeaderboardVersion.leaderboard_id == LeaderboardScores.leaderboard_id, + LeaderboardVersion.version_number + == LeaderboardScores.leaderboard_version_number, + ), + ) + .filter( + LeaderboardVersion.published == True, + LeaderboardVersion.version_number == latest_version, + ) + .filter(LeaderboardScores.leaderboard_id == leaderboard_id) + .filter(LeaderboardScores.address == address) + ) + + return query.one_or_none() + + def get_leaderboard_positions( db_session: Session, leaderboard_id, diff --git a/engineapi/engineapi/routes/leaderboard.py b/engineapi/engineapi/routes/leaderboard.py index 20c6e0ddf..e18f99576 100644 --- a/engineapi/engineapi/routes/leaderboard.py +++ b/engineapi/engineapi/routes/leaderboard.py @@ -60,6 +60,7 @@ "/leaderboard/": "GET", "/leaderboard/rank": "GET", "/leaderboard/ranks": "GET", + "/leaderboard/scores": "GET", "/scores/changes": "GET", "/leaderboard/docs": "GET", "/leaderboard/openapi.json": "GET", @@ -619,6 +620,57 @@ async def ranks( return results +@app.get( + "/scores", + response_model=data.LeaderboardScore, + tags=["Public Endpoints"], +) +async def leaderboard_score( + address: str = Query(..., description="Address to get position for."), + leaderboard_id: UUID = Query(..., description="Leaderboard ID"), + version: Optional[int] = Query(None, description="Version of the leaderboard."), + normalize_addresses: bool = Query( + True, description="Normalize addresses to checksum." + ), + db_session: Session = Depends(db.yield_db_session), +) -> data.LeaderboardScore: + """ + Returns the leaderboard posotion for the given address. + """ + + ### Check if leaderboard exists + try: + actions.get_leaderboard_by_id(db_session, leaderboard_id) + except NoResultFound as e: + raise EngineHTTPException( + status_code=404, + detail="Leaderboard not found.", + ) + except Exception as e: + logger.error(f"Error while getting leaderboard: {e}") + raise EngineHTTPException(status_code=500, detail="Internal server error") + + if normalize_addresses: + address = Web3.toChecksumAddress(address) + + score = actions.get_leaderboard_score( + db_session, + leaderboard_id, + address, + version, + ) + + if score is None: + raise EngineHTTPException(status_code=204, detail="Score not found.") + + return data.LeaderboardScore( + leaderboard_id=score.leaderboard_id, + address=score.address, + score=score.score, + points_data=score.points_data, + ) + + @app.put( "/{leaderboard_id}/scores", response_model=List[data.LeaderboardScore], From 00a5b21505d5fb9b8ba583fd2a6bab92c495c381 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Thu, 11 Jan 2024 09:41:01 +0000 Subject: [PATCH 37/40] Fixed monitoring tag for deployment --- crawlers/deploy/deploy-monitoring.bash | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crawlers/deploy/deploy-monitoring.bash b/crawlers/deploy/deploy-monitoring.bash index 12600b482..48f4c7734 100755 --- a/crawlers/deploy/deploy-monitoring.bash +++ b/crawlers/deploy/deploy-monitoring.bash @@ -39,7 +39,7 @@ chown ubuntu:ubuntu "/home/ubuntu/monitoring" echo echo echo -e "${PREFIX_INFO} Retrieving monitoring deployment parameters" -AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION}" /home/ubuntu/go/bin/checkenv show aws_ssm+crawlers:true,monitoring:true > "${PARAMETERS_ENV_MONITORING_PATH}" +AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION}" /home/ubuntu/go/bin/checkenv show aws_ssm+service:true,monitoring:true > "${PARAMETERS_ENV_MONITORING_PATH}" chmod 0640 "${PARAMETERS_ENV_MONITORING_PATH}" echo @@ -49,7 +49,7 @@ echo "AWS_LOCAL_IPV4=$(ec2metadata --local-ipv4)" >> "${PARAMETERS_ENV_MONITORIN echo echo -echo -e "${PREFIX_INFO} Add AWS default region to monitring parameters" +echo -e "${PREFIX_INFO} Add AWS default region to monitoring parameters" echo "AWS_DEFAULT_REGION=${AWS_DEFAULT_REGION}" >> "${PARAMETERS_ENV_MONITORING_PATH}" echo From a3745888aef079295738eb55986a520db86a4ef2 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Tue, 23 Jan 2024 06:32:18 +0000 Subject: [PATCH 38/40] Latest block check for starknet goerli --- nodebalancer/cmd/nodebalancer/balancer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nodebalancer/cmd/nodebalancer/balancer.go b/nodebalancer/cmd/nodebalancer/balancer.go index d8dea3c47..701d6dd22 100644 --- a/nodebalancer/cmd/nodebalancer/balancer.go +++ b/nodebalancer/cmd/nodebalancer/balancer.go @@ -198,7 +198,7 @@ func (bpool *BlockchainPool) HealthCheck() { for _, b := range bpool.Blockchains { var timeout time.Duration getLatestBlockReq := `{"jsonrpc":"2.0","method":"eth_getBlockByNumber","params":["latest", false],"id":1}` - if b.Blockchain == "starknet" { + if b.Blockchain == "starknet" || b.Blockchain == "starknet-goerli" { getLatestBlockReq = `{"jsonrpc":"2.0","method":"starknet_getBlockWithTxHashes","params":["latest"],"id":"0"}` timeout = NB_HEALTH_CHECK_CALL_TIMEOUT * 2 } From 03c3a2bc345a4abcc41b9cdf28849a268a27e817 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Tue, 23 Jan 2024 06:45:11 +0000 Subject: [PATCH 39/40] Fix starknet goerli latest check for nb --- nodebalancer/cmd/nodebalancer/balancer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nodebalancer/cmd/nodebalancer/balancer.go b/nodebalancer/cmd/nodebalancer/balancer.go index 701d6dd22..1496b049b 100644 --- a/nodebalancer/cmd/nodebalancer/balancer.go +++ b/nodebalancer/cmd/nodebalancer/balancer.go @@ -241,7 +241,7 @@ func (bpool *BlockchainPool) HealthCheck() { } var blockNumber uint64 - if b.Blockchain == "starknet" { + if b.Blockchain == "starknet" || b.Blockchain == "starknet-goerli" { blockNumber = statusResponse.Result.BlockNumber } else { blockNumberHex := strings.Replace(statusResponse.Result.Number, "0x", "", -1) From b7aa897c50b5907a91fc2d9341381788d1ac6a95 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Tue, 23 Jan 2024 07:01:23 +0000 Subject: [PATCH 40/40] Env for healthcheck interval for nb --- nodebalancer/cmd/nodebalancer/configs.go | 2 +- nodebalancer/cmd/nodebalancer/server.go | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/nodebalancer/cmd/nodebalancer/configs.go b/nodebalancer/cmd/nodebalancer/configs.go index 49ffdd00d..e606dc836 100644 --- a/nodebalancer/cmd/nodebalancer/configs.go +++ b/nodebalancer/cmd/nodebalancer/configs.go @@ -38,7 +38,7 @@ var ( NB_CONNECTION_RETRIES = 2 NB_CONNECTION_RETRIES_INTERVAL = time.Millisecond * 10 - NB_HEALTH_CHECK_INTERVAL = time.Millisecond * 5000 + NB_HEALTH_CHECK_INTERVAL = os.Getenv("NB_HEALTH_CHECK_INTERVAL") NB_HEALTH_CHECK_CALL_TIMEOUT = time.Second * 2 NB_CACHE_CLEANING_INTERVAL = time.Second * 10 diff --git a/nodebalancer/cmd/nodebalancer/server.go b/nodebalancer/cmd/nodebalancer/server.go index 648fa3dac..e1cbe155a 100644 --- a/nodebalancer/cmd/nodebalancer/server.go +++ b/nodebalancer/cmd/nodebalancer/server.go @@ -12,6 +12,7 @@ import ( "net/http/httputil" "net/url" "os" + "strconv" "strings" "time" @@ -28,7 +29,11 @@ var ( // initHealthCheck runs a routine for check status of the nodes every 5 seconds func initHealthCheck(debug bool) { - t := time.NewTicker(NB_HEALTH_CHECK_INTERVAL) + healthCheckInterval, convErr := strconv.Atoi(NB_HEALTH_CHECK_INTERVAL) + if convErr != nil { + healthCheckInterval = 5 + } + t := time.NewTicker(time.Second * time.Duration(healthCheckInterval)) for { select { case <-t.C: