Merge branch 'main' into add-raw-transactions

crawlers/deploy/arbitrum-one-state.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=Execute Arbitrum one state crawler
+After=network.target
+
+[Service]
+Type=oneshot
+WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
+EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli crawl-jobs --moonstream-token "${MOONSTREAM_PUBLIC_QUERIES_DATA_ACCESS_TOKEN}" --blockchain arbitrum_one
+CPUWeight=60
+SyslogIdentifier=arbitrum-one-state

crawlers/deploy/arbitrum-one-state.timer

@@ -0,0 +1,9 @@
+[Unit]
+Description=Execute Arbitrum one state crawler each 5m
+
+[Timer]
+OnBootSec=15s
+OnUnitActiveSec=5m
+
+[Install]
+WantedBy=timers.target

crawlers/deploy/arbitrum-sepolia-state.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=Execute Arbitrum Sepolia state crawler
+After=network.target
+
+[Service]
+Type=oneshot
+WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
+EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli crawl-jobs --moonstream-token "${MOONSTREAM_PUBLIC_QUERIES_DATA_ACCESS_TOKEN}" --blockchain arbitrum_sepolia
+CPUWeight=60
+SyslogIdentifier=arbitrum-sepolia-state

crawlers/deploy/arbitrum-sepolia-state.timer

@@ -0,0 +1,9 @@
+[Unit]
+Description=Execute Arbitrum Sepolia state crawler each 5m
+
+[Timer]
+OnBootSec=15s
+OnUnitActiveSec=5m
+
+[Install]
+WantedBy=timers.target

crawlers/deploy/deploy-state.bash

@@ -32,6 +32,10 @@ ETHEREUM_STATE_CLEAN_TIMER_FILE="ethereum-state-clean.timer"
 ETHEREUM_METADATA_SERVICE_FILE="ethereum-metadata.service"
 ETHEREUM_METADATA_TIMER_FILE="ethereum-metadata.timer"
 
+# Ethereum Sepolia
+SEPOLIA_STATE_SERVICE_FILE="sepolia-state.service"
+SEPOLIA_STATE_TIMER_FILE="sepolia-state.timer"
+
 # Polygon service files
 POLYGON_STATE_SERVICE_FILE="polygon-state.service"
 POLYGON_STATE_TIMER_FILE="polygon-state.timer"
@@ -46,6 +50,13 @@ ZKSYNC_ERA_STATE_TIMER_FILE="zksync-era-state.timer"
 ZKSYNC_ERA_STATE_CLEAN_SERVICE_FILE="zksync-era-state-clean.service"
 ZKSYNC_ERA_STATE_CLEAN_TIMER_FILE="zksync-era-state-clean.timer"
 
+# Arbitrum one
+ARBITRUM_ONE_STATE_SERVICE_FILE="arbitrum-one-state.service"
+ARBITRUM_ONE_STATE_TIMER_FILE="arbitrum-one-state.timer"
+
+# Arbitrum Sepolia
+ARBITRUM_SEPOLIA_STATE_SERVICE_FILE="arbitrum-sepolia-state.service"
+ARBITRUM_SEPOLIA_STATE_TIMER_FILE="arbitrum-sepolia-state.timer"
 
 # Xai
 XAI_STATE_SERVICE_FILE="xai-state.service"
@@ -63,6 +74,19 @@ XAI_SEPOLIA_STATE_CLEAN_TIMER_FILE="xai-sepolia-state-clean.timer"
 XAI_SEPOLIA_METADATA_SERVICE_FILE="xai-sepolia-metadata.service"
 XAI_SEPOLIA_METADATA_TIMER_FILE="xai-sepolia-metadata.timer"
 
+# Game7
+GAME7_METADATA_SERVICE_FILE="game7-metadata.service"
+GAME7_METADATA_TIMER_FILE="game7-metadata.timer"
+GAME7_STATE_SERVICE_FILE="game7-state.service"
+GAME7_STATE_TIMER_FILE="game7-state.timer"
+
+# Game7 testnet
+GAME7_TESTNET_METADATA_SERVICE_FILE="game7-testnet-metadata.service"
+GAME7_TESTNET_METADATA_TIMER_FILE="game7-testnet-metadata.timer"
+GAME7_TESTNET_STATE_SERVICE_FILE="game7-testnet-state.service"
+GAME7_TESTNET_STATE_TIMER_FILE="game7-testnet-state.timer"
+
+
 set -eu
 
 echo
@@ -131,6 +155,21 @@ cp "${SCRIPT_DIR}/${ETHEREUM_METADATA_TIMER_FILE}" "/home/ubuntu/.config/systemd
 XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
 XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ETHEREUM_METADATA_TIMER_FILE}"
 
+
+# Ethereum Sepolia
+
+echo
+echo
+echo -e "${PREFIX_INFO} Replacing existing Ethereum Sepolia state service and timer with: ${SEPOLIA_STATE_SERVICE_FILE}, ${SEPOLIA_STATE_TIMER_FILE}"
+chmod 644 "${SCRIPT_DIR}/${SEPOLIA_STATE_SERVICE_FILE}" "${SCRIPT_DIR}/${SEPOLIA_STATE_TIMER_FILE}"
+cp "${SCRIPT_DIR}/${SEPOLIA_STATE_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${SEPOLIA_STATE_SERVICE_FILE}"
+cp "${SCRIPT_DIR}/${SEPOLIA_STATE_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${SEPOLIA_STATE_TIMER_FILE}"
+XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
+XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${SEPOLIA_STATE_TIMER_FILE}"
+
+
+# Polygon
+
 echo
 echo
 echo -e "${PREFIX_INFO} Replacing existing Polygon state service and timer with: ${POLYGON_STATE_SERVICE_FILE}, ${POLYGON_STATE_TIMER_FILE}"
@@ -176,6 +215,31 @@ cp "${SCRIPT_DIR}/${ZKSYNC_ERA_STATE_CLEAN_TIMER_FILE}" "/home/ubuntu/.config/sy
 XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
 XDG_RUNTIME_DIR="/run/user/1000"  systemctl --user restart --no-block "${ZKSYNC_ERA_STATE_CLEAN_TIMER_FILE}"
 
+# Arbitrum one
+
+echo
+echo
+echo -e "${PREFIX_INFO} Replacing existing Arbitrum one state service and timer with: ${ARBITRUM_ONE_STATE_SERVICE_FILE}, ${ARBITRUM_ONE_STATE_TIMER_FILE}"
+chmod 644 "${SCRIPT_DIR}/${ARBITRUM_ONE_STATE_SERVICE_FILE}" "${SCRIPT_DIR}/${ARBITRUM_ONE_STATE_TIMER_FILE}"
+cp "${SCRIPT_DIR}/${ARBITRUM_ONE_STATE_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${ARBITRUM_ONE_STATE_SERVICE_FILE}"
+cp "${SCRIPT_DIR}/${ARBITRUM_ONE_STATE_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${ARBITRUM_ONE_STATE_TIMER_FILE}"
+XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
+XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ARBITRUM_ONE_STATE_TIMER_FILE}"
+
+
+# Arbitrum Sepolia
+
+echo
+echo
+echo -e "${PREFIX_INFO} Replacing existing Arbitrum Sepolia state service and timer with: ${ARBITRUM_SEPOLIA_STATE_SERVICE_FILE}, ${ARBITRUM_SEPOLIA_STATE_TIMER_FILE}"
+chmod 644 "${SCRIPT_DIR}/${ARBITRUM_SEPOLIA_STATE_SERVICE_FILE}" "${SCRIPT_DIR}/${ARBITRUM_SEPOLIA_STATE_TIMER_FILE}"
+cp "${SCRIPT_DIR}/${ARBITRUM_SEPOLIA_STATE_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${ARBITRUM_SEPOLIA_STATE_SERVICE_FILE}"
+cp "${SCRIPT_DIR}/${ARBITRUM_SEPOLIA_STATE_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${ARBITRUM_SEPOLIA_STATE_TIMER_FILE}"
+XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
+XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ARBITRUM_SEPOLIA_STATE_TIMER_FILE}"
+
+
+# Xai
 
 echo
 echo
@@ -229,4 +293,44 @@ chmod 644 "${SCRIPT_DIR}/${XAI_SEPOLIA_METADATA_SERVICE_FILE}" "${SCRIPT_DIR}/${
 cp "${SCRIPT_DIR}/${XAI_SEPOLIA_METADATA_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${XAI_SEPOLIA_METADATA_SERVICE_FILE}"
 cp "${SCRIPT_DIR}/${XAI_SEPOLIA_METADATA_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${XAI_SEPOLIA_METADATA_TIMER_FILE}"
 XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
-XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${XAI_SEPOLIA_METADATA_TIMER_FILE}"
+XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${XAI_SEPOLIA_METADATA_TIMER_FILE}"
+
+# Game7
+
+echo
+echo
+echo -e "${PREFIX_INFO} Replacing existing Game7 metadata service and timer with: ${GAME7_METADATA_SERVICE_FILE}, ${GAME7_METADATA_TIMER_FILE}"
+chmod 644 "${SCRIPT_DIR}/${GAME7_METADATA_SERVICE_FILE}" "${SCRIPT_DIR}/${GAME7_METADATA_TIMER_FILE}"
+cp "${SCRIPT_DIR}/${GAME7_METADATA_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${GAME7_METADATA_SERVICE_FILE}"
+cp "${SCRIPT_DIR}/${GAME7_METADATA_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${GAME7_METADATA_TIMER_FILE}"
+XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
+XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${GAME7_METADATA_TIMER_FILE}"
+
+echo
+echo
+echo -e "${PREFIX_INFO} Replacing existing Game7 state service and timer with: ${GAME7_STATE_SERVICE_FILE}, ${GAME7_STATE_TIMER_FILE}"
+chmod 644 "${SCRIPT_DIR}/${GAME7_STATE_SERVICE_FILE}" "${SCRIPT_DIR}/${GAME7_STATE_TIMER_FILE}"
+cp "${SCRIPT_DIR}/${GAME7_STATE_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${GAME7_STATE_SERVICE_FILE}"
+cp "${SCRIPT_DIR}/${GAME7_STATE_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${GAME7_STATE_TIMER_FILE}"
+XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
+XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${GAME7_STATE_TIMER_FILE}"
+
+# Game7 testnet
+
+echo
+echo
+echo -e "${PREFIX_INFO} Replacing existing Game7 testnet metadata service and timer with: ${GAME7_TESTNET_METADATA_SERVICE_FILE}, ${GAME7_TESTNET_METADATA_TIMER_FILE}"
+chmod 644 "${SCRIPT_DIR}/${GAME7_TESTNET_METADATA_SERVICE_FILE}" "${SCRIPT_DIR}/${GAME7_TESTNET_METADATA_TIMER_FILE}"
+cp "${SCRIPT_DIR}/${GAME7_TESTNET_METADATA_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${GAME7_TESTNET_METADATA_SERVICE_FILE}"
+cp "${SCRIPT_DIR}/${GAME7_TESTNET_METADATA_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${GAME7_TESTNET_METADATA_TIMER_FILE}"
+XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
+XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${GAME7_TESTNET_METADATA_TIMER_FILE}"
+
+echo
+echo
+echo -e "${PREFIX_INFO} Replacing existing Game7 testnet state service and timer with: ${GAME7_TESTNET_STATE_SERVICE_FILE}, ${GAME7_TESTNET_STATE_TIMER_FILE}"
+chmod 644 "${SCRIPT_DIR}/${GAME7_TESTNET_STATE_SERVICE_FILE}" "${SCRIPT_DIR}/${GAME7_TESTNET_STATE_TIMER_FILE}"
+cp "${SCRIPT_DIR}/${GAME7_TESTNET_STATE_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${GAME7_TESTNET_STATE_SERVICE_FILE}"
+cp "${SCRIPT_DIR}/${GAME7_TESTNET_STATE_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${GAME7_TESTNET_STATE_TIMER_FILE}"
+XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
+XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${GAME7_TESTNET_STATE_TIMER_FILE}"

crawlers/deploy/deploy.bash

@@ -47,6 +47,10 @@ ETHEREUM_HISTORICAL_CRAWL_TRANSACTIONS_TIMER_FILE="ethereum-historical-crawl-tra
 ETHEREUM_HISTORICAL_CRAWL_EVENTS_SERVICE_FILE="ethereum-historical-crawl-events.service"
 ETHEREUM_HISTORICAL_CRAWL_EVENTS_TIMER_FILE="ethereum-historical-crawl-events.timer"
 
+## Sepolia services files
+SEPOLIA_STATE_SERVICE_FILE="sepolia-state.service"
+SEPOLIA_STATE_TIMER_FILE="sepolia-state.timer"
+
 # Polygon service files
 POLYGON_SYNCHRONIZE_SERVICE="polygon-synchronize.service"
 POLYGON_MISSING_SERVICE_FILE="polygon-missing.service"
@@ -124,12 +128,16 @@ ARBITRUM_ONE_HISTORICAL_CRAWL_TRANSACTIONS_SERVICE_FILE="arbitrum-one-historical
 ARBITRUM_ONE_HISTORICAL_CRAWL_TRANSACTIONS_TIMER_FILE="arbitrum-one-historical-crawl-transactions.timer"
 ARBITRUM_ONE_HISTORICAL_CRAWL_EVENTS_SERVICE_FILE="arbitrum-one-historical-crawl-events.service"
 ARBITRUM_ONE_HISTORICAL_CRAWL_EVENTS_TIMER_FILE="arbitrum-one-historical-crawl-events.timer"
+ARBITRUM_ONE_STATE_SERVICE_FILE="arbitrum-one-state.service"
+ARBITRUM_ONE_STATE_TIMER_FILE="arbitrum-one-state.timer"
 
 # Arbitrum Sepolia
 ARBITRUM_SEPOLIA_MISSING_SERVICE_FILE="arbitrum-sepolia-missing.service"
 ARBITRUM_SEPOLIA_MISSING_TIMER_FILE="arbitrum-sepolia-missing.timer"
 ARBITRUM_SEPOLIA_MOONWORM_CRAWLER_SERVICE_FILE="arbitrum-sepolia-moonworm-crawler.service"
 ARBITRUM_SEPOLIA_SYNCHRONIZE_SERVICE="arbitrum-sepolia-synchronize.service"
+ARBITRUM_SEPOLIA_STATE_SERVICE_FILE="arbitrum-sepolia-state.service"
+ARBITRUM_SEPOLIA_STATE_TIMER_FILE="arbitrum-sepolia-state.timer"
 
 # Xai
 XAI_MISSING_SERVICE_FILE="xai-missing.service"
@@ -217,6 +225,18 @@ MANTLE_SEPOLIA_HISTORICAL_CRAWL_EVENTS_TIMER_FILE="mantle-sepolia-historical-cra
 MANTLE_SEPOLIA_HISTORICAL_CRAWL_TRANSACTIONS_SERVICE_FILE="mantle-sepolia-historical-crawl-transactions.service"
 MANTLE_SEPOLIA_HISTORICAL_CRAWL_TRANSACTIONS_TIMER_FILE="mantle-sepolia-historical-crawl-transactions.timer"
 
+# Game7
+GAME7_METADATA_SERVICE_FILE="game7-metadata.service"
+GAME7_METADATA_TIMER_FILE="game7-metadata.timer"
+GAME7_STATE_SERVICE_FILE="game7-state.service"
+GAME7_STATE_TIMER_FILE="game7-state.timer"
+
+# Game7 testnet
+GAME7_TESTNET_METADATA_SERVICE_FILE="game7-testnet-metadata.service"
+GAME7_TESTNET_METADATA_TIMER_FILE="game7-testnet-metadata.timer"
+GAME7_TESTNET_STATE_SERVICE_FILE="game7-testnet-state.service"
+GAME7_TESTNET_STATE_TIMER_FILE="game7-testnet-state.timer"
+
 set -eu
 
 echo
@@ -346,6 +366,20 @@ XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
 XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ETHEREUM_HISTORICAL_CRAWL_EVENTS_TIMER_FILE}"
 
 
+## Sepolia
+
+echo
+echo
+echo -e "${PREFIX_INFO} Replacing existing Sepolia state service and timer with: ${SEPOLIA_STATE_SERVICE_FILE}, ${SEPOLIA_STATE_TIMER_FILE}"
+chmod 644 "${SCRIPT_DIR}/${SEPOLIA_STATE_SERVICE_FILE}" "${SCRIPT_DIR}/${SEPOLIA_STATE_TIMER_FILE}"
+cp "${SCRIPT_DIR}/${SEPOLIA_STATE_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${SEPOLIA_STATE_SERVICE_FILE}"
+cp "${SCRIPT_DIR}/${SEPOLIA_STATE_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${SEPOLIA_STATE_TIMER_FILE}"
+XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
+XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${SEPOLIA_STATE_TIMER_FILE}"
+
+
+## Polygon
+
 echo
 echo
 echo -e "${PREFIX_INFO} Replacing existing Polygon block with transactions syncronizer service definition with ${POLYGON_SYNCHRONIZE_SERVICE}"
@@ -701,6 +735,14 @@ cp "${SCRIPT_DIR}/${ARBITRUM_ONE_HISTORICAL_CRAWL_EVENTS_TIMER_FILE}" "/home/ubu
 XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
 XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ARBITRUM_ONE_HISTORICAL_CRAWL_EVENTS_TIMER_FILE}"
 
+echo
+echo
+echo -e "${PREFIX_INFO} Replacing existing Arbitrum one state service and timer with: ${ARBITRUM_ONE_STATE_SERVICE_FILE}, ${ARBITRUM_ONE_STATE_TIMER_FILE}"
+chmod 644 "${SCRIPT_DIR}/${ARBITRUM_ONE_STATE_SERVICE_FILE}" "${SCRIPT_DIR}/${ARBITRUM_ONE_STATE_TIMER_FILE}"
+cp "${SCRIPT_DIR}/${ARBITRUM_ONE_STATE_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${ARBITRUM_ONE_STATE_SERVICE_FILE}"
+cp "${SCRIPT_DIR}/${ARBITRUM_ONE_STATE_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${ARBITRUM_ONE_STATE_TIMER_FILE}"
+XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
+XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ARBITRUM_ONE_STATE_TIMER_FILE}"
 
 # Arbitrum Sepolia
 echo
@@ -728,6 +770,15 @@ cp "${SCRIPT_DIR}/${ARBITRUM_SEPOLIA_MOONWORM_CRAWLER_SERVICE_FILE}" "/home/ubun
 XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
 XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ARBITRUM_SEPOLIA_MOONWORM_CRAWLER_SERVICE_FILE}"
 
+echo
+echo
+echo -e "${PREFIX_INFO} Replacing existing Arbitrum Sepolia state service and timer with: ${ARBITRUM_SEPOLIA_STATE_SERVICE_FILE}, ${ARBITRUM_SEPOLIA_STATE_TIMER_FILE}"
+chmod 644 "${SCRIPT_DIR}/${ARBITRUM_SEPOLIA_STATE_SERVICE_FILE}" "${SCRIPT_DIR}/${ARBITRUM_SEPOLIA_STATE_TIMER_FILE}"
+cp "${SCRIPT_DIR}/${ARBITRUM_SEPOLIA_STATE_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${ARBITRUM_SEPOLIA_STATE_SERVICE_FILE}"
+cp "${SCRIPT_DIR}/${ARBITRUM_SEPOLIA_STATE_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${ARBITRUM_SEPOLIA_STATE_TIMER_FILE}"
+XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
+XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ARBITRUM_SEPOLIA_STATE_TIMER_FILE}"
+
 # Xai
 echo
 echo
@@ -1109,3 +1160,42 @@ cp "${SCRIPT_DIR}/${MANTLE_SEPOLIA_HISTORICAL_CRAWL_TRANSACTIONS_SERVICE_FILE}"
 cp "${SCRIPT_DIR}/${MANTLE_SEPOLIA_HISTORICAL_CRAWL_TRANSACTIONS_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${MANTLE_SEPOLIA_HISTORICAL_CRAWL_TRANSACTIONS_TIMER_FILE}"
 XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
 XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${MANTLE_SEPOLIA_HISTORICAL_CRAWL_TRANSACTIONS_TIMER_FILE}"
+
+
+# Game7
+echo
+echo
+echo -e "${PREFIX_INFO} Replacing existing Game7 metadata service and timer with: ${GAME7_METADATA_SERVICE_FILE}, ${GAME7_METADATA_TIMER_FILE}"
+chmod 644 "${SCRIPT_DIR}/${GAME7_METADATA_SERVICE_FILE}" "${SCRIPT_DIR}/${GAME7_METADATA_TIMER_FILE}"
+cp "${SCRIPT_DIR}/${GAME7_METADATA_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${GAME7_METADATA_SERVICE_FILE}"
+cp "${SCRIPT_DIR}/${GAME7_METADATA_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${GAME7_METADATA_TIMER_FILE}"
+XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
+XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${GAME7_METADATA_TIMER_FILE}"
+
+echo
+echo
+echo -e "${PREFIX_INFO} Replacing existing Game7 state service and timer with: ${GAME7_STATE_SERVICE_FILE}, ${GAME7_STATE_TIMER_FILE}"
+chmod 644 "${SCRIPT_DIR}/${GAME7_STATE_SERVICE_FILE}" "${SCRIPT_DIR}/${GAME7_STATE_TIMER_FILE}"
+cp "${SCRIPT_DIR}/${GAME7_STATE_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${GAME7_STATE_SERVICE_FILE}"
+cp "${SCRIPT_DIR}/${GAME7_STATE_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${GAME7_STATE_TIMER_FILE}"
+XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
+XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${GAME7_STATE_TIMER_FILE}"
+
+# Game7 testnet
+echo
+echo
+echo -e "${PREFIX_INFO} Replacing existing Game7 testnet metadata service and timer with: ${GAME7_TESTNET_METADATA_SERVICE_FILE}, ${GAME7_TESTNET_METADATA_TIMER_FILE}"
+chmod 644 "${SCRIPT_DIR}/${GAME7_TESTNET_METADATA_SERVICE_FILE}" "${SCRIPT_DIR}/${GAME7_TESTNET_METADATA_TIMER_FILE}"
+cp "${SCRIPT_DIR}/${GAME7_TESTNET_METADATA_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${GAME7_TESTNET_METADATA_SERVICE_FILE}"
+cp "${SCRIPT_DIR}/${GAME7_TESTNET_METADATA_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${GAME7_TESTNET_METADATA_TIMER_FILE}"
+XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
+XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${GAME7_TESTNET_METADATA_TIMER_FILE}"
+
+echo
+echo
+echo -e "${PREFIX_INFO} Replacing existing Game7 testnet state service and timer with: ${GAME7_TESTNET_STATE_SERVICE_FILE}, ${GAME7_TESTNET_STATE_TIMER_FILE}"
+chmod 644 "${SCRIPT_DIR}/${GAME7_TESTNET_STATE_SERVICE_FILE}" "${SCRIPT_DIR}/${GAME7_TESTNET_STATE_TIMER_FILE}"
+cp "${SCRIPT_DIR}/${GAME7_TESTNET_STATE_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${GAME7_TESTNET_STATE_SERVICE_FILE}"
+cp "${SCRIPT_DIR}/${GAME7_TESTNET_STATE_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${GAME7_TESTNET_STATE_TIMER_FILE}"
+XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
+XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${GAME7_TESTNET_STATE_TIMER_FILE}"

crawlers/deploy/ethereum-state.timer

@@ -1,9 +1,9 @@
 [Unit]
-Description=Execute Ethereum state crawler each 10m
+Description=Execute Ethereum state crawler each 5m
 
 [Timer]
 OnBootSec=15s
-OnUnitActiveSec=10m
+OnUnitActiveSec=5m
 
 [Install]
 WantedBy=timers.target

crawlers/deploy/game7-metadata copy.timer

@@ -0,0 +1,9 @@
+[Unit]
+Description=Execute Game7 metadata crawler each 10m
+
+[Timer]
+OnBootSec=20s
+OnUnitActiveSec=60m
+
+[Install]
+WantedBy=timers.target

crawlers/deploy/game7-metadata.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=Execute metadata crawler
+After=network.target
+
+[Service]
+Type=oneshot
+WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
+EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.metadata_crawler.cli crawl --blockchain game7
+CPUWeight=60
+SyslogIdentifier=game7-metadata

crawlers/deploy/game7-metadata.timer

@@ -0,0 +1,9 @@
+[Unit]
+Description=Execute Game7 metadata crawler each 10m
+
+[Timer]
+OnBootSec=20s
+OnUnitActiveSec=60m
+
+[Install]
+WantedBy=timers.target

crawlers/deploy/game7-state.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=Execute Game7 state crawler
+After=network.target
+
+[Service]
+Type=oneshot
+WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
+EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli crawl-jobs --moonstream-token "${MOONSTREAM_PUBLIC_QUERIES_DATA_ACCESS_TOKEN}" --blockchain game7
+CPUWeight=60
+SyslogIdentifier=game7-state

crawlers/deploy/game7-state.timer

@@ -0,0 +1,9 @@
+[Unit]
+Description=Execute Game7 state crawler each 5m
+
+[Timer]
+OnBootSec=15s
+OnUnitActiveSec=5m
+
+[Install]
+WantedBy=timers.target

crawlers/deploy/game7-testnet-metadata.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=Execute metadata crawler
+After=network.target
+
+[Service]
+Type=oneshot
+WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
+EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.metadata_crawler.cli crawl --blockchain game7_testnet
+CPUWeight=60
+SyslogIdentifier=game7-testnet-metadata

crawlers/deploy/game7-testnet-metadata.timer

@@ -0,0 +1,9 @@
+[Unit]
+Description=Execute Game7 testnet metadata crawler each 10m
+
+[Timer]
+OnBootSec=20s
+OnUnitActiveSec=60m
+
+[Install]
+WantedBy=timers.target

crawlers/deploy/game7-testnet-state.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=Execute Game7 testnet state crawler
+After=network.target
+
+[Service]
+Type=oneshot
+WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
+EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli crawl-jobs --moonstream-token "${MOONSTREAM_PUBLIC_QUERIES_DATA_ACCESS_TOKEN}" --blockchain game7_testnet
+CPUWeight=60
+SyslogIdentifier=game7-testnet-state

crawlers/deploy/game7-testnet-state.timer

@@ -0,0 +1,9 @@
+[Unit]
+Description=Execute Game7 testnet state crawler each 5m
+
+[Timer]
+OnBootSec=15s
+OnUnitActiveSec=5m
+
+[Install]
+WantedBy=timers.target

crawlers/deploy/sepolia-state.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=Execute Sepolia state crawler
+After=network.target
+
+[Service]
+Type=oneshot
+WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
+EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli crawl-jobs --moonstream-token "${MOONSTREAM_PUBLIC_QUERIES_DATA_ACCESS_TOKEN}" --blockchain sepolia
+CPUWeight=60
+SyslogIdentifier=sepolia-state

crawlers/deploy/sepolia-state.timer

@@ -0,0 +1,9 @@
+[Unit]
+Description=Execute Sepolia state crawler each 5m
+
+[Timer]
+OnBootSec=15s
+OnUnitActiveSec=5m
+
+[Install]
+WantedBy=timers.target

crawlers/mooncrawl/mooncrawl/actions.py

@@ -16,7 +16,15 @@ from moonstream.client import (  # type: ignore
     Moonstream,
     MoonstreamQueryResultUrl,
 )
+from sqlalchemy import text, TextClause
+from moonstreamtypes.blockchain import (
+    AvailableBlockchainType,
+    get_block_model,
+    get_label_model,
+    get_transaction_model,
+)
 
+from .data import QueryDataUpdate
 from .middleware import MoonstreamHTTPException
 from .settings import (
     bugout_client as bc,
@@ -34,6 +42,12 @@ class EntityCollectionNotFoundException(Exception):
     """
 
 
+class QueryTextClauseException(Exception):
+    """
+    Raised when query can't be transformed to TextClause
+    """
+
+
 def push_data_to_bucket(
     data: Any, key: str, bucket: str, metadata: Dict[str, Any] = {}
 ) -> None:
@@ -116,6 +130,7 @@ def recive_S3_data_from_query(
     client: Moonstream,
     token: Union[str, uuid.UUID],
     query_name: str,
+    query_params: Dict[str, Any] = {},
     params: Dict[str, Any] = {},
     time_await: int = 2,
     max_retries: int = 30,
@@ -133,18 +148,21 @@ def recive_S3_data_from_query(
     if_modified_since = if_modified_since_datetime.strftime("%a, %d %b %Y %H:%M:%S GMT")
 
     time.sleep(2)
-    if custom_body:
+    if custom_body or query_params:
         headers = {
             "Authorization": f"Bearer {token}",
+            "Content-Type": "application/json",
         }
         json = custom_body
 
         response = requests.post(
             url=f"{client.api.endpoints[ENDPOINT_QUERIES]}/{query_name}/update_data",
             headers=headers,
+            params=query_params,
             json=json,
             timeout=5,
         )
+
         data_url = MoonstreamQueryResultUrl(url=response.json()["url"])
     else:
         data_url = client.exec_query(
@@ -226,3 +244,96 @@ def get_customer_db_uri(
     except Exception as e:
         logger.error(f"Error get customer db uri: {str(e)}")
         raise MoonstreamHTTPException(status_code=500, internal_error=e)
+
+
+def resolve_table_names(request_data: QueryDataUpdate) -> Dict[str, str]:
+    """
+    Determines the table names based on the blockchain and labels version.
+    Returns an empty dictionary if blockchain is not provided.
+    """
+    if not request_data.blockchain:
+        return {"labels_table": "ethereum_labels"}
+
+    if request_data.blockchain not in [i.value for i in AvailableBlockchainType]:
+        logger.error(f"Unknown blockchain {request_data.blockchain}")
+        raise MoonstreamHTTPException(status_code=403, detail="Unknown blockchain")
+
+    blockchain = AvailableBlockchainType(request_data.blockchain)
+    labels_version = 2
+
+    if request_data.customer_id is not None and request_data.instance_id is not None:
+        labels_version = 3
+
+    print(labels_version, blockchain)
+
+    tables = {
+        "labels_table": get_label_model(blockchain, labels_version).__tablename__,
+    }
+
+    if labels_version != 3:
+        tables.update(
+            {
+                "transactions_table": get_transaction_model(blockchain).__tablename__,
+                "blocks_table": get_block_model(blockchain).__tablename__,
+            }
+        )
+
+    return tables
+
+
+def prepare_query(
+    requested_query: str, tables: Dict[str, str], query_id: str
+) -> TextClause:
+    """
+    Prepares the SQL query by replacing placeholders with actual table names.
+    """
+    # Check and replace placeholders only if they exist in the query
+    if "__labels_table__" in requested_query:
+        requested_query = requested_query.replace(
+            "__labels_table__", tables.get("labels_table", "ethereum_labels")
+        )
+
+    if "__transactions_table__" in requested_query and "transactions_table" in tables:
+        requested_query = requested_query.replace(
+            "__transactions_table__", tables["transactions_table"]
+        )
+
+    if "__blocks_table__" in requested_query and "blocks_table" in tables:
+        requested_query = requested_query.replace(
+            "__blocks_table__", tables["blocks_table"]
+        )
+
+        # Check if it can transform to TextClause
+    try:
+        query = text(requested_query)
+    except Exception as e:
+        logger.error(
+            f"Can't parse query {query_id} to TextClause in drones /query_update endpoint, error: {e}"
+        )
+        raise QueryTextClauseException(
+            f"Can't parse query {query_id} to TextClause in drones /query_update endpoint, error: {e}"
+        )
+
+    return query
+
+## DB V3 
+
+def request_connection_string(
+    customer_id: str,
+    instance_id: int,
+    token: str,
+    user: str = "seer",  # token with write access
+) -> str:
+    """
+    Request connection string from the Moonstream DB V3 Controller API.
+    Default user is seer with write access
+    """
+    response = requests.get(
+        f"{MOONSTREAM_DB_V3_CONTROLLER_API}/customers/{customer_id}/instances/{instance_id}/creds/{user}/url",
+        headers={"Authorization": f"Bearer {token}"},
+    )
+
+    response.raise_for_status()
+
+    return response.text.replace('"', "")
+

crawlers/mooncrawl/mooncrawl/api.py

@@ -13,13 +13,6 @@ import boto3  # type: ignore
 from bugout.data import BugoutJournalEntity, BugoutResource
 from fastapi import BackgroundTasks, FastAPI
 from fastapi.middleware.cors import CORSMiddleware
-from moonstreamdb.blockchain import (
-    AvailableBlockchainType,
-    get_block_model,
-    get_label_model,
-    get_transaction_model,
-)
-from sqlalchemy import text
 
 from . import data
 from .actions import (
@@ -27,6 +20,9 @@ from .actions import (
     generate_s3_access_links,
     get_entity_subscription_collection_id,
     query_parameter_hash,
+    prepare_query,
+    resolve_table_names,
+    QueryTextClauseException,
 )
 from .middleware import MoonstreamHTTPException
 from .settings import (
@@ -230,43 +226,20 @@ async def queries_data_update_handler(
         logger.error(f"Unhandled query execute exception, error: {e}")
         raise MoonstreamHTTPException(status_code=500)
 
-    requested_query = request_data.query
+    # Resolve table names based on the request data default ethereum
+    tables = resolve_table_names(request_data)
 
-    blockchain_table = "polygon_labels"
-    if request_data.blockchain:
-        if request_data.blockchain not in [i.value for i in AvailableBlockchainType]:
-            logger.error(f"Unknown blockchain {request_data.blockchain}")
-            raise MoonstreamHTTPException(status_code=403, detail="Unknown blockchain")
-
-        blockchain = AvailableBlockchainType(request_data.blockchain)
-
-        requested_query = (
-            requested_query.replace(
-                "__transactions_table__",
-                get_transaction_model(blockchain).__tablename__,
-            )
-            .replace(
-                "__blocks_table__",
-                get_block_model(blockchain).__tablename__,
-            )
-            .replace(
-                "__labels_table__",
-                get_label_model(blockchain).__tablename__,
-            )
-        )
-
-        blockchain_table = get_label_model(blockchain).__tablename__
-
-    # Check if it can transform to TextClause
+    # Prepare the query with the resolved table names
     try:
-        query = text(requested_query)
+        query = prepare_query(request_data.query, tables, query_id)
+    except QueryTextClauseException as e:
+        logger.error(f"Error preparing query for query id: {query_id}, error: {e}")
+        raise MoonstreamHTTPException(status_code=500, detail="Error preparing query")
     except Exception as e:
-        logger.error(
-            f"Can't parse query {query_id} to TextClause in drones /query_update endpoint, error: {e}"
-        )
-        raise MoonstreamHTTPException(status_code=500, detail="Can't parse query")
+        logger.error(f"Error preparing query for query id: {query_id}, error: {e}")
+        raise MoonstreamHTTPException(status_code=500, detail="Error preparing query")
 
-    # Get requried keys for query
+    # Get required keys for query
     expected_query_parameters = query._bindparams.keys()
 
     # request.params validations
@@ -301,9 +274,9 @@ async def queries_data_update_handler(
             params_hash=params_hash,
             customer_id=request_data.customer_id,
             instance_id=request_data.instance_id,
-            blockchain_table=blockchain_table,
+            blockchain_table=tables["labels_table"],
+            # Add any additional parameters needed for the task
         )
-
     except Exception as e:
         logger.error(f"Unhandled query execute exception, error: {e}")
         raise MoonstreamHTTPException(status_code=500)

crawlers/mooncrawl/mooncrawl/blockchain.py

@@ -46,6 +46,7 @@ from .settings import (
     MOONSTREAM_NODE_GAME7_ORBIT_ARBITRUM_SEPOLIA_A_EXTERNAL_URI,
     MOONSTREAM_NODE_IMX_ZKEVM_A_EXTERNAL_URI,
     MOONSTREAM_NODE_GAME7_TESTNET_A_EXTERNAL_URI,
+    MOONSTREAM_NODE_GAME7_A_EXTERNAL_URI,
     MOONSTREAM_NODE_SEPOLIA_A_EXTERNAL_URI,
     WEB3_CLIENT_REQUEST_TIMEOUT_SECONDS,
 )
@@ -85,6 +86,7 @@ default_uri_mapping = {
     AvailableBlockchainType.GAME7_ORBIT_ARBITRUM_SEPOLIA: MOONSTREAM_NODE_GAME7_ORBIT_ARBITRUM_SEPOLIA_A_EXTERNAL_URI,
     AvailableBlockchainType.IMX_ZKEVM: MOONSTREAM_NODE_IMX_ZKEVM_A_EXTERNAL_URI,
     AvailableBlockchainType.GAME7_TESTNET: MOONSTREAM_NODE_GAME7_TESTNET_A_EXTERNAL_URI,
+    AvailableBlockchainType.GAME7: MOONSTREAM_NODE_GAME7_A_EXTERNAL_URI,
     AvailableBlockchainType.SEPOLIA: MOONSTREAM_NODE_SEPOLIA_A_EXTERNAL_URI,
 }
 
@@ -161,7 +163,7 @@ def add_block(db_session, block: Any, blockchain_type: AvailableBlockchainType)
         size=block.size,
         state_root=block.stateRoot.hex(),
         timestamp=block.timestamp,
-        total_difficulty=block.totalDifficulty,
+        total_difficulty=block.get("totalDifficulty", None),
         transactions_root=block.transactionsRoot.hex(),
     )
     if blockchain_type == AvailableBlockchainType.XDAI:

crawlers/mooncrawl/mooncrawl/data.py

@@ -60,6 +60,7 @@ class TokenURIs(BaseModel):
     block_number: str
     block_timestamp: str
     address: str
+    block_hash: Optional[str] = None # for v3 only
 
 
 class ViewTasks(BaseModel):
@@ -69,3 +70,6 @@ class ViewTasks(BaseModel):
     name: str
     outputs: List[Dict[str, Any]]
     address: str
+    customer_id: Optional[str] = None
+    instance_id: Optional[str] = None
+    v3: Optional[bool] = False

crawlers/mooncrawl/mooncrawl/metadata_crawler/Readme.md

@@ -0,0 +1,180 @@
+# Metadata Crawler Architecture
+
+## Overview
+The metadata crawler is designed to fetch and store metadata for NFTs (Non-Fungible Tokens) from various blockchains. It supports both traditional database TokenURI view methods queries and Spire journal-based job configurations, with the ability to handle both v2 and v3 database structures.
+
+
+## Core Components
+
+### 1. Update Strategies
+
+#### Leak-Based Strategy (Legacy v2)
+- Uses probabilistic approach to determine which tokens to update
+- Controlled by `max_recrawl` parameter
+- Suitable for large collections with infrequent updates
+
+#### SQL-Based Strategy (v3)
+- Uses SQL queries to determine which tokens need updates
+- More precise tracking of token updates
+- Better suited for active collections
+
+### 2. Database Connections
+
+The crawler supports multiple database connection strategies:
+- Default Moonstream database connection
+- Custom database URI via `--custom-db-uri`
+- Per-customer instance connections (v3)
+  ```json
+  {
+    "customer_id": "...",
+    "instance_id": "...",
+    "blockchain": "ethereum",
+    "v3": true
+  }
+  ```
+
+### 3. Job Configuration
+Jobs can be configured in two ways:
+- Through Spire journal entries with tags `#metadata-job #{blockchain}`
+- Direct database queries (legacy mode) using TokenURI view method
+Example Spire journal entry:
+```json
+{
+    "type": "metadata-job",
+    "query_api": {
+        "name": "new_tokens_to_crawl",
+        "params": {
+            "address": "0x...",
+            "blockchain": "ethereum"
+        }
+    },
+    "contract_address": "0x...",
+    "blockchain": "ethereum",
+    "update_existing": false,
+    "v3": true,
+    "customer_id": "...", // Optional, for custom database
+    "instance_id": "..." // Optional, for custom database
+}
+```
+
+### 2. Data Flow
+1. **Token Discovery**
+   - Query API integration for dynamic token discovery
+   - Database queries for existing tokens
+   - Support for multiple addresses per job
+
+2. **Metadata Fetching**
+   - Parallel processing with ThreadPoolExecutor
+   - IPFS gateway support
+   - Automatic retry mechanism
+   - Rate limiting and batch processing
+
+3. **Storage**
+   - Supports both v2 and v3 database structures
+   - Batch upsert operations
+   - Efficient cleaning of old labels
+
+### 3. Database Structures
+
+v2:
+```python
+{
+    "label": METADATA_CRAWLER_LABEL,
+    "label_data": {
+        "type": "metadata",
+        "token_id": "...",
+        "metadata": {...}
+    },
+    "block_number": 1234567890
+    "block_timestamp": 456
+}
+```
+
+v3:
+```python
+{
+    "label": METADATA_CRAWLER_LABEL,
+    "label_type": "metadata",
+    "label_data": {
+        "token_id": "...",
+        "metadata": {...}
+    },
+    "address": "0x...",
+    "block_number": 123,
+    "block_timestamp": 456,
+    "block_hash": "0x..."
+}
+
+```
+
+## Key Features
+
+1. **Flexible Token Selection**
+   - Query API integration
+   - Support for multiple addresses
+   - Configurable update strategies
+
+2. **Efficient Processing**
+   - Batch processing
+   - Parallel metadata fetching
+   - Optimized database operations
+
+3. **Error Handling**
+   - Retry mechanism for failed requests
+   - Transaction management
+   - Detailed logging
+
+4. **Database Management**
+   - Efficient upsert operations
+   - Label cleaning
+   - Version compatibility (v2/v3)
+
+## Usage
+
+### CLI Options
+
+```bash
+metadata-crawler crawl \
+--blockchain ethereum \
+--commit-batch-size 50 \
+--max-recrawl 300 \
+--threads 4 \
+--spire true \
+--custom-db-uri "postgresql://..." # Optional
+```
+### Environment Variables
+- `MOONSTREAM_ADMIN_ACCESS_TOKEN`: Required for API access
+- `METADATA_CRAWLER_LABEL`: Label for database entries
+- `METADATA_TASKS_JOURNAL_ID`: Journal ID for metadata tasks
+
+
+### Database Modes
+
+1. **Legacy Mode (v2)**
+   - Uses leak-based update strategy
+   - Single database connection
+   - Simple metadata structure
+
+2. **Modern Mode (v3)**
+   - SQL-based update tracking
+   - Support for multiple database instances
+   - Enhanced metadata structure
+   - Per-customer database isolation
+
+
+## Best Practices
+
+1. **Job Configuration**
+   - Use descriptive job names
+   - Group related addresses
+   - Set appropriate update intervals
+
+2. **Performance Optimization**
+   - Adjust batch sizes based on network conditions
+   - Monitor thread count vs. performance
+   - Use appropriate IPFS gateways
+
+3. **Maintenance**
+   - Regular cleaning of old labels
+   - Monitor database size
+   - Check for failed metadata fetches

crawlers/mooncrawl/mooncrawl/metadata_crawler/cli.py

@@ -3,21 +3,32 @@ import json
 import logging
 import random
 import urllib.request
-from concurrent.futures import ThreadPoolExecutor
-from typing import Any, Dict, List, Optional
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from sqlalchemy.orm import Session
+from typing import Any, Dict, List, Optional, Tuple
 from urllib.error import HTTPError
 
-from moonstreamdb.blockchain import AvailableBlockchainType
+from bugout.exceptions import BugoutResponseException
+from moonstreamtypes.blockchain import AvailableBlockchainType
+from moonstreamdb.blockchain import AvailableBlockchainType as AvailableBlockchainTypeV2
 
-from ..db import yield_db_preping_session_ctx, yield_db_read_only_preping_session_ctx
+
+from ..actions import get_all_entries_from_search, request_connection_string
+from ..settings import MOONSTREAM_ADMIN_ACCESS_TOKEN, MOONSTREAM_METADATA_TASKS_JOURNAL, MOONSTREAM_PUBLIC_QUERIES_DATA_ACCESS_TOKEN
+from ..db import yield_db_preping_session_ctx, yield_db_read_only_preping_session_ctx, create_moonstream_engine, sessionmaker
+from ..data import TokenURIs
 from .db import (
     clean_labels_from_db,
     get_current_metadata_for_address,
     get_tokens_id_wich_may_updated,
     get_uris_of_tokens,
     metadata_to_label,
+    get_tokens_to_crawl,
+    upsert_metadata_labels,
 )
 
+from ..settings import moonstream_client as mc
+
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 
@@ -50,7 +61,6 @@ def crawl_uri(metadata_uri: str) -> Any:
     result = None
     while retry < 3:
         try:
-
             if metadata_uri.startswith("ipfs://"):
                 metadata_uri = metadata_uri.replace(
                     "ipfs://", "https://ipfs.io/ipfs/", 1
@@ -61,10 +71,7 @@ def crawl_uri(metadata_uri: str) -> Any:
 
             response = urllib.request.urlopen(req, timeout=10)
 
-            if (
-                metadata_uri.startswith("data:application/json")
-                or response.status == 200
-            ):
+            if metadata_uri.startswith("data:application/json") or response.status == 200:
                 result = json.loads(response.read())
                 break
             retry += 1
@@ -72,6 +79,8 @@ def crawl_uri(metadata_uri: str) -> Any:
         except HTTPError as error:
             logger.error(f"request end with error statuscode: {error.code}")
             retry += 1
+            if error.code == 404:
+                return None
             continue
         except Exception as err:
             logger.error(err)
@@ -81,167 +90,329 @@ def crawl_uri(metadata_uri: str) -> Any:
     return result
 
 
+def process_address_metadata_with_leak(
+    address: str,
+    blockchain_type: AvailableBlockchainType,
+    batch_size: int,
+    max_recrawl: int,
+    threads: int,
+    tokens: List[TokenURIs],
+) -> None:
+    """
+    Process metadata for a single address with v3 support
+    """
+    with yield_db_read_only_preping_session_ctx() as db_session_read_only:
+        try:
+
+            already_parsed = get_current_metadata_for_address(
+                db_session=db_session_read_only,
+                blockchain_type=blockchain_type,
+                address=address,
+            )
+
+            maybe_updated = get_tokens_id_wich_may_updated(
+                db_session=db_session_read_only,
+                blockchain_type=blockchain_type,
+                address=address,
+            )
+        except Exception as err:
+            logger.warning(f"Error while getting metadata state for address {address}: {err}")
+            return
+
+    with yield_db_preping_session_ctx() as db_session:
+        try:
+            logger.info(f"Starting to crawl metadata for address: {address}")
+            logger.info(f"Maybe updated: {len(maybe_updated)}")
+
+            # Calculate how many tokens we can 'leak' so total recrawled (maybe_updated + leaked) <= max_recrawl
+            num_already_parsed = len(already_parsed)
+            num_maybe_updated = len(maybe_updated)
+            free_spots = max(0, max_recrawl - num_maybe_updated)
+
+            if num_already_parsed > 0 and free_spots > 0:
+                leak_rate = free_spots / num_already_parsed
+            else:
+                leak_rate = 0
+
+            logger.info(
+                f"Leak rate: {leak_rate} for {address} with maybe updated {len(maybe_updated)}"
+            )
+
+            # TODO: Fully random leak is not correct, we should leak based on created_at
+            parsed_with_leak = leak_of_crawled_uri(
+                already_parsed, leak_rate, maybe_updated
+            )
+
+            logger.info(f"Already parsed: {len(already_parsed)} for {address}")
+            logger.info(f"Amount of tokens to parse: {len(tokens)} for {address}")
+
+            # Remove already parsed tokens
+            new_tokens = [
+                token for token in tokens
+                if token.token_id not in parsed_with_leak
+            ]
+
+            for requests_chunk in [
+                new_tokens[i : i + batch_size]
+                for i in range(0, len(new_tokens), batch_size)
+            ]:
+                metadata_batch = []
+                try:
+
+                    # Gather all metadata in parallel
+                    with ThreadPoolExecutor(max_workers=threads) as executor:
+                        future_to_token = {
+                            executor.submit(crawl_uri, token.token_uri): token
+                            for token in requests_chunk
+                        }
+                        for future in as_completed(future_to_token):
+                            token = future_to_token[future]
+                            try:
+                                metadata = future.result(timeout=10)
+                                if metadata:
+                                    metadata_batch.append((token, metadata))
+                            except Exception as e:
+                                logger.error(f"Error fetching metadata for token {token.token_id}: {e}")
+                                continue
+
+                    if metadata_batch:
+                        # Batch upsert all metadata
+                        upsert_metadata_labels(
+                            db_session=db_session,
+                            blockchain_type=blockchain_type,
+                            metadata_batch=metadata_batch,
+                            v3=False
+                        )
+                        
+                        clean_labels_from_db(
+                            db_session=db_session,
+                            blockchain_type=blockchain_type,
+                            address=address,
+                        )
+                        logger.info(f"Write {len(metadata_batch)} labels for {address}")
+
+                except Exception as err:
+                    logger.warning(f"Error while writing labels for address {address}: {err}")
+                    db_session.rollback()
+
+        except Exception as err:
+            logger.warning(f"Error while crawling metadata for address {address}: {err}")
+            db_session.rollback()
+
+
+
+def process_address_metadata(
+    address: str,
+    blockchain_type: AvailableBlockchainType,
+    db_session: Session,
+    batch_size: int,
+    max_recrawl: int,
+    threads: int,
+    tokens: List[TokenURIs],
+) -> None:
+    """
+    Process metadata for a single address with v3 support
+    Leak logic is implemented in sql statement
+    """
+
+
+    logger.info(f"Processing address {address} with {len(tokens)} tokens")
+
+    total_tokens = len(tokens)
+    total_chunks = (total_tokens + batch_size - 1) // batch_size
+
+    for chunk_index, requests_chunk in enumerate([
+        tokens[i : i + batch_size]
+        for i in range(0, len(tokens), batch_size)
+    ]):
+        logger.info(
+            f"Processing chunk {chunk_index + 1}/{total_chunks} "
+            f"({len(requests_chunk)} tokens) for address {address}"
+        )
+
+
+        metadata_batch = []
+        with ThreadPoolExecutor(max_workers=threads) as executor:
+            future_to_token = {
+                executor.submit(crawl_uri, token.token_uri): token
+                for token in requests_chunk
+            }
+            for future in as_completed(future_to_token):
+                token = future_to_token[future]
+                metadata = future.result(timeout=10)
+                metadata_batch.append((token, metadata))
+
+
+        upsert_metadata_labels(
+            db_session=db_session,
+            blockchain_type=blockchain_type,
+            metadata_batch=metadata_batch,
+            v3=True
+        )
+
+        logger.info(f"Wrote {len(metadata_batch)} labels for {address}")
+
+        db_session.commit()
+
+        clean_labels_from_db(
+            db_session=db_session,
+            blockchain_type=blockchain_type,
+            address=address,
+            version=3
+        )
+
+        db_session.commit()
+    
+
 def parse_metadata(
     blockchain_type: AvailableBlockchainType,
     batch_size: int,
     max_recrawl: int,
     threads: int,
+    custom_db_uri: Optional[str] = None,
 ):
     """
     Parse all metadata of tokens.
     """
-
     logger.info("Starting metadata crawler")
     logger.info(f"Processing blockchain {blockchain_type.value}")
 
-    # run crawling of levels
-    with yield_db_read_only_preping_session_ctx() as db_session_read_only:
+    # Check if blockchain exists in v2 package
+    if blockchain_type.value in [chain.value for chain in AvailableBlockchainTypeV2]:
         try:
-            # get all tokens with uri
-            logger.info("Requesting all tokens with uri from database")
-            uris_of_tokens = get_uris_of_tokens(db_session_read_only, blockchain_type)
-
-            tokens_uri_by_address: Dict[str, Any] = {}
-
-            for token_uri_data in uris_of_tokens:
-                if token_uri_data.address not in tokens_uri_by_address:
-                    tokens_uri_by_address[token_uri_data.address] = []
-                tokens_uri_by_address[token_uri_data.address].append(token_uri_data)
+            logger.info(f"Processing v2 blockchain: {blockchain_type.value}")
+            # Get tokens to crawl v2 flow
+            with yield_db_read_only_preping_session_ctx() as db_session_read_only:
+                tokens_uri_by_address = get_tokens_to_crawl(
+                    db_session_read_only,
+                    blockchain_type,
+                    {},
+                )
 
+            # Process each address
+            for address, tokens in tokens_uri_by_address.items():
+                process_address_metadata_with_leak(
+                    address=address,
+                    blockchain_type=blockchain_type,
+                    batch_size=batch_size,
+                    max_recrawl=max_recrawl,
+                    threads=threads,
+                    tokens=tokens,
+                )
         except Exception as err:
-            logger.error(f"Error while requesting tokens with uri from database: {err}")
-            return
+            logger.error(f"V2 flow failed: {err}, continuing with Spire flow")
 
-    for address in tokens_uri_by_address:
-        with yield_db_read_only_preping_session_ctx() as db_session_read_only:
+    # Continue with Spire flow regardless of v2 result
+    spire_jobs = []
+
+    # Get all jobs for this blockchain from Spire
+    search_query = f"#metadata-job #{blockchain_type.value}"
+    try:
+        entries = get_all_entries_from_search(
+            journal_id=MOONSTREAM_METADATA_TASKS_JOURNAL,
+            search_query=search_query,
+            token=MOONSTREAM_ADMIN_ACCESS_TOKEN,
+            content=True,
+            limit=1000,
+        )
+        
+        logger.info(f"Found {len(entries)} metadata jobs for blockchain {blockchain_type.value}")
+        
+        for entry in entries:
             try:
-                already_parsed = get_current_metadata_for_address(
-                    db_session=db_session_read_only,
-                    blockchain_type=blockchain_type,
-                    address=address,
-                )
-
-                maybe_updated = get_tokens_id_wich_may_updated(
-                    db_session=db_session_read_only,
-                    blockchain_type=blockchain_type,
-                    address=address,
-                )
+                if not entry.content:
+                    continue
+                
+                job = json.loads(entry.content)
+                if job.get("blockchain") != blockchain_type.value:
+                    logger.warning(f"Skipping job with mismatched blockchain: {job.get('blockchain')} != {blockchain_type.value}")
+                    continue
+                spire_jobs.append(job)
             except Exception as err:
-                logger.warning(err)
-                logger.warning(
-                    f"Error while requesting metadata for address: {address}"
-                )
+                id = entry.entry_url.split("/")[-1]
+                logger.error(f"Error parsing job from entry {id}: {err}")
                 continue
+    except BugoutResponseException as err:
+        logger.error(f"Bugout error fetching jobs from journal: {err.detail}")
+    except Exception as err:
+        logger.error(f"Error fetching jobs from journal: {err}")
+        return
 
-        with yield_db_preping_session_ctx() as db_session:
+    # Process each job
+
+    # sessions list for each customer and instance
+    sessions_by_customer: Dict[Tuple[str, str], Session] = {}
+
+    # all sessions in one try block
+    try: 
+        for job in spire_jobs:
             try:
-                logger.info(f"Starting to crawl metadata for address: {address}")
+                customer_id = job.get("customer_id")
+                instance_id = job.get("instance_id")
 
-                leak_rate = 0.0
-
-                if len(maybe_updated) > 0:
-                    free_spots = len(maybe_updated) / max_recrawl
-
-                    if free_spots > 1:
-                        leak_rate = 0
+                if (customer_id, instance_id) not in sessions_by_customer:
+                    # Create session
+                    # Assume fetch_connection_string fetches the connection string
+                    if custom_db_uri:
+                        connection_string = custom_db_uri
                     else:
-                        leak_rate = 1 - (
-                            len(already_parsed) - max_recrawl + len(maybe_updated)
-                        ) / len(already_parsed)
-
-                parsed_with_leak = leak_of_crawled_uri(
-                    already_parsed, leak_rate, maybe_updated
-                )
-
-                logger.info(
-                    f"Leak rate: {leak_rate} for {address} with maybe updated {len(maybe_updated)}"
-                )
-
-                logger.info(f"Already parsed: {len(already_parsed)} for {address}")
-
-                logger.info(
-                    f"Amount of state in database: {len(tokens_uri_by_address[address])} for {address}"
-                )
-
-                logger.info(
-                    f"Amount of tokens parsed with leak: {len(parsed_with_leak)} for {address}"
-                )
-
-                # Remove already parsed tokens
-                new_tokens_uri_by_address = [
-                    token_uri_data
-                    for token_uri_data in tokens_uri_by_address[address]
-                    if token_uri_data.token_id not in parsed_with_leak
-                ]
-
-                logger.info(
-                    f"Amount of tokens to parse: {len(new_tokens_uri_by_address)} for {address}"
-                )
-
-                for requests_chunk in [
-                    new_tokens_uri_by_address[i : i + batch_size]
-                    for i in range(0, len(new_tokens_uri_by_address), batch_size)
-                ]:
-                    writed_labels = 0
-                    db_session.commit()
-
-                    try:
-                        with db_session.begin():
-                            for token_uri_data in requests_chunk:
-                                with ThreadPoolExecutor(
-                                    max_workers=threads
-                                ) as executor:
-                                    future = executor.submit(
-                                        crawl_uri, token_uri_data.token_uri
-                                    )
-                                    metadata = future.result(timeout=10)
-                                db_session.add(
-                                    metadata_to_label(
-                                        blockchain_type=blockchain_type,
-                                        metadata=metadata,
-                                        token_uri_data=token_uri_data,
-                                    )
-                                )
-                                writed_labels += 1
-
-                            if writed_labels > 0:
-                                clean_labels_from_db(
-                                    db_session=db_session,
-                                    blockchain_type=blockchain_type,
-                                    address=address,
-                                )
-                                logger.info(
-                                    f"Write {writed_labels} labels for {address}"
-                                )
-                        # trasaction is commited here
-                    except Exception as err:
-                        logger.warning(err)
-                        logger.warning(
-                            f"Error while writing labels for address: {address}"
+                        connection_string = request_connection_string(
+                            customer_id=customer_id,
+                            instance_id=instance_id,
+                            token=MOONSTREAM_ADMIN_ACCESS_TOKEN,
                         )
-                        db_session.rollback()
+                    engine = create_moonstream_engine(connection_string, 2, 100000)
+                    session = sessionmaker(bind=engine)
+                    try:
+                        sessions_by_customer[(customer_id, instance_id)] = session()
+                    except Exception as e:
+                        logger.error(f"Connection to {engine} failed: {e}")
+                        continue
 
-                clean_labels_from_db(
-                    db_session=db_session,
-                    blockchain_type=blockchain_type,
-                    address=address,
-                )
+                
+                # Get tokens to crawl
+                tokens_uri_by_address = get_tokens_to_crawl(
+                    sessions_by_customer[(customer_id, instance_id)],
+                    blockchain_type,
+                    job,
+                ) 
+
+                for address, tokens in tokens_uri_by_address.items():
+                    process_address_metadata(
+                        address=address,
+                        blockchain_type=blockchain_type,
+                        db_session=sessions_by_customer[(customer_id, instance_id)],
+                        batch_size=batch_size,
+                        max_recrawl=max_recrawl,
+                        threads=threads,
+                        tokens=tokens,
+                    )
             except Exception as err:
-                logger.warning(err)
-                logger.warning(f"Error while crawling metadata for address: {address}")
-                db_session.rollback()
+                logger.error(f"Error processing job: {err}")
                 continue
+    except Exception as err:
+        logger.error(f"Error processing jobs: {err}")
+        raise err
+     
+    finally:
+        for session in sessions_by_customer.values():
+            try:
+                session.close()
+            except Exception as err:
+                logger.error(f"Error closing session: {err}")
 
 
 def handle_crawl(args: argparse.Namespace) -> None:
     """
     Parse all metadata of tokens.
     """
-
     blockchain_type = AvailableBlockchainType(args.blockchain)
-
     parse_metadata(
-        blockchain_type, args.commit_batch_size, args.max_recrawl, args.threads
+        blockchain_type,
+        args.commit_batch_size,
+        args.max_recrawl,
+        args.threads,
+        args.custom_db_uri,
     )
 
 
@@ -259,7 +430,7 @@ def main() -> None:
         "--blockchain",
         "-b",
         type=str,
-        help="Type of blockchain wich writng in database",
+        help="Type of blockchain which writing in database",
         required=True,
     )
     metadata_crawler_parser.add_argument(
@@ -283,6 +454,11 @@ def main() -> None:
         default=4,
         help="Amount of threads for crawling",
     )
+    metadata_crawler_parser.add_argument(
+        "--custom-db-uri",
+        type=str,
+        help="Custom db uri to use for crawling",
+    )
     metadata_crawler_parser.set_defaults(func=handle_crawl)
 
     args = parser.parse_args()

crawlers/mooncrawl/mooncrawl/metadata_crawler/db.py

@@ -1,13 +1,29 @@
 import json
 import logging
-from typing import Any, Dict, List, Optional
+from hexbytes import HexBytes
+from typing import Any, Dict, List, Optional, Tuple
+###from sqlalchemy import 
+from sqlalchemy.dialects.postgresql import insert
 
-from moonstreamdb.blockchain import AvailableBlockchainType, get_label_model
+from datetime import datetime
+
+from moonstreamtypes.blockchain import AvailableBlockchainType, get_label_model
 from sqlalchemy.orm import Session
 from sqlalchemy.sql import text
 
+from ..actions import recive_S3_data_from_query
 from ..data import TokenURIs
-from ..settings import CRAWLER_LABEL, METADATA_CRAWLER_LABEL, VIEW_STATE_CRAWLER_LABEL
+from ..settings import (
+    CRAWLER_LABEL,
+    METADATA_CRAWLER_LABEL,
+    VIEW_STATE_CRAWLER_LABEL,
+    MOONSTREAM_ADMIN_ACCESS_TOKEN,
+    MOONSTREAM_PUBLIC_QUERIES_DATA_ACCESS_TOKEN,
+    bugout_client as bc,
+    moonstream_client as mc,
+)
+from moonstream.client import Moonstream  # type: ignore
+
 
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
@@ -18,11 +34,13 @@ def metadata_to_label(
     metadata: Optional[Dict[str, Any]],
     token_uri_data: TokenURIs,
     label_name=METADATA_CRAWLER_LABEL,
+    v3: bool = False,
 ):
     """
-    Creates a label model.
+    Creates a label model with support for v2 and v3 database structures.
     """
-    label_model = get_label_model(blockchain_type)
+    version = 3 if v3 else 2
+    label_model = get_label_model(blockchain_type, version=version)
 
     sanityzed_label_data = json.loads(
         json.dumps(
@@ -34,14 +52,34 @@ def metadata_to_label(
         ).replace(r"\u0000", "")
     )
 
-    label = label_model(
-        label=label_name,
-        label_data=sanityzed_label_data,
-        address=token_uri_data.address,
-        block_number=token_uri_data.block_number,
-        transaction_hash=None,
-        block_timestamp=token_uri_data.block_timestamp,
-    )
+    if v3:
+        # V3 structure similar to state crawler
+        label_data = {
+            "token_id": token_uri_data.token_id,
+            "metadata": metadata,
+        }
+
+        label = label_model(
+            label=label_name,
+            label_name="metadata",  # Fixed name for metadata labels
+            label_type="metadata",
+            label_data=label_data,
+            address=HexBytes(token_uri_data.address),
+            block_number=token_uri_data.block_number,
+            # Use a fixed tx hash for metadata since it's not from a transaction
+            block_timestamp=token_uri_data.block_timestamp,
+            block_hash=token_uri_data.block_hash if hasattr(token_uri_data, 'block_hash') else None,
+        )
+    else:
+        # Original v2 structure
+        label = label_model(
+            label=label_name,
+            label_data=sanityzed_label_data,
+            address=token_uri_data.address,
+            block_number=token_uri_data.block_number,
+            transaction_hash=None,
+            block_timestamp=token_uri_data.block_timestamp,
+        )
 
     return label
 
@@ -60,13 +98,13 @@ def commit_session(db_session: Session) -> None:
 
 
 def get_uris_of_tokens(
-    db_session: Session, blockchain_type: AvailableBlockchainType
+    db_session: Session, blockchain_type: AvailableBlockchainType, version: int = 2
 ) -> List[TokenURIs]:
     """
     Get meatadata URIs.
     """
 
-    label_model = get_label_model(blockchain_type)
+    label_model = get_label_model(blockchain_type, version=version)
 
     table = label_model.__tablename__
 
@@ -113,13 +151,13 @@ def get_uris_of_tokens(
 
 
 def get_current_metadata_for_address(
-    db_session: Session, blockchain_type: AvailableBlockchainType, address: str
+    db_session: Session, blockchain_type: AvailableBlockchainType, address: str, version: int = 2
 ):
     """
     Get existing metadata.
     """
 
-    label_model = get_label_model(blockchain_type)
+    label_model = get_label_model(blockchain_type, version=version)
 
     table = label_model.__tablename__
 
@@ -149,7 +187,7 @@ def get_current_metadata_for_address(
 
 
 def get_tokens_id_wich_may_updated(
-    db_session: Session, blockchain_type: AvailableBlockchainType, address: str
+    db_session: Session, blockchain_type: AvailableBlockchainType, address: str, version: int = 2
 ):
     """
     Returns a list of tokens which may have updated information.
@@ -163,7 +201,7 @@ def get_tokens_id_wich_may_updated(
     Required integration with entity API and opcodes crawler.
     """
 
-    label_model = get_label_model(blockchain_type)
+    label_model = get_label_model(blockchain_type, version=version)
 
     table = label_model.__tablename__
 
@@ -233,14 +271,14 @@ def get_tokens_id_wich_may_updated(
 
 
 def clean_labels_from_db(
-    db_session: Session, blockchain_type: AvailableBlockchainType, address: str
+    db_session: Session, blockchain_type: AvailableBlockchainType, address: str, version: int = 2
 ):
     """
     Remove existing labels.
     But keep the latest one for each token.
     """
 
-    label_model = get_label_model(blockchain_type)
+    label_model = get_label_model(blockchain_type, version=version)
 
     table = label_model.__tablename__
 
@@ -273,3 +311,165 @@ def clean_labels_from_db(
         ),
         {"address": address, "label": METADATA_CRAWLER_LABEL},
     )
+
+
+def get_tokens_from_query_api(
+    client: Moonstream,
+    blockchain_type: AvailableBlockchainType,
+    query_name: str,
+    params: dict,
+    token: str,
+    customer_id: Optional[str] = None,
+    instance_id: Optional[str] = None,
+) -> List[TokenURIs]:
+    """
+    Get token URIs from Query API results
+    """
+
+    query_params = {}
+    
+    if customer_id and instance_id:
+        query_params["customer_id"] = customer_id
+        query_params["instance_id"] = instance_id
+
+    try:
+        data = recive_S3_data_from_query(
+            client=client,
+            token=token,
+            query_name=query_name,
+            params={},
+            query_params=query_params,
+            custom_body={
+                "blockchain": blockchain_type.value,
+                "params": params,
+            }
+        )
+        
+        # Convert query results to TokenURIs format
+        results = []
+        for item in data.get("data", []):
+            results.append(
+                TokenURIs(
+                    token_id=str(item.get("token_id")),
+                    address=item.get("address"),
+                    token_uri=item.get("token_uri"),
+                    block_number=item.get("block_number"),
+                    block_timestamp=item.get("block_timestamp"),
+                )
+            )
+        return results
+    except Exception as err:
+        logger.error(f"Error fetching data from Query API: {err}")
+        return []
+
+def get_tokens_to_crawl(
+    db_session: Session,
+    blockchain_type: AvailableBlockchainType,
+    spire_job: Optional[dict] = None,
+) -> Dict[str, List[TokenURIs]]:
+    """`
+    Get tokens to crawl either from Query API (if specified in Spire job) or database
+    """
+    tokens_uri_by_address = {}
+
+    if spire_job:
+        if "query_api" not in spire_job:
+            raise ValueError("Query API is not specified in Spire job")
+
+        # Get tokens from Query API
+        query_config = spire_job["query_api"]
+        client = Moonstream()
+        
+        tokens = get_tokens_from_query_api(
+            client=client,
+            blockchain_type=blockchain_type,
+            query_name=query_config["name"],
+            params=query_config["params"],
+            token=MOONSTREAM_PUBLIC_QUERIES_DATA_ACCESS_TOKEN,
+            customer_id=spire_job["customer_id"],
+            instance_id=spire_job["instance_id"],
+        )
+        
+        # Group by address
+        for token in tokens:
+            if token.address not in tokens_uri_by_address:
+                tokens_uri_by_address[token.address] = []
+            tokens_uri_by_address[token.address].append(token)
+    else:
+        # Get tokens from database (existing logic)
+        uris_of_tokens = get_uris_of_tokens(db_session, blockchain_type)
+        for token_uri_data in uris_of_tokens:
+            if token_uri_data.address not in tokens_uri_by_address:
+                tokens_uri_by_address[token_uri_data.address] = []
+            tokens_uri_by_address[token_uri_data.address].append(token_uri_data)
+
+    return tokens_uri_by_address
+
+def upsert_metadata_labels(
+    db_session: Session,
+    blockchain_type: AvailableBlockchainType,
+    metadata_batch: List[Tuple[TokenURIs, Optional[Dict[str, Any]]]],
+    v3: bool = False,
+    db_batch_size: int = 100,
+
+) -> None:
+    """
+    Batch upsert metadata labels - update if exists, insert if not.
+    """
+    try:
+        version = 3 if v3 else 2
+        label_model = get_label_model(blockchain_type, version=version)
+        
+        # Prepare batch of labels
+        labels_data = []
+        for token_uri_data, metadata in metadata_batch:
+
+            if v3:
+                # V3 structure
+                label_data = {
+                    "token_id": token_uri_data.token_id,
+                    "metadata": metadata,
+                }
+                
+                labels_data.append({
+                    "label": METADATA_CRAWLER_LABEL,
+                    "label_name": "metadata",
+                    "label_type": "metadata",
+                    "label_data": label_data,
+                    "address": HexBytes(token_uri_data.address),
+                    "block_number": token_uri_data.block_number,
+                    "block_timestamp": token_uri_data.block_timestamp,
+                    "block_hash": getattr(token_uri_data, 'block_hash', None),
+                })
+            else:
+                # V2 structure
+                label_data = {
+                    "type": "metadata",
+                    "token_id": token_uri_data.token_id,
+                    "metadata": metadata,
+                }
+                
+                labels_data.append({
+                    "label": METADATA_CRAWLER_LABEL,
+                    "label_data": label_data,
+                    "address": token_uri_data.address,
+                    "block_number": token_uri_data.block_number,
+                    "transaction_hash": None,
+                    "block_timestamp": token_uri_data.block_timestamp,
+                })
+
+        if not labels_data:
+            return
+
+        # Create insert statement
+        insert_stmt = insert(label_model).values(labels_data)
+        result_stmt = insert_stmt.on_conflict_do_nothing(
+        )
+
+        db_session.execute(result_stmt)
+        
+        db_session.commit()
+
+    except Exception as err:
+        logger.error(f"Error batch upserting metadata labels: {err}")
+        raise

crawlers/mooncrawl/mooncrawl/settings.py

@@ -3,21 +3,16 @@ from typing import Dict, Optional
 from uuid import UUID
 
 from bugout.app import Bugout
-from moonstreamtypes.blockchain import AvailableBlockchainType
+from moonstreamtypes.blockchain import AvailableBlockchainType  # type: ignore
+from moonstream.client import Moonstream  # type: ignore
 
-# Bugout
+# APIs
+## Bugout
 BUGOUT_BROOD_URL = os.environ.get("BUGOUT_BROOD_URL", "https://auth.bugout.dev")
 BUGOUT_SPIRE_URL = os.environ.get("BUGOUT_SPIRE_URL", "https://spire.bugout.dev")
 
 bugout_client = Bugout(brood_api_url=BUGOUT_BROOD_URL, spire_api_url=BUGOUT_SPIRE_URL)
 
-
-MOONSTREAM_API_URL = os.environ.get("MOONSTREAM_API_URL", "https://api.moonstream.to")
-MOONSTREAM_ENGINE_URL = os.environ.get(
-    "MOONSTREAM_ENGINE_URL", "https://engineapi.moonstream.to"
-)
-
-
 BUGOUT_REQUEST_TIMEOUT_SECONDS_RAW = os.environ.get(
     "MOONSTREAM_BUGOUT_TIMEOUT_SECONDS", 30
 )
@@ -31,6 +26,24 @@ except:
 
 HUMBUG_REPORTER_CRAWLERS_TOKEN = os.environ.get("HUMBUG_REPORTER_CRAWLERS_TOKEN")
 
+
+## Moonstream
+MOONSTREAM_API_URL = os.environ.get("MOONSTREAM_API_URL", "https://api.moonstream.to")
+
+moonstream_client = Moonstream()
+
+
+## Moonstream Engine
+MOONSTREAM_ENGINE_URL = os.environ.get(
+    "MOONSTREAM_ENGINE_URL", "https://engineapi.moonstream.to"
+)
+
+## Moonstream DB
+MOONSTREAM_DB_V3_CONTROLLER_API = os.environ.get(
+    "MOONSTREAM_DB_V3_CONTROLLER_API", "https://mdb-v3-api.moonstream.to"
+)
+
+
 # Origin
 RAW_ORIGINS = os.environ.get("MOONSTREAM_CORS_ALLOWED_ORIGINS")
 if RAW_ORIGINS is None:
@@ -241,6 +254,12 @@ if MOONSTREAM_NODE_GAME7_TESTNET_A_EXTERNAL_URI == "":
         "MOONSTREAM_NODE_GAME7_TESTNET_A_EXTERNAL_URI env variable is not set"
     )
 
+MOONSTREAM_NODE_GAME7_A_EXTERNAL_URI = os.environ.get(
+    "MOONSTREAM_NODE_GAME7_A_EXTERNAL_URI", ""
+)
+if MOONSTREAM_NODE_GAME7_A_EXTERNAL_URI == "":
+    raise Exception("MOONSTREAM_NODE_GAME7_A_EXTERNAL_URI env variable is not set")
+
 
 MOONSTREAM_NODE_SEPOLIA_A_EXTERNAL_URI = os.environ.get(
     "MOONSTREAM_NODE_SEPOLIA_A_EXTERNAL_URI", ""
@@ -381,6 +400,8 @@ multicall_contracts: Dict[AvailableBlockchainType, str] = {
     AvailableBlockchainType.BLAST: "0xcA11bde05977b3631167028862bE2a173976CA11",
     AvailableBlockchainType.MANTLE: "0xcA11bde05977b3631167028862bE2a173976CA11",
     AvailableBlockchainType.MANTLE_SEPOLIA: "0xcA11bde05977b3631167028862bE2a173976CA11",
+    AvailableBlockchainType.GAME7_TESTNET: "0xcA11bde05977b3631167028862bE2a173976CA11",
+    AvailableBlockchainType.GAME7: "0x1422d8aC9b5E102E6EbA56F0949a2377AB3D8CE9",
 }
 
 
@@ -490,3 +511,19 @@ MOONSTREAM_DB_V3_CONTROLLER_API = os.environ.get(
 MOONSTREAM_DB_V3_SCHEMA_NAME = os.environ.get(
     "MOONSTREAM_DB_V3_SCHEMA_NAME", "blockchain"
 )
+
+
+MOONSTREAM_METADATA_TASKS_JOURNAL = os.environ.get(
+    "MOONSTREAM_METADATA_TASKS_JOURNAL", ""
+)
+
+
+### MOONSTREAM_PUBLIC_QUERIES_USER_TOKEN
+
+MOONSTREAM_PUBLIC_QUERIES_DATA_ACCESS_TOKEN = os.environ.get(
+    "MOONSTREAM_PUBLIC_QUERIES_DATA_ACCESS_TOKEN", ""
+)
+if MOONSTREAM_PUBLIC_QUERIES_DATA_ACCESS_TOKEN == "":
+    raise ValueError(
+        "MOONSTREAM_PUBLIC_QUERIES_DATA_ACCESS_TOKEN environment variable must be set"
+    )

crawlers/mooncrawl/mooncrawl/state_crawler/cli.py

@@ -8,7 +8,9 @@ from concurrent.futures import ThreadPoolExecutor
 from concurrent.futures._base import TimeoutError
 from pprint import pprint
 from typing import Any, Dict, List, Optional
+import requests
 from uuid import UUID
+from web3 import Web3
 
 from moonstream.client import Moonstream  # type: ignore
 from moonstreamtypes.blockchain import AvailableBlockchainType
@@ -17,7 +19,7 @@ from mooncrawl.moonworm_crawler.crawler import _retry_connect_web3
 from ..actions import recive_S3_data_from_query, get_all_entries_from_search
 from ..blockchain import connect
 from ..data import ViewTasks
-from ..db import PrePing_SessionLocal
+from ..db import PrePing_SessionLocal, create_moonstream_engine, sessionmaker
 from ..settings import (
     bugout_client as bc,
     INFURA_PROJECT_ID,
@@ -25,6 +27,7 @@ from ..settings import (
     multicall_contracts,
     MOONSTREAM_ADMIN_ACCESS_TOKEN,
     MOONSTREAM_STATE_CRAWLER_JOURNAL_ID,
+    MOONSTREAM_DB_V3_CONTROLLER_API,
 )
 from .db import clean_labels, commit_session, view_call_to_label
 from .Multicall2_interface import Contract as Multicall2
@@ -37,7 +40,26 @@ logger = logging.getLogger(__name__)
 client = Moonstream()
 
 
-def execute_query(query: Dict[str, Any], token: str):
+def request_connection_string(
+    customer_id: str,
+    instance_id: int,
+    token: str,
+    user: str = "seer",  # token with write access
+) -> str:
+    """
+    Request connection string from the Moonstream API.
+    """
+    response = requests.get(
+        f"{MOONSTREAM_DB_V3_CONTROLLER_API}/customers/{customer_id}/instances/{instance_id}/creds/{user}/url",
+        headers={"Authorization": f"Bearer {token}"},
+    )
+
+    response.raise_for_status()
+
+    return response.text.replace('"', "")
+
+
+def execute_query(query: Dict[str, Any], token: str) -> Any:
     """
     Query task example:
 
@@ -55,6 +77,8 @@ def execute_query(query: Dict[str, Any], token: str):
 
     """
 
+    print(f"Executing query: {query}")
+
     # get the query url
     query_url = query["query_url"]
 
@@ -65,6 +89,11 @@ def execute_query(query: Dict[str, Any], token: str):
     params = query["params"]
 
     body = {"params": params}
+    query_params = dict()
+
+    if query.get("customer_id") and query.get("instance_id"):
+        query_params["customer_id"] = query["customer_id"]
+        query_params["instance_id"] = query["instance_id"]
 
     if blockchain:
         body["blockchain"] = blockchain
@@ -76,6 +105,8 @@ def execute_query(query: Dict[str, Any], token: str):
         token=token,
         query_name=query_url,
         custom_body=body,
+        params=params,
+        query_params=query_params,
     )
 
     # extract the keys as a list
@@ -100,136 +131,142 @@ def execute_query(query: Dict[str, Any], token: str):
     return result
 
 
-def make_multicall(
-    multicall_method: Any,
-    calls: List[Any],
-    block_timestamp: int,
-    block_number: str = "latest",
-) -> Any:
+def encode_calls(calls: List[Dict[str, Any]]) -> List[tuple]:
+    """Encodes the call data for multicall."""
     multicall_calls = []
-
     for call in calls:
         try:
-            multicall_calls.append(
-                (
-                    call["address"],
-                    call["method"].encode_data(call["inputs"]).hex(),
-                )
-            )
+            encoded_data = call["method"].encode_data(call["inputs"]).hex()
+            multicall_calls.append((call["address"], encoded_data))
         except Exception as e:
             logger.error(
-                f'Error encoding data for method {call["method"].name} call: {call}'
+                f'Error encoding data for method {call["method"].name} call: {call}. Error: {e}'
             )
+    return multicall_calls
 
-    multicall_result = multicall_method(False, calls=multicall_calls).call(
-        block_identifier=block_number
-    )
 
+def perform_multicall(
+    multicall_method: Any, multicall_calls: List[tuple], block_identifier: str
+) -> Any:
+    """Performs the multicall and returns the result."""
+    return multicall_method(False, calls=multicall_calls).call(block_identifier=block_identifier)
+
+
+def process_multicall_result(
+    calls: List[Dict[str, Any]],
+    multicall_result: Any,
+    multicall_calls: List[tuple],
+    block_timestamp: int,
+    block_number: str,
+    block_hash: Optional[str],
+) -> List[Dict[str, Any]]:
+    """Processes the multicall result and decodes the data."""
     results = []
-
-    # Handle the case with not successful calls
     for index, encoded_data in enumerate(multicall_result):
+        call = calls[index]
         try:
-            if encoded_data[0]:
-                results.append(
-                    {
-                        "result": calls[index]["method"].decode_data(encoded_data[1]),
-                        "hash": calls[index]["hash"],
-                        "method": calls[index]["method"],
-                        "address": calls[index]["address"],
-                        "name": calls[index]["method"].name,
-                        "inputs": calls[index]["inputs"],
-                        "call_data": multicall_calls[index][1],
-                        "block_number": block_number,
-                        "block_timestamp": block_timestamp,
-                        "status": encoded_data[0],
-                    }
-                )
-            else:
-                results.append(
-                    {
-                        "result": calls[index]["method"].decode_data(encoded_data[1]),
-                        "hash": calls[index]["hash"],
-                        "method": calls[index]["method"],
-                        "address": calls[index]["address"],
-                        "name": calls[index]["method"].name,
-                        "inputs": calls[index]["inputs"],
-                        "call_data": multicall_calls[index][1],
-                        "block_number": block_number,
-                        "block_timestamp": block_timestamp,
-                        "status": encoded_data[0],
-                    }
-                )
+            result_data = call["method"].decode_data(encoded_data[1])
+            result = {
+                "result": result_data,
+                "hash": call["hash"],
+                "method": call["method"],
+                "address": call["address"],
+                "name": call["method"].name,
+                "inputs": call["inputs"],
+                "call_data": multicall_calls[index][1],
+                "block_number": block_number,
+                "block_timestamp": block_timestamp,
+                "block_hash": block_hash,
+                "status": encoded_data[0],
+                "v3": call.get("v3", False),
+                "customer_id": call.get("customer_id"),
+                "instance_id": call.get("instance_id"),
+            }
+            results.append(result)
         except Exception as e:
-            results.append(
-                {
-                    "result": str(encoded_data[1]),
-                    "hash": calls[index]["hash"],
-                    "method": calls[index]["method"],
-                    "address": calls[index]["address"],
-                    "name": calls[index]["method"].name,
-                    "inputs": calls[index]["inputs"],
-                    "call_data": multicall_calls[index][1],
-                    "block_number": block_number,
-                    "block_timestamp": block_timestamp,
-                    "status": encoded_data[0],
-                    "error": str(e),
-                }
-            )
-
+            result = {
+                "result": str(encoded_data[1]),
+                "hash": call["hash"],
+                "method": call["method"],
+                "address": call["address"],
+                "name": call["method"].name,
+                "inputs": call["inputs"],
+                "call_data": multicall_calls[index][1],
+                "block_number": block_number,
+                "block_timestamp": block_timestamp,
+                "block_hash": block_hash,
+                "status": encoded_data[0],
+                "error": str(e),
+                "v3": call.get("v3", False),
+                "customer_id": call.get("customer_id"),
+                "instance_id": call.get("instance_id"),
+            }
+            results.append(result)
             logger.error(
-                f"Error decoding data for for method {call['method'].name} call {calls[index]}: {e}."
+                f"Error decoding data for method {call['method'].name} call {call}: {e}."
             )
-            # data is not decoded, return the encoded data
             logger.error(f"Encoded data: {encoded_data}")
-
     return results
 
 
-def crawl_calls_level(
-    web3_client,
-    db_session,
-    calls,
-    responces,
-    contracts_ABIs,
-    interfaces,
-    batch_size,
-    multicall_method,
-    block_number,
-    blockchain_type,
-    block_timestamp,
-    max_batch_size=3000,
-    min_batch_size=4,
-):
-    calls_of_level = []
+def make_multicall(
+    multicall_method: Any,
+    calls: List[Dict[str, Any]],
+    block_timestamp: int,
+    block_number: str = "latest",
+    block_hash: Optional[str] = None,
+) -> List[Dict[str, Any]]:
+    """Makes a multicall to the blockchain and processes the results."""
+    multicall_calls = encode_calls(calls)
+    # breakpoint()
+    multicall_result = perform_multicall(
+        multicall_method, multicall_calls, block_number
+    )
+    results = process_multicall_result(
+        calls,
+        multicall_result,
+        multicall_calls,
+        block_timestamp,
+        block_number,
+        block_hash,
+    )
+    return results
 
+
+def generate_calls_of_level(
+    calls: List[Dict[str, Any]],
+    responses: Dict[str, Any],
+    contracts_ABIs: Dict[str, Any],
+    interfaces: Dict[str, Any],
+) -> List[Dict[str, Any]]:
+    """Generates the calls for the current level."""
+    calls_of_level = []
     for call in calls:
-        if call["generated_hash"] in responces:
+        if call["generated_hash"] in responses:
             continue
         parameters = []
-
         for input in call["inputs"]:
-            if type(input["value"]) in (str, int):
-                if input["value"] not in responces:
+            if isinstance(input["value"], (str, int)):
+                if input["value"] not in responses:
                     parameters.append([input["value"]])
                 else:
-                    if input["value"] in contracts_ABIs[call["address"]] and (
-                        contracts_ABIs[call["address"]][input["value"]]["name"]
+                    if (
+                        input["value"] in contracts_ABIs[call["address"]]
+                        and contracts_ABIs[call["address"]][input["value"]]["name"]
                         == "totalSupply"
-                    ):  # hack for totalSupply TODO(Andrey): need add propper support for response parsing
+                    ):
+                        # Hack for totalSupply
                         parameters.append(
-                            list(range(1, responces[input["value"]][0][0] + 1))
+                            list(range(1, responses[input["value"]][0][0] + 1))
                         )
                     else:
-                        parameters.append(responces[input["value"]])
-            elif type(input["value"]) == list:
+                        parameters.append(responses[input["value"]])
+            elif isinstance(input["value"], list):
                 parameters.append(input["value"])
             else:
-                raise
-
+                raise Exception("Unknown input value type")
         for call_parameters in itertools.product(*parameters):
-            # hack for tuples product
-            if len(call_parameters) == 1 and type(call_parameters[0]) == tuple:
+            if len(call_parameters) == 1 and isinstance(call_parameters[0], tuple):
                 call_parameters = call_parameters[0]
             calls_of_level.append(
                 {
@@ -239,21 +276,76 @@ def crawl_calls_level(
                     ),
                     "hash": call["generated_hash"],
                     "inputs": call_parameters,
+                    "v3": call.get("v3", False),
+                    "customer_id": call.get("customer_id"),
+                    "instance_id": call.get("instance_id"),
                 }
             )
+    return calls_of_level
 
+
+def process_results(
+    make_multicall_result: List[Dict[str, Any]],
+    db_sessions: Dict[Any, Any],
+    responses: Dict[str, Any],
+    blockchain_type: Any,
+) -> int:
+    """Processes the results and adds them to the appropriate database sessions."""
+    add_to_session_count = 0
+    sessions_to_commit = set()
+    for result in make_multicall_result:
+        v3 = result.get("v3", False)
+        if v3:
+            customer_id = result.get("customer_id")
+            instance_id = result.get("instance_id")
+            db_session = db_sessions.get((customer_id, instance_id))
+        else:
+            db_session = db_sessions.get("v2")
+        if db_session is None:
+            logger.error(f"No db_session found for result {result}")
+            continue
+        db_view = view_call_to_label(blockchain_type, result, v3)
+        db_session.add(db_view)
+        sessions_to_commit.add(db_session)
+        add_to_session_count += 1
+        if result["hash"] not in responses:
+            responses[result["hash"]] = []
+        responses[result["hash"]].append(result["result"])
+    # Commit all sessions
+    for session in sessions_to_commit:
+        commit_session(session)
+    logger.info(f"{add_to_session_count} labels committed to database.")
+    return add_to_session_count
+
+
+def crawl_calls_level(
+    web3_client: Web3,
+    db_sessions: Dict[Any, Any],
+    calls: List[Dict[str, Any]],
+    responses: Dict[str, Any],
+    contracts_ABIs: Dict[str, Any],
+    interfaces: Dict[str, Any],
+    batch_size: int,
+    multicall_method: Any,
+    block_number: str,
+    blockchain_type: Any,
+    block_timestamp: int,
+    max_batch_size: int = 3000,
+    min_batch_size: int = 4,
+    block_hash: Optional[str] = None,
+) -> int:
+    """Crawls calls at a specific level."""
+    calls_of_level = generate_calls_of_level(
+        calls, responses, contracts_ABIs, interfaces
+    )
     retry = 0
-
     while len(calls_of_level) > 0:
         make_multicall_result = []
         try:
             call_chunk = calls_of_level[:batch_size]
-
             logger.info(
                 f"Calling multicall2 with {len(call_chunk)} calls at block {block_number}"
             )
-
-            # 1 thead with timeout for hung multicall calls
             with ThreadPoolExecutor(max_workers=1) as executor:
                 future = executor.submit(
                     make_multicall,
@@ -261,251 +353,354 @@ def crawl_calls_level(
                     call_chunk,
                     block_timestamp,
                     block_number,
+                    block_hash,
                 )
                 make_multicall_result = future.result(timeout=20)
             retry = 0
             calls_of_level = calls_of_level[batch_size:]
-            logger.info(f"lenght of task left {len(calls_of_level)}.")
+            logger.info(f"Length of tasks left: {len(calls_of_level)}.")
             batch_size = min(batch_size * 2, max_batch_size)
-        except ValueError as e:  # missing trie node
+        except ValueError as e:
             logger.error(f"ValueError: {e}, retrying")
             retry += 1
             if "missing trie node" in str(e):
                 time.sleep(4)
             if retry > 5:
-                raise (e)
+                raise e
             batch_size = max(batch_size // 4, min_batch_size)
-        except TimeoutError as e:  # timeout
+        except TimeoutError as e:
             logger.error(f"TimeoutError: {e}, retrying")
             retry += 1
             if retry > 5:
-                raise (e)
+                raise e
             batch_size = max(batch_size // 3, min_batch_size)
         except Exception as e:
             logger.error(f"Exception: {e}")
-            raise (e)
+            raise e
         time.sleep(2)
         logger.debug(f"Retry: {retry}")
-        # results parsing and writing to database
-        add_to_session_count = 0
-        for result in make_multicall_result:
-            db_view = view_call_to_label(blockchain_type, result)
-            db_session.add(db_view)
-            add_to_session_count += 1
-
-            if result["hash"] not in responces:
-                responces[result["hash"]] = []
-            responces[result["hash"]].append(result["result"])
-        commit_session(db_session)
-        logger.info(f"{add_to_session_count} labels commit to database.")
-
+        process_results(make_multicall_result, db_sessions, responses, blockchain_type)
     return batch_size
 
 
-def parse_jobs(
-    jobs: List[Any],
-    blockchain_type: AvailableBlockchainType,
+def connect_to_web3(
+    blockchain_type: Any,
     web3_provider_uri: Optional[str],
-    block_number: Optional[int],
-    batch_size: int,
-    moonstream_token: str,
-    web3_uri: Optional[str] = None,
-):
-    """
-    Parse jobs from list and generate web3 interfaces for each contract.
-    """
-
-    contracts_ABIs: Dict[str, Any] = {}
-    contracts_methods: Dict[str, Any] = {}
-    calls: Dict[int, Any] = {0: []}
-    responces: Dict[str, Any] = {}
-
+    web3_uri: Optional[str],
+) -> Web3:
+    """Connects to the Web3 client."""
     if web3_provider_uri is not None:
         try:
             logger.info(
                 f"Connecting to blockchain: {blockchain_type} with custom provider!"
             )
-
             web3_client = connect(
                 blockchain_type=blockchain_type, web3_uri=web3_provider_uri
             )
         except Exception as e:
             logger.error(
-                f"Web3 connection to custom provider {web3_provider_uri} failed error: {e}"
+                f"Web3 connection to custom provider {web3_provider_uri} failed. Error: {e}"
             )
-            raise (e)
+            raise e
     else:
-        logger.info(f"Connecting to blockchain: {blockchain_type} with Node balancer.")
+        logger.info(f"Connecting to blockchain: {blockchain_type} with node balancer.")
         web3_client = _retry_connect_web3(
             blockchain_type=blockchain_type, web3_uri=web3_uri
         )
-
     logger.info(f"Crawler started connected to blockchain: {blockchain_type}")
+    return web3_client
 
+
+def get_block_info(web3_client: Web3, block_number: Optional[int]) -> tuple:
+    """Retrieves block information."""
     if block_number is None:
         block_number = web3_client.eth.get_block("latest").number  # type: ignore
-
     logger.info(f"Current block number: {block_number}")
+    block = web3_client.eth.get_block(block_number)  # type: ignore
+    block_timestamp = block.timestamp  # type: ignore
+    block_hash = block.hash.hex()  # type: ignore
+    return block_number, block_timestamp, block_hash
 
-    block_timestamp = web3_client.eth.get_block(block_number).timestamp  # type: ignore
+
+def recursive_unpack(
+    method_abi: Any,
+    level: int,
+    calls: Dict[int, List[Any]],
+    contracts_methods: Dict[str, Any],
+    contracts_ABIs: Dict[str, Any],
+    responses: Dict[str, Any],
+    moonstream_token: str,
+    v3: bool,
+    customer_id: Optional[str] = None,
+    instance_id: Optional[str] = None,
+) -> str:
+    """Recursively unpacks method ABIs to generate a tree of calls."""
+    have_subcalls = False
+    if method_abi["type"] == "queryAPI":
+        # Make queryAPI call
+        response = execute_query(method_abi, token=moonstream_token)
+        # Generate hash for queryAPI call
+        generated_hash = hashlib.md5(
+            json.dumps(
+                method_abi,
+                sort_keys=True,
+                indent=4,
+                separators=(",", ": "),
+            ).encode("utf-8")
+        ).hexdigest()
+        # Add response to responses
+        responses[generated_hash] = response
+        return generated_hash
+
+    abi = {
+        "inputs": [],
+        "outputs": method_abi["outputs"],
+        "name": method_abi["name"],
+        "type": "function",
+        "stateMutability": "view",
+        "v3": v3,
+        "customer_id": customer_id,
+        "instance_id": instance_id,
+    }
+
+    for input in method_abi["inputs"]:
+        if isinstance(input["value"], (int, list, str)):
+            abi["inputs"].append(input)
+        elif isinstance(input["value"], dict):
+            if input["value"]["type"] in ["function", "queryAPI"]:
+                hash_link = recursive_unpack(
+                    input["value"],
+                    level + 1,
+                    calls,
+                    contracts_methods,
+                    contracts_ABIs,
+                    responses,
+                    moonstream_token,
+                    v3,
+                    customer_id,
+                    instance_id,
+                )
+                input["value"] = hash_link
+                have_subcalls = True
+            abi["inputs"].append(input)
+
+    abi["address"] = method_abi["address"]
+    generated_hash = hashlib.md5(
+        json.dumps(abi, sort_keys=True, indent=4, separators=(",", ": ")).encode(
+            "utf-8"
+        )
+    ).hexdigest()
+    abi["generated_hash"] = generated_hash
+
+    if have_subcalls:
+        level += 1
+        calls.setdefault(level, []).append(abi)
+    else:
+        level = 0
+        calls.setdefault(level, []).append(abi)
+
+    contracts_methods.setdefault(method_abi["address"], [])
+    if generated_hash not in contracts_methods[method_abi["address"]]:
+        contracts_methods[method_abi["address"]].append(generated_hash)
+        contracts_ABIs.setdefault(method_abi["address"], {})
+        contracts_ABIs[method_abi["address"]][generated_hash] = abi
+
+    return generated_hash
+
+
+def build_interfaces(
+    contracts_ABIs: Dict[str, Any], contracts_methods: Dict[str, Any], web3_client: Web3
+) -> Dict[str, Any]:
+    """Builds contract interfaces with deduplication of ABIs."""
+    interfaces = {}
+    for contract_address in contracts_ABIs:
+        # Use a dictionary to deduplicate ABIs by function signature
+        unique_abis = {}
+        for method_hash in contracts_methods[contract_address]:
+            abi = contracts_ABIs[contract_address][method_hash]
+            # Create a unique key based on name and input types
+            if abi["name"] not in unique_abis:
+                unique_abis[abi["name"]] = abi
+
+        interfaces[contract_address] = web3_client.eth.contract(
+            address=web3_client.toChecksumAddress(contract_address),
+            abi=list(unique_abis.values())
+        )
+    return interfaces
+
+
+def process_address_field(job: Dict[str, Any], moonstream_token: str) -> List[str]:
+    """Processes the address field of a job and returns a list of addresses."""
+    if isinstance(job["address"], str):
+        return [Web3.toChecksumAddress(job["address"])]
+    elif isinstance(job["address"], list):
+        return [
+            Web3.toChecksumAddress(address) for address in job["address"]
+        ]  # manual job multiplication
+    elif isinstance(job["address"], dict):
+        if job["address"].get("type") == "queryAPI":
+            # QueryAPI job multiplication
+            addresses = execute_query(job["address"], token=moonstream_token)
+            checsum_addresses = []
+            for address in addresses:
+                try:
+                    checsum_addresses.append(Web3.toChecksumAddress(address))
+                except Exception as e:
+                    logger.error(f"Invalid address: {address}")
+                    continue
+            return checsum_addresses
+        else:
+            raise ValueError(f"Invalid address type: {type(job['address'])}")
+    else:
+        raise ValueError(f"Invalid address type: {type(job['address'])}")
+
+
+def parse_jobs(
+    jobs: List[Any],
+    blockchain_type: Any,
+    web3_provider_uri: Optional[str],
+    block_number: Optional[int],
+    batch_size: int,
+    moonstream_token: str,
+    web3_uri: Optional[str] = None,
+    customer_db_uri: Optional[str] = None,
+):
+    """
+    Parses jobs from a list and generates web3 interfaces for each contract.
+    """
+    contracts_ABIs: Dict[str, Any] = {}
+    contracts_methods: Dict[str, Any] = {}
+    calls: Dict[int, List[Any]] = {0: []}
+    responses: Dict[str, Any] = {}
+    db_sessions: Dict[Any, Any] = {}
+
+    web3_client = connect_to_web3(blockchain_type, web3_provider_uri, web3_uri)
+    block_number, block_timestamp, block_hash = get_block_info(
+        web3_client, block_number
+    )
 
     multicaller = Multicall2(
         web3_client, web3_client.toChecksumAddress(multicall_contracts[blockchain_type])
     )
-
     multicall_method = multicaller.tryAggregate
 
-    def recursive_unpack(method_abi: Any, level: int = 0) -> Any:
-        """
-        Generate tree of calls for crawling
-        """
-        have_subcalls = False
-
-        ### we add queryAPI to that tree
-
-        if method_abi["type"] == "queryAPI":
-            # make queryAPI call
-
-            responce = execute_query(method_abi, token=moonstream_token)
-
-            # generate hash for queryAPI call
-
-            generated_hash = hashlib.md5(
-                json.dumps(
-                    method_abi,
-                    sort_keys=True,
-                    indent=4,
-                    separators=(",", ": "),
-                ).encode("utf-8")
-            ).hexdigest()
-
-            # add responce to responces
-
-            responces[generated_hash] = responce
-
-            return generated_hash
-
-        abi = {
-            "inputs": [],
-            "outputs": method_abi["outputs"],
-            "name": method_abi["name"],
-            "type": "function",
-            "stateMutability": "view",
-        }
-
-        for input in method_abi["inputs"]:
-            if type(input["value"]) in (int, list):
-                abi["inputs"].append(input)
-
-            elif type(input["value"]) == str:
-                abi["inputs"].append(input)
-
-            elif type(input["value"]) == dict:
-                if input["value"]["type"] == "function":
-                    hash_link = recursive_unpack(input["value"], level + 1)
-                    # replace defenition by hash pointing to the result of the recursive_unpack
-                    input["value"] = hash_link
-                    have_subcalls = True
-                elif input["value"]["type"] == "queryAPI":
-                    input["value"] = recursive_unpack(input["value"], level + 1)
-                    have_subcalls = True
-                abi["inputs"].append(input)
-        abi["address"] = method_abi["address"]
-        generated_hash = hashlib.md5(
-            json.dumps(abi, sort_keys=True, indent=4, separators=(",", ": ")).encode(
-                "utf-8"
-            )
-        ).hexdigest()
-
-        abi["generated_hash"] = generated_hash
-        if have_subcalls:
-            level += 1
-            if not calls.get(level):
-                calls[level] = []
-            calls[level].append(abi)
-        else:
-            level = 0
-
-            if not calls.get(level):
-                calls[level] = []
-            calls[level].append(abi)
-
-        if not contracts_methods.get(job["address"]):
-            contracts_methods[job["address"]] = []
-        if generated_hash not in contracts_methods[job["address"]]:
-            contracts_methods[job["address"]].append(generated_hash)
-            if not contracts_ABIs.get(job["address"]):
-                contracts_ABIs[job["address"]] = {}
-            contracts_ABIs[job["address"]][generated_hash] = abi
-
-        return generated_hash
-
-    for job in jobs:
-        if job["address"] not in contracts_ABIs:
-            contracts_ABIs[job["address"]] = []
-
-        recursive_unpack(job, 0)
-
-    # generate contracts interfaces
-
-    interfaces = {}
-
-    for contract_address in contracts_ABIs:
-        # collect abis for each contract
-        abis = []
-
-        for method_hash in contracts_methods[contract_address]:
-            abis.append(contracts_ABIs[contract_address][method_hash])
-
-        # generate interface
-        interfaces[contract_address] = web3_client.eth.contract(
-            address=web3_client.toChecksumAddress(contract_address), abi=abis
-        )
-
-    # reverse call_tree
-    call_tree_levels = sorted(calls.keys(), reverse=True)[:-1]
-
-    db_session = PrePing_SessionLocal()
-
-    # run crawling of levels
+    # All sessions are stored in the dictionary db_sessions
+    # Under one try block
     try:
-        # initial call of level 0 all call without subcalls directly moved there
+        # Process jobs and create session
+
+        for job in jobs:
+
+            ### process address field
+            ### Handle case when 1 job represents multiple contracts
+            addresses = process_address_field(job, moonstream_token)
+
+            for address in addresses[1:]:
+                new_job = job.copy()
+                new_job["address"] = address
+                jobs.append(new_job)
+
+            job["address"] = addresses[0]
+
+            v3 = job.get("v3", False)
+            customer_id = job.get("customer_id")
+            instance_id = job.get("instance_id")
+
+            ### DB sessions
+            if customer_db_uri is not None:
+                if v3 and (customer_id, instance_id) not in db_sessions:
+                    # Create session
+                    engine = create_moonstream_engine(customer_db_uri, 2, 100000)
+                    session = sessionmaker(bind=engine)
+                    try:
+                        db_sessions[(customer_id, instance_id)] = session()
+                    except Exception as e:
+                        logger.error(f"Connection to {engine} failed: {e}")
+                        continue
+                else:
+                    if "v2" not in db_sessions:
+                        engine = create_moonstream_engine(customer_db_uri, 2, 100000)
+                        db_sessions["v2"] = sessionmaker(bind=engine)()
+            elif v3:
+                if (customer_id, instance_id) not in db_sessions:
+                    # Create session
+                    # Assume fetch_connection_string fetches the connection string
+                    connection_string = request_connection_string(
+                        customer_id=customer_id,
+                        instance_id=instance_id,
+                        token=moonstream_token,
+                    )
+                    engine = create_moonstream_engine(connection_string, 2, 100000)
+                    session = sessionmaker(bind=engine)
+                    try:
+                        db_sessions[(customer_id, instance_id)] = session()
+                    except Exception as e:
+                        logger.error(f"Connection to {engine} failed: {e}")
+                        continue
+            else:
+                if "v2" not in db_sessions:
+                    db_sessions["v2"] = PrePing_SessionLocal()
+
+            if job["address"] not in contracts_ABIs:
+                contracts_ABIs[job["address"]] = {}
+
+            recursive_unpack(
+                job,
+                0,
+                calls,
+                contracts_methods,
+                contracts_ABIs,
+                responses,
+                moonstream_token,
+                v3,
+                customer_id,
+                instance_id,
+            )
+        interfaces = build_interfaces(contracts_ABIs, contracts_methods, web3_client)
+
+        call_tree_levels = sorted(calls.keys(), reverse=True)[:-1]
+
         logger.info(f"Crawl level: 0. Jobs amount: {len(calls[0])}")
-        logger.info(f"call_tree_levels: {call_tree_levels}")
+        logger.info(f"Call tree levels: {call_tree_levels}")
 
         batch_size = crawl_calls_level(
-            web3_client,
-            db_session,
-            calls[0],
-            responces,
-            contracts_ABIs,
-            interfaces,
-            batch_size,
-            multicall_method,
-            block_number,
-            blockchain_type,
-            block_timestamp,
+            web3_client=web3_client,
+            db_sessions=db_sessions,
+            calls=calls[0],
+            responses=responses,
+            contracts_ABIs=contracts_ABIs,
+            interfaces=interfaces,
+            batch_size=batch_size,
+            multicall_method=multicall_method,
+            block_number=block_number,  # type: ignore
+            blockchain_type=blockchain_type,
+            block_timestamp=block_timestamp,
+            block_hash=block_hash,
         )
 
         for level in call_tree_levels:
             logger.info(f"Crawl level: {level}. Jobs amount: {len(calls[level])}")
-
             batch_size = crawl_calls_level(
-                web3_client,
-                db_session,
-                calls[level],
-                responces,
-                contracts_ABIs,
-                interfaces,
-                batch_size,
-                multicall_method,
-                block_number,
-                blockchain_type,
-                block_timestamp,
+                web3_client=web3_client,
+                db_sessions=db_sessions,
+                calls=calls[level],
+                responses=responses,
+                contracts_ABIs=contracts_ABIs,
+                interfaces=interfaces,
+                batch_size=batch_size,
+                multicall_method=multicall_method,
+                block_number=block_number,  # type: ignore
+                blockchain_type=blockchain_type,
+                block_timestamp=block_timestamp,
+                block_hash=block_hash,
             )
-
     finally:
-        db_session.close()
+        # Close all sessions
+        for session in db_sessions.values():
+            try:
+                session.close()
+            except Exception as e:
+                logger.error(f"Failed to close session: {e}")
 
 
 def handle_crawl(args: argparse.Namespace) -> None:
@@ -576,6 +771,7 @@ def handle_crawl(args: argparse.Namespace) -> None:
         args.batch_size,
         args.moonstream_token,
         args.web3_uri,
+        args.customer_db_uri,
     )
 
 
@@ -762,6 +958,11 @@ def main() -> None:
         default=500,
         help="Size of chunks wich send to Multicall2 contract.",
     )
+    view_state_crawler_parser.add_argument(
+        "--customer-db-uri",
+        type=str,
+        help="URI for the customer database",
+    )
     view_state_crawler_parser.set_defaults(func=handle_crawl)
 
     view_state_migration_parser = subparsers.add_parser(
@@ -824,4 +1025,4 @@ def main() -> None:
 
 
 if __name__ == "__main__":
-    main()
+    main()

crawlers/mooncrawl/mooncrawl/state_crawler/db.py

@@ -1,6 +1,8 @@
 import json
 import logging
 from typing import Any, Dict
+from hexbytes import HexBytes
+
 
 from moonstreamtypes.blockchain import AvailableBlockchainType, get_label_model
 from sqlalchemy.orm import Session
@@ -14,13 +16,15 @@ logger = logging.getLogger(__name__)
 def view_call_to_label(
     blockchain_type: AvailableBlockchainType,
     call: Dict[str, Any],
+    v3: bool = False,
     label_name=VIEW_STATE_CRAWLER_LABEL,
 ):
     """
     Creates a label model.
 
     """
-    label_model = get_label_model(blockchain_type)
+    version = 3 if v3 else 2
+    label_model = get_label_model(blockchain_type, version=version)
 
     sanityzed_label_data = json.loads(
         json.dumps(
@@ -35,14 +39,35 @@ def view_call_to_label(
         ).replace(r"\u0000", "")
     )
 
-    label = label_model(
-        label=label_name,
-        label_data=sanityzed_label_data,
-        address=call["address"],
-        block_number=call["block_number"],
-        transaction_hash=None,
-        block_timestamp=call["block_timestamp"],
-    )
+    if v3:
+
+        del sanityzed_label_data["type"]
+        del sanityzed_label_data["name"]
+
+        ## add zero transaction hash
+
+        label = label_model(
+            label=label_name,
+            label_name=call["name"],
+            label_type="view",
+            label_data=sanityzed_label_data,
+            ### bytea
+            address=HexBytes(call["address"]),
+            block_number=call["block_number"],
+            block_timestamp=call["block_timestamp"],
+            block_hash=call["block_hash"],
+        )
+
+    else:
+
+        label = label_model(
+            label=label_name,
+            label_data=sanityzed_label_data,
+            address=call["address"],
+            block_number=call["block_number"],
+            transaction_hash=None,
+            block_timestamp=call["block_timestamp"],
+        )
 
     return label
 

crawlers/mooncrawl/mooncrawl/version.py

@@ -2,4 +2,4 @@
 Moonstream crawlers version.
 """
 
-MOONCRAWL_VERSION = "0.5.1"
+MOONCRAWL_VERSION = "0.5.4"

crawlers/mooncrawl/setup.py

@@ -38,8 +38,8 @@ setup(
         "chardet",
         "fastapi",
         "moonstreamdb>=0.4.6",
-        "moonstreamdb-v3>=0.1.3",
-        "moonstream-types>=0.0.10",
+        "moonstreamdb-v3>=0.1.4",
+        "moonstream-types>=0.0.11",
         "moonstream>=0.1.2",
         "moonworm[moonstream]>=0.9.3",
         "humbug",

moonstreamapi/moonstreamapi/actions.py

@@ -611,39 +611,18 @@ def delete_seer_subscription(
 ) -> None:
     """
     Delete seer subscription from db
-    If there are no more subscriptions for this address,abi_selector delete all abis
     """
-
-    ## Delete subscription from db
-
+    ### TEMPORARY disable deleting abi jobs from db
+    ### TODO(ANDREY): fix this
     try:
-        db_session.query(AbiSubscriptions).filter(
+        db_session.query(AbiSubscriptions).filter(  
             AbiSubscriptions.subscription_id == subscription_id
         ).delete(synchronize_session=False)
         db_session.commit()
     except Exception as e:
-        logger.error(f"Error delete subscription from db: {str(e)}")
+        logger.error(f"Error deleting subscription from db: {str(e)}")
         db_session.rollback()
-
-    not_connected_abi_jobs = (
-        db_session.query(AbiJobs)
-        .join(AbiSubscriptions, AbiJobs.id == AbiSubscriptions.abi_job_id, isouter=True)
-        .filter(AbiSubscriptions.subscription_id == None)
-        .cte("not_connected_abi_jobs")
-    )
-
-    ## Delete abi jobs from db
-
-    try:
-        db_session.query(AbiJobs).filter(
-            AbiJobs.id.in_(db_session.query(not_connected_abi_jobs.c.id))
-        ).delete(synchronize_session=False)
-
-        db_session.commit()
-    except Exception as e:
-        logger.error(f"Error delete abi jobs from db: {str(e)}")
-        db_session.rollback()
-
+        return
 
 def add_abi_to_db(
     db_session: Session,

moonstreamapi/moonstreamapi/routes/queries.py

@@ -16,7 +16,7 @@ from bugout.data import (
 )
 from bugout.exceptions import BugoutResponseException
 from fastapi import APIRouter, Body, Path, Query, Request
-from moonstreamdb.blockchain import AvailableBlockchainType
+from moonstreamtypes.blockchain import AvailableBlockchainType
 from sqlalchemy import text
 
 from .. import data

moonstreamapi/moonstreamapi/version.py

@@ -2,4 +2,4 @@
 Moonstream library and API version.
 """
 
-MOONSTREAMAPI_VERSION = "0.4.11"
+MOONSTREAMAPI_VERSION = "0.4.12"

moonstreamapi/requirements.txt

@@ -38,7 +38,8 @@ Mako==1.2.3
 MarkupSafe==2.1.1
 moonstream==0.1.2
 moonstreamdb==0.4.6
-moonstreamdb-v3==0.1.3
+moonstreamdb-v3==0.1.4
+moonstream-types==0.0.11
 multiaddr==0.0.9
 multidict==6.0.2
 netaddr==0.8.0

moonstreamapi/setup.py

@@ -18,6 +18,7 @@ setup(
         "moonstream>=0.1.2",
         "moonstreamdb>=0.4.6",
         "moonstreamdb-v3>=0.1.3",
+        "moonstream-types>=0.0.11",
         "humbug",
         "pydantic==1.10.2",
         "pyevmasm",

moonstreamdb-v3/moonstreamdbv3/alembic/versions/bebe640146b6_add_raw_transactions.py

@@ -1,8 +1,8 @@
 """Add raw transactions
 
-Revision ID: 3ea877d6e22d
-Revises: dae735afc98f
-Create Date: 2025-01-16 12:47:48.572482
+Revision ID: bebe640146b6
+Revises: c79044e047fe
+Create Date: 2025-03-04 00:34:20.551263
 
 """
 from typing import Sequence, Union
@@ -12,8 +12,8 @@ import sqlalchemy as sa
 from sqlalchemy.dialects import postgresql
 
 # revision identifiers, used by Alembic.
-revision: str = '3ea877d6e22d'
-down_revision: Union[str, None] = 'dae735afc98f'
+revision: str = 'bebe640146b6'
+down_revision: Union[str, None] = 'c79044e047fe'
 branch_labels: Union[str, Sequence[str], None] = None
 depends_on: Union[str, Sequence[str], None] = None
 
@@ -869,7 +869,7 @@ def upgrade() -> None:
     op.create_index(op.f('ix_zksync_era_transactions_hash'), 'zksync_era_transactions', ['hash'], unique=True)
     op.create_index(op.f('ix_zksync_era_transactions_to_address'), 'zksync_era_transactions', ['to_address'], unique=False)
     op.create_index(op.f('ix_zksync_era_transactions_value'), 'zksync_era_transactions', ['value'], unique=False)
-   # ### end Alembic commands ###
+    # ### end Alembic commands ###
 
 
 def downgrade() -> None:

moonstreamdb-v3/moonstreamdbv3/alembic/versions/c79044e047fe_declarative_indexes.py

@@ -0,0 +1,147 @@
+"""declarative indexes
+
+Revision ID: c79044e047fe
+Revises: dae735afc98f
+Create Date: 2025-02-10 13:43:05.099092
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision: str = 'c79044e047fe'
+down_revision: Union[str, None] = 'dae735afc98f'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+  
+    op.create_index('ix_amoy_labels_label_addr_name', 'amoy_labels', ['label', 'address', 'label_name'], unique=False)
+    op.create_index('ix_arbitrum_nova_labels_label_addr_name', 'arbitrum_nova_labels', ['label', 'address', 'label_name'], unique=False)
+    op.create_index('ix_arbitrum_one_labels_label_addr_name', 'arbitrum_one_labels', ['label', 'address', 'label_name'], unique=False)
+    op.create_index('ix_arbitrum_sepolia_labels_addr_block_ts', 'arbitrum_sepolia_labels', ['address', 'block_timestamp'], unique=False)
+    op.create_index('ix_arbitrum_sepolia_labels_label_addr_name', 'arbitrum_sepolia_labels', ['label', 'address', 'label_name'], unique=False)
+    op.create_index('ix_avalanche_fuji_labels_addr_block_ts', 'avalanche_fuji_labels', ['address', 'block_timestamp'], unique=False)
+    op.create_index('ix_avalanche_fuji_labels_label_addr_name', 'avalanche_fuji_labels', ['label', 'address', 'label_name'], unique=False)
+    op.create_index('ix_avalanche_labels_addr_block_ts', 'avalanche_labels', ['address', 'block_timestamp'], unique=False)
+    op.create_index('ix_avalanche_labels_label_addr_name', 'avalanche_labels', ['label', 'address', 'label_name'], unique=False)
+    op.create_index('ix_b3_labels_label_addr_name', 'b3_labels', ['label', 'address', 'label_name'], unique=False)
+    op.create_index('ix_b3_sepolia_labels_label_addr_name', 'b3_sepolia_labels', ['label', 'address', 'label_name'], unique=False)
+    op.create_index('ix_base_labels_addr_block_ts', 'base_labels', ['address', 'block_timestamp'], unique=False)
+    op.create_index('ix_base_labels_label_addr_name', 'base_labels', ['label', 'address', 'label_name'], unique=False)
+    op.create_index('ix_blast_labels_label_addr_name', 'blast_labels', ['label', 'address', 'label_name'], unique=False)
+    op.create_index('ix_blast_sepolia_labels_label_addr_name', 'blast_sepolia_labels', ['label', 'address', 'label_name'], unique=False)
+    op.create_index('ix_ethereum_labels_addr_block_ts', 'ethereum_labels', ['address', 'block_timestamp'], unique=False)
+    op.create_index('ix_ethereum_labels_label_addr_name', 'ethereum_labels', ['label', 'address', 'label_name'], unique=False)
+    op.create_index('ix_game7_labels_label_addr_name', 'game7_labels', ['label', 'address', 'label_name'], unique=False)
+    op.drop_index('ix_g7o_arbitrum_sepolia_labels_addr_block_num', table_name='game7_orbit_arbitrum_sepolia_labels')
+    op.drop_index('ix_g7o_arbitrum_sepolia_labels_addr_block_ts', table_name='game7_orbit_arbitrum_sepolia_labels')
+    op.drop_index('uk_g7o_arbitrum_sepolia_labels_tx_hash_log_idx_evt', table_name='game7_orbit_arbitrum_sepolia_labels', postgresql_where="(((label)::text = 'seer'::text) AND ((label_type)::text = 'event'::text))")
+    op.drop_index('uk_g7o_arbitrum_sepolia_labels_tx_hash_log_idx_evt_raw', table_name='game7_orbit_arbitrum_sepolia_labels', postgresql_where="(((label)::text = 'seer-raw'::text) AND ((label_type)::text = 'event'::text))")
+    op.drop_index('uk_g7o_arbitrum_sepolia_labels_tx_hash_tx_call', table_name='game7_orbit_arbitrum_sepolia_labels', postgresql_where="(((label)::text = 'seer'::text) AND ((label_type)::text = 'tx_call'::text))")
+    op.drop_index('uk_g7o_arbitrum_sepolia_labels_tx_hash_tx_call_raw', table_name='game7_orbit_arbitrum_sepolia_labels', postgresql_where="(((label)::text = 'seer-raw'::text) AND ((label_type)::text = 'tx_call'::text))")
+    op.create_index('ix_game7_orbit_arbitrum_sepolia_labels_addr_block_num', 'game7_orbit_arbitrum_sepolia_labels', ['address', 'block_number'], unique=False)
+    op.create_index('ix_game7_orbit_arbitrum_sepolia_labels_addr_block_ts', 'game7_orbit_arbitrum_sepolia_labels', ['address', 'block_timestamp'], unique=False)
+    op.create_index('ix_game7_orbit_arbitrum_sepolia_labels_label_addr_name', 'game7_orbit_arbitrum_sepolia_labels', ['label', 'address', 'label_name'], unique=False)
+    op.create_index('uk_game7_orbit_arbitrum_sepolia_labels_tx_hash_log_idx_evt', 'game7_orbit_arbitrum_sepolia_labels', ['transaction_hash', 'log_index'], unique=True, postgresql_where=sa.text("label='seer' and label_type='event'"))
+    op.create_index('uk_game7_orbit_arbitrum_sepolia_labels_tx_hash_log_idx_evt_raw', 'game7_orbit_arbitrum_sepolia_labels', ['transaction_hash', 'log_index'], unique=True, postgresql_where=sa.text("label='seer-raw' and label_type='event'"))
+    op.create_index('uk_game7_orbit_arbitrum_sepolia_labels_tx_hash_tx_call', 'game7_orbit_arbitrum_sepolia_labels', ['transaction_hash'], unique=True, postgresql_where=sa.text("label='seer' and label_type='tx_call'"))
+    op.create_index('uk_game7_orbit_arbitrum_sepolia_labels_tx_hash_tx_call_raw', 'game7_orbit_arbitrum_sepolia_labels', ['transaction_hash'], unique=True, postgresql_where=sa.text("label='seer-raw' and label_type='tx_call'"))
+    op.create_index('ix_game7_testnet_labels_label_addr_name', 'game7_testnet_labels', ['label', 'address', 'label_name'], unique=False)
+    op.create_index('ix_imx_zkevm_labels_label_addr_name', 'imx_zkevm_labels', ['label', 'address', 'label_name'], unique=False)
+    op.create_index('ix_imx_zkevm_sepolia_labels_label_addr_name', 'imx_zkevm_sepolia_labels', ['label', 'address', 'label_name'], unique=False)
+    op.create_index('ix_mantle_labels_label_addr_name', 'mantle_labels', ['label', 'address', 'label_name'], unique=False)
+    op.create_index('ix_mantle_sepolia_labels_label_addr_name', 'mantle_sepolia_labels', ['label', 'address', 'label_name'], unique=False)
+    op.create_index('ix_mumbai_labels_addr_block_ts', 'mumbai_labels', ['address', 'block_timestamp'], unique=False)
+    op.create_index('ix_mumbai_labels_label_addr_name', 'mumbai_labels', ['label', 'address', 'label_name'], unique=False)
+    op.create_index('ix_polygon_labels_addr_block_ts', 'polygon_labels', ['address', 'block_timestamp'], unique=False)
+    op.create_index('ix_polygon_labels_label_addr_name', 'polygon_labels', ['label', 'address', 'label_name'], unique=False)
+    op.create_index('ix_proofofplay_apex_labels_label_addr_name', 'proofofplay_apex_labels', ['label', 'address', 'label_name'], unique=False)
+    op.create_index('ix_ronin_labels_label_addr_name', 'ronin_labels', ['label', 'address', 'label_name'], unique=False)
+    op.create_index('ix_ronin_saigon_labels_label_addr_name', 'ronin_saigon_labels', ['label', 'address', 'label_name'], unique=False)
+    op.create_index('ix_sepolia_labels_addr_block_ts', 'sepolia_labels', ['address', 'block_timestamp'], unique=False)
+    op.create_index('ix_sepolia_labels_label_addr_name', 'sepolia_labels', ['label', 'address', 'label_name'], unique=False)
+    op.create_index('ix_starknet_labels_addr_block_ts', 'starknet_labels', ['address', 'block_timestamp'], unique=False)
+    op.create_index('ix_starknet_labels_label_addr_name', 'starknet_labels', ['label', 'address', 'label_name'], unique=False)
+    op.create_index('ix_starknet_sepolia_labels_addr_block_ts', 'starknet_sepolia_labels', ['address', 'block_timestamp'], unique=False)
+    op.create_index('ix_starknet_sepolia_labels_label_addr_name', 'starknet_sepolia_labels', ['label', 'address', 'label_name'], unique=False)
+    op.create_index('ix_xai_labels_addr_block_ts', 'xai_labels', ['address', 'block_timestamp'], unique=False)
+    op.create_index('ix_xai_labels_label_addr_name', 'xai_labels', ['label', 'address', 'label_name'], unique=False)
+    op.create_index('ix_xai_sepolia_labels_addr_block_ts', 'xai_sepolia_labels', ['address', 'block_timestamp'], unique=False)
+    op.create_index('ix_xai_sepolia_labels_label_addr_name', 'xai_sepolia_labels', ['label', 'address', 'label_name'], unique=False)
+    op.create_index('ix_xdai_labels_addr_block_ts', 'xdai_labels', ['address', 'block_timestamp'], unique=False)
+    op.create_index('ix_xdai_labels_label_addr_name', 'xdai_labels', ['label', 'address', 'label_name'], unique=False)
+    op.create_index('ix_zksync_era_labels_addr_block_ts', 'zksync_era_labels', ['address', 'block_timestamp'], unique=False)
+    op.create_index('ix_zksync_era_labels_label_addr_name', 'zksync_era_labels', ['label', 'address', 'label_name'], unique=False)
+    op.create_index('ix_zksync_era_sepolia_labels_addr_block_ts', 'zksync_era_sepolia_labels', ['address', 'block_timestamp'], unique=False)
+    op.create_index('ix_zksync_era_sepolia_labels_label_addr_name', 'zksync_era_sepolia_labels', ['label', 'address', 'label_name'], unique=False)
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index('ix_zksync_era_sepolia_labels_label_addr_name', table_name='zksync_era_sepolia_labels')
+    op.drop_index('ix_zksync_era_sepolia_labels_addr_block_ts', table_name='zksync_era_sepolia_labels')
+    op.drop_index('ix_zksync_era_labels_label_addr_name', table_name='zksync_era_labels')
+    op.drop_index('ix_zksync_era_labels_addr_block_ts', table_name='zksync_era_labels')
+    op.drop_index('ix_xdai_labels_label_addr_name', table_name='xdai_labels')
+    op.drop_index('ix_xdai_labels_addr_block_ts', table_name='xdai_labels')
+    op.drop_index('ix_xai_sepolia_labels_label_addr_name', table_name='xai_sepolia_labels')
+    op.drop_index('ix_xai_sepolia_labels_addr_block_ts', table_name='xai_sepolia_labels')
+    op.drop_index('ix_xai_labels_label_addr_name', table_name='xai_labels')
+    op.drop_index('ix_xai_labels_addr_block_ts', table_name='xai_labels')
+    op.drop_index('ix_starknet_sepolia_labels_label_addr_name', table_name='starknet_sepolia_labels')
+    op.drop_index('ix_starknet_sepolia_labels_addr_block_ts', table_name='starknet_sepolia_labels')
+    op.drop_index('ix_starknet_labels_label_addr_name', table_name='starknet_labels')
+    op.drop_index('ix_starknet_labels_addr_block_ts', table_name='starknet_labels')
+    op.drop_index('ix_sepolia_labels_label_addr_name', table_name='sepolia_labels')
+    op.drop_index('ix_sepolia_labels_addr_block_ts', table_name='sepolia_labels')
+    op.drop_index('ix_ronin_saigon_labels_label_addr_name', table_name='ronin_saigon_labels')
+    op.drop_index('ix_ronin_labels_label_addr_name', table_name='ronin_labels')
+    op.drop_index('ix_proofofplay_apex_labels_label_addr_name', table_name='proofofplay_apex_labels')
+    op.drop_index('ix_polygon_labels_label_addr_name', table_name='polygon_labels')
+    op.drop_index('ix_polygon_labels_addr_block_ts', table_name='polygon_labels')
+    op.drop_index('ix_mumbai_labels_label_addr_name', table_name='mumbai_labels')
+    op.drop_index('ix_mumbai_labels_addr_block_ts', table_name='mumbai_labels')
+    op.drop_index('ix_mantle_sepolia_labels_label_addr_name', table_name='mantle_sepolia_labels')
+    op.drop_index('ix_mantle_labels_label_addr_name', table_name='mantle_labels')
+    op.drop_index('ix_imx_zkevm_sepolia_labels_label_addr_name', table_name='imx_zkevm_sepolia_labels')
+    op.drop_index('ix_imx_zkevm_labels_label_addr_name', table_name='imx_zkevm_labels')
+    op.drop_index('ix_game7_testnet_labels_label_addr_name', table_name='game7_testnet_labels')
+    op.drop_index('uk_game7_orbit_arbitrum_sepolia_labels_tx_hash_tx_call_raw', table_name='game7_orbit_arbitrum_sepolia_labels', postgresql_where=sa.text("label='seer-raw' and label_type='tx_call'"))
+    op.drop_index('uk_game7_orbit_arbitrum_sepolia_labels_tx_hash_tx_call', table_name='game7_orbit_arbitrum_sepolia_labels', postgresql_where=sa.text("label='seer' and label_type='tx_call'"))
+    op.drop_index('uk_game7_orbit_arbitrum_sepolia_labels_tx_hash_log_idx_evt_raw', table_name='game7_orbit_arbitrum_sepolia_labels', postgresql_where=sa.text("label='seer-raw' and label_type='event'"))
+    op.drop_index('uk_game7_orbit_arbitrum_sepolia_labels_tx_hash_log_idx_evt', table_name='game7_orbit_arbitrum_sepolia_labels', postgresql_where=sa.text("label='seer' and label_type='event'"))
+    op.drop_index('ix_game7_orbit_arbitrum_sepolia_labels_label_addr_name', table_name='game7_orbit_arbitrum_sepolia_labels')
+    op.drop_index('ix_game7_orbit_arbitrum_sepolia_labels_addr_block_ts', table_name='game7_orbit_arbitrum_sepolia_labels')
+    op.drop_index('ix_game7_orbit_arbitrum_sepolia_labels_addr_block_num', table_name='game7_orbit_arbitrum_sepolia_labels')
+    op.create_index('uk_g7o_arbitrum_sepolia_labels_tx_hash_tx_call_raw', 'game7_orbit_arbitrum_sepolia_labels', ['transaction_hash'], unique=True, postgresql_where="(((label)::text = 'seer-raw'::text) AND ((label_type)::text = 'tx_call'::text))")
+    op.create_index('uk_g7o_arbitrum_sepolia_labels_tx_hash_tx_call', 'game7_orbit_arbitrum_sepolia_labels', ['transaction_hash'], unique=True, postgresql_where="(((label)::text = 'seer'::text) AND ((label_type)::text = 'tx_call'::text))")
+    op.create_index('uk_g7o_arbitrum_sepolia_labels_tx_hash_log_idx_evt_raw', 'game7_orbit_arbitrum_sepolia_labels', ['transaction_hash', 'log_index'], unique=True, postgresql_where="(((label)::text = 'seer-raw'::text) AND ((label_type)::text = 'event'::text))")
+    op.create_index('uk_g7o_arbitrum_sepolia_labels_tx_hash_log_idx_evt', 'game7_orbit_arbitrum_sepolia_labels', ['transaction_hash', 'log_index'], unique=True, postgresql_where="(((label)::text = 'seer'::text) AND ((label_type)::text = 'event'::text))")
+    op.create_index('ix_g7o_arbitrum_sepolia_labels_addr_block_ts', 'game7_orbit_arbitrum_sepolia_labels', ['address', 'block_timestamp'], unique=False)
+    op.create_index('ix_g7o_arbitrum_sepolia_labels_addr_block_num', 'game7_orbit_arbitrum_sepolia_labels', ['address', 'block_number'], unique=False)
+    op.drop_index('ix_game7_labels_label_addr_name', table_name='game7_labels')
+    op.drop_index('ix_ethereum_labels_label_addr_name', table_name='ethereum_labels')
+    op.drop_index('ix_ethereum_labels_addr_block_ts', table_name='ethereum_labels')
+    op.drop_index('ix_blast_sepolia_labels_label_addr_name', table_name='blast_sepolia_labels')
+    op.drop_index('ix_blast_labels_label_addr_name', table_name='blast_labels')
+    op.drop_index('ix_base_labels_label_addr_name', table_name='base_labels')
+    op.drop_index('ix_base_labels_addr_block_ts', table_name='base_labels')
+    op.drop_index('ix_b3_sepolia_labels_label_addr_name', table_name='b3_sepolia_labels')
+    op.drop_index('ix_b3_labels_label_addr_name', table_name='b3_labels')
+    op.drop_index('ix_avalanche_labels_label_addr_name', table_name='avalanche_labels')
+    op.drop_index('ix_avalanche_labels_addr_block_ts', table_name='avalanche_labels')
+    op.drop_index('ix_avalanche_fuji_labels_label_addr_name', table_name='avalanche_fuji_labels')
+    op.drop_index('ix_avalanche_fuji_labels_addr_block_ts', table_name='avalanche_fuji_labels')
+    op.drop_index('ix_arbitrum_sepolia_labels_label_addr_name', table_name='arbitrum_sepolia_labels')
+    op.drop_index('ix_arbitrum_sepolia_labels_addr_block_ts', table_name='arbitrum_sepolia_labels')
+    op.drop_index('ix_arbitrum_one_labels_label_addr_name', table_name='arbitrum_one_labels')
+    op.drop_index('ix_arbitrum_nova_labels_label_addr_name', table_name='arbitrum_nova_labels')
+    op.drop_index('ix_amoy_labels_label_addr_name', table_name='amoy_labels')
+    # ### end Alembic commands ###

moonstreamdb-v3/moonstreamdbv3/version.txt

@@ -1 +1 @@
-0.1.5
+0.1.6

nodebalancer/README.md

@@ -123,3 +123,108 @@ python migrations/migrations.py run --key 20230522 \
     --token-new-owner "$MOONSTREAM_ADMIN_OR_OTHER_CONTROLLER" \
     --new-application-id "$MOONSTREAM_APPLICATION_ID"
 ```
+
+## Balances Endpoint
+
+The `/balances` endpoint allows you to retrieve token balances for a specified Ethereum address across multiple blockchains.
+
+### Request
+
+```
+GET /balances?address=<ethereumAddress>
+```
+
+Parameters:
+- `address` (required): The Ethereum address to query balances for
+
+### Response
+
+The endpoint returns a JSON object with the following structure:
+
+```json
+{
+  "1": {
+    "chain_id": "1",
+    "name": "ethereum",
+    "image_url": "https://example.com/eth.png",
+    "balances": {
+      "0x0000000000000000000000000000000000000000": "1000000000000000000",
+      "0xdac17f958d2ee523a2206206994597c13d831ec7": "2000000000000000000",
+      "0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48": "3000000000000000"
+    }
+  },
+  "137": {
+    "chain_id": "137", 
+    "name": "polygon",
+    "image_url": "https://example.com/matic.png",
+    "balances": {
+      "0x0000000000000000000000000000000000000000": "4000000000000000000",
+      "0x2791bca1f2de4661ed88a30c99a7a9449aa84174": "5000000000000000",
+      "0xc2132d05d31c914a87c6611c10748aeb04b58e8f": "6000000000000000000"
+    }
+  }
+}
+```
+
+Where:
+- The top-level keys are chain IDs (e.g. "1" for Ethereum, "137" for Polygon)
+- Each chain object contains:
+  - `chain_id`: The chain identifier as a string
+  - `name`: The human-readable name of the chain
+  - `image_url`: URL to the chain's logo/image
+  - `balances`: Map of token addresses to their balances
+    - Native token (ETH, MATIC etc) is represented by the zero address: `0x0000000000000000000000000000000000000000`
+    - All balances are returned as strings in the token's smallest unit (e.g., wei for ETH)
+
+### Features
+
+1. **Caching**: Responses are cached for 10 seconds to minimize blockchain RPC calls
+2. **Multicall**: Uses Multicall3 contract to batch balance queries for efficiency
+3. **Error Handling**: Individual token or blockchain failures don't affect other 
+
+### Example
+
+```bash
+curl "http://localhost:8080/balances?address=0x742d35Cc6634C0532925a3b844Bc454e4438f44e"
+```
+
+### Contracts Config Structure
+
+The `contracts.json` file should follow this structure:
+
+```json
+{
+  "ethereum": {
+    "multicall3": "0xcA11bde05977b3631167028862bE2a173976CA11",
+    "chain_id": "1",
+    "name": "Ethereum",
+    "image_url": "https://example.com/eth.png",
+    "native_token": "ETH",
+    "tokens": {
+      "0xdac17f958d2ee523a2206206994597c13d831ec7": "USDT",
+      "0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48": "USDC"
+    }
+  },
+  "polygon": {
+    "multicall3": "0xcA11bde05977b3631167028862bE2a173976CA11",
+    "chain_id": "137",
+    "name": "Polygon",
+    "image_url": "https://example.com/matic.png",
+    "native_token": "MATIC",
+    "tokens": {
+      "0x2791bca1f2de4661ed88a30c99a7a9449aa84174": "USDC",
+      "0xc2132d05d31c914a87c6611c10748aeb04b58e8f": "USDT"
+    }
+  }
+}
+```
+
+Where:
+- Top-level keys are blockchain identifiers used internally
+- Each chain configuration contains:
+  - `multicall3`: Address of the Multicall3 contract on that chain
+  - `chain_id`: The chain identifier (e.g. "1" for Ethereum)
+  - `name`: Human-readable name of the chain
+  - `image_url`: URL to the chain's logo/image
+  - `native_token`: Symbol for the chain's native token (ETH, MATIC etc)
+  - `tokens`: Map of token addresses to their symbols

nodebalancer/cmd/nodebalancer/balances.go

@@ -0,0 +1,278 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"math/big"
+	"sync"
+	"time"
+
+	ethereum "github.com/ethereum/go-ethereum"
+	"github.com/ethereum/go-ethereum/accounts/abi/bind"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/ethclient"
+)
+
+type TokenBalance struct {
+	Address string `json:"address"`
+	Balance string `json:"balance"`
+}
+
+type ChainBalances map[string]string
+
+type ChainInfo struct {
+	ChainID  string        `json:"chain_id"`
+	Name     string        `json:"name"`
+	ImageURL string        `json:"image_url"`
+	Balances ChainBalances `json:"balances"`
+}
+
+// Map of blockchain -> token balances
+type BalancesResponse map[string]ChainInfo
+
+type chainResult struct {
+	blockchain string
+	balances   ChainBalances
+	err        error
+}
+
+// getBalancesMulticall queries token balances using Multicall3
+func getBalancesMulticall(ctx context.Context, client *ethclient.Client, tokens []string, checksumAddress string, blockchain string) (ChainBalances, error) {
+	chainBalances := make(ChainBalances)
+
+	// Get Multicall3 contract address
+	multicallAddr := getMulticall3Address(blockchain)
+	if multicallAddr == "" {
+		return nil, fmt.Errorf("multicall3 not supported for blockchain %s", blockchain)
+	}
+
+	// Create Multicall3 contract instance
+	multicallAddress := common.HexToAddress(multicallAddr)
+	mc3, err := NewMulticall3(multicallAddress, client)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create multicall3 contract: %v", err)
+	}
+
+	// Prepare calls for each token
+	calls := make([]Multicall3Call, len(tokens))
+	for i, token := range tokens {
+		callData := createBalanceOfCallData(checksumAddress)
+		calls[i] = Multicall3Call{
+			Target:   common.HexToAddress(token),
+			CallData: callData,
+		}
+	}
+
+	session := Multicall3Session{
+		Contract: mc3,
+		CallOpts: bind.CallOpts{Context: ctx},
+	}
+
+	var result []interface{}
+	err = session.Contract.Multicall3Caller.contract.Call(&session.CallOpts, &result, "tryBlockAndAggregate", false, calls)
+	if err != nil {
+		return nil, fmt.Errorf("multicall failed: %v", err)
+	}
+
+	if len(result) != 3 {
+		return nil, fmt.Errorf("unexpected result length: got %d, want 3", len(result))
+	}
+
+	returnData := result[2].([]struct {
+		Success    bool   `json:"success"`
+		ReturnData []byte `json:"returnData"`
+	})
+
+	for i, data := range returnData {
+		if data.Success && len(data.ReturnData) > 0 {
+			balance := new(big.Int)
+			balance.SetBytes(data.ReturnData)
+			chainBalances[tokens[i]] = balance.String()
+		}
+	}
+
+	return chainBalances, nil
+}
+
+// getBalances fetches token balances across all supported blockchains
+func getBalances(ctx context.Context, address string) (BalancesResponse, error) {
+	if !common.IsHexAddress(address) {
+		return nil, fmt.Errorf("invalid ethereum address")
+	}
+
+	// Convert to checksum address
+	checksumAddress := common.HexToAddress(address).Hex()
+
+	// Initialize response
+	response := make(BalancesResponse)
+
+	// Create channel for collecting results
+	resultChan := make(chan chainResult)
+
+	// Count active goroutines
+	var wg sync.WaitGroup
+
+	// get supported blockchains which are present in contractsConfig
+	supportedBlockchainsFiltered := make(map[string]bool)
+	for blockchain := range contractsConfig {
+		if _, ok := supportedBlockchains[blockchain]; ok {
+			supportedBlockchainsFiltered[blockchain] = true
+		}
+	}
+
+	// Process each blockchain in parallel
+	for blockchain := range supportedBlockchainsFiltered {
+		wg.Add(1)
+		go func(blockchain string) {
+			defer wg.Done()
+
+			// Create timeout context
+			chainCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
+			defer cancel()
+
+			// Get the node for this blockchain
+			node := blockchainPool.GetNextNode(blockchain)
+			if node == nil {
+				resultChan <- chainResult{blockchain: blockchain, err: fmt.Errorf("no available node")}
+				return
+			}
+
+			// Connect to client
+			client, err := ethclient.Dial(node.Endpoint.String())
+			if err != nil {
+				resultChan <- chainResult{blockchain: blockchain, err: fmt.Errorf("failed to connect: %v", err)}
+				return
+			}
+			defer client.Close()
+
+			// Initialize chain balances
+			chainBalances := make(ChainBalances)
+
+			// Get native token balance first
+			nativeBalance, err := client.BalanceAt(chainCtx, common.HexToAddress(checksumAddress), nil)
+			if err != nil {
+				log.Printf("Failed to get native balance for %s: %v", blockchain, err)
+			} else {
+				nativeSymbol := getNativeTokenSymbol(blockchain)
+				chainBalances[nativeSymbol] = nativeBalance.String()
+			}
+
+			// Get token list for this blockchain
+			tokens := getTokenList(blockchain)
+			if len(tokens) > 0 {
+				// Try Multicall3 first for ERC20 tokens
+				tokenBalances, err := getBalancesMulticall(chainCtx, client, tokens, checksumAddress, blockchain)
+				if err != nil {
+					// Fallback to individual calls
+					tokenBalances, err = getBalancesFallback(chainCtx, client, tokens, checksumAddress)
+					if err != nil {
+						resultChan <- chainResult{blockchain: blockchain, err: err}
+						return
+					}
+				}
+				// Merge token balances into chain balances
+				for token, balance := range tokenBalances {
+					chainBalances[token] = balance
+				}
+			}
+
+			select {
+			case <-chainCtx.Done():
+				resultChan <- chainResult{blockchain: blockchain, err: fmt.Errorf("timeout exceeded")}
+			case resultChan <- chainResult{blockchain: blockchain, balances: chainBalances}:
+			}
+		}(blockchain)
+	}
+
+	// Close results channel when all goroutines are done
+	go func() {
+		wg.Wait()
+		close(resultChan)
+	}()
+
+	// Collect results
+	for result := range resultChan {
+		if result.err != nil {
+			log.Printf("Error fetching balances for %s: %v", result.blockchain, result.err)
+			continue
+		}
+		if len(result.balances) > 0 {
+			response[contractsConfig[result.blockchain].ChainID] = ChainInfo{
+				ChainID:  contractsConfig[result.blockchain].ChainID,
+				Name:     result.blockchain,
+				ImageURL: contractsConfig[result.blockchain].ImageURL,
+				Balances: result.balances,
+			}
+		}
+	}
+
+	return response, nil
+}
+
+// getBalancesFallback queries token balances one by one
+func getBalancesFallback(ctx context.Context, client *ethclient.Client, tokens []string, checksumAddress string) (ChainBalances, error) {
+	chainBalances := make(ChainBalances)
+	for _, token := range tokens {
+		select {
+		case <-ctx.Done():
+			return chainBalances, fmt.Errorf("timeout exceeded")
+		default:
+			callData := createBalanceOfCallData(checksumAddress)
+			tokenAddr := common.HexToAddress(token)
+			result, err := client.CallContract(ctx, ethereum.CallMsg{
+				To:   &tokenAddr,
+				Data: callData,
+			}, nil)
+			if err != nil {
+				log.Printf("Failed to get balance for token %s: %v", token, err)
+				continue
+			}
+			if len(result) > 0 {
+				balance := new(big.Int)
+				balance.SetBytes(result)
+				chainBalances[token] = balance.String()
+			}
+		}
+	}
+	return chainBalances, nil
+}
+
+// Helper functions
+
+func createBalanceOfCallData(address string) []byte {
+	// ERC20 balanceOf function signature: balanceOf(address)
+	methodID := crypto.Keccak256([]byte("balanceOf(address)"))[0:4]
+
+	// Pack address parameter
+	paddedAddress := common.LeftPadBytes(common.HexToAddress(address).Bytes(), 32)
+
+	// Combine method ID and parameters
+	return append(methodID, paddedAddress...)
+}
+
+func getMulticall3Address(blockchain string) string {
+	if chain, ok := contractsConfig[blockchain]; ok {
+		return chain.Multicall3
+	}
+	return ""
+}
+
+func getTokenList(blockchain string) []string {
+	if chain, ok := contractsConfig[blockchain]; ok {
+		tokens := make([]string, 0, len(chain.Tokens))
+		for _, addr := range chain.Tokens {
+			tokens = append(tokens, addr)
+		}
+		return tokens
+	}
+	return nil
+}
+
+func getNativeTokenSymbol(blockchain string) string {
+	if chain, ok := contractsConfig[blockchain]; ok {
+		return chain.NativeToken
+	}
+	return ""
+}

nodebalancer/cmd/nodebalancer/cli.go

@@ -524,6 +524,11 @@ var CommonCommands = []*cli.Command{
 				Usage:    "Path to configuration file",
 				Required: true,
 			},
+			&cli.StringFlag{
+				Name:     "contracts-config",
+				Usage:    "Path to contracts configuration file",
+				Required: true,
+			},
 			&cli.StringFlag{
 				Name:  "host",
 				Usage: "Server listening address",
@@ -555,7 +560,7 @@ var CommonCommands = []*cli.Command{
 
 			CheckEnvVarSet()
 
-			servErr := Server(c.String("config"), c.String("host"), c.String("port"), c.Bool("healthcheck"))
+			servErr := Server(c.String("config"), c.String("contracts-config"), c.String("host"), c.String("port"), c.Bool("healthcheck"))
 			if servErr != nil {
 				return servErr
 			}

nodebalancer/cmd/nodebalancer/clients.go

@@ -84,7 +84,7 @@ func (ca *ClientAccess) UpdateClientResourceCallCounter(tsNow int64) error {
 		[]string{},
 	)
 	if err != nil {
-		return err
+		return fmt.Errorf("resource %s update error %v", ca.ResourceID, err)
 	}
 	log.Printf("Resource %s updated\n", updatedResource.Id)
 

nodebalancer/cmd/nodebalancer/configs.go

@@ -25,10 +25,13 @@ var (
 
 	bugoutClient *bugout.BugoutClient
 
+	contractsConfig ContractsConfig
+
 	// Bugout client
 	// TODO(kompotkot): Find out why it cuts out the port
 	BUGOUT_BROOD_URL = "https://auth.bugout.dev"
 	// BUGOUT_BROOD_URL              = os.Getenv("BUGOUT_BROOD_URL")
+	NB_BUGOUT_TIMEOUT_SECONDS     = 10
 	NB_BUGOUT_TIMEOUT_SECONDS_RAW = os.Getenv("NB_BUGOUT_TIMEOUT_SECONDS")
 
 	// Bugout and application configuration
@@ -43,13 +46,22 @@ var (
 	NB_ENABLE_DEBUG = false
 
 	NB_CONNECTION_RETRIES          = 2
-	NB_CONNECTION_RETRIES_INTERVAL = time.Millisecond * 10
-	NB_HEALTH_CHECK_INTERVAL       = os.Getenv("NB_HEALTH_CHECK_INTERVAL")
+	NB_CONNECTION_RETRIES_INTERVAL = time.Second * 5
+	NB_HEALTH_CHECK_INTERVAL       = 30
+	NB_HEALTH_CHECK_INTERVAL_RAW   = os.Getenv("NB_HEALTH_CHECK_INTERVAL")
 	NB_HEALTH_CHECK_CALL_TIMEOUT   = time.Second * 2
 
-	NB_CACHE_CLEANING_INTERVAL          = time.Second * 10
-	NB_CACHE_ACCESS_ID_LIFETIME         = int64(120) // 2 minutes
-	NB_CACHE_ACCESS_ID_SESSION_LIFETIME = int64(600) // 10 minutes
+	NB_CACHE_CLEANING_INTERVAL              = 10
+	NB_CACHE_CLEANING_INTERVAL_RAW          = os.Getenv("NB_CACHE_CLEANING_INTERVAL")
+	NB_CACHE_ACCESS_ID_LIFETIME             = int64(120) // After 2 minutes, the access ID will be deleted from the cache if there has been no activity
+	NB_CACHE_ACCESS_ID_LIFETIME_RAW         = os.Getenv("NB_CACHE_ACCESS_ID_LIFETIME")
+	NB_CACHE_ACCESS_ID_SESSION_LIFETIME     = int64(900) // After 15 minutes, the access ID will be deleted from the cache to refresh access limits
+	NB_CACHE_ACCESS_ID_SESSION_LIFETIME_RAW = os.Getenv("NB_CACHE_ACCESS_ID_SESSION_LIFETIME")
+
+	NB_BALANCES_CACHE_EXPIRATION            = 10
+	NB_BALANCES_CACHE_EXPIRATION_RAW        = os.Getenv("NB_BALANCES_CACHE_EXPIRATION")
+	NB_BALANCES_CACHE_CLEANING_INTERVAL     = 20
+	NB_BALANCES_CACHE_CLEANING_INTERVAL_RAW = os.Getenv("NB_BALANCES_CACHE_CLEANING_INTERVAL")
 
 	NB_MAX_COUNTER_NUMBER = uint64(10000000)
 
@@ -70,11 +82,14 @@ var (
 )
 
 func CreateBugoutClient() (*bugout.BugoutClient, error) {
-	bugoutTimeoutSeconds, err := strconv.Atoi(NB_BUGOUT_TIMEOUT_SECONDS_RAW)
-	if err != nil {
-		return nil, fmt.Errorf("unable to parse environment variable as integer: %v", err)
+	bugoutTimeoutSeconds, atoiErr := strconv.Atoi(NB_BUGOUT_TIMEOUT_SECONDS_RAW)
+	if atoiErr != nil {
+		log.Printf("Unable to parse environment variable NB_BUGOUT_TIMEOUT_SECONDS as integer and set to default %d, err: %v", NB_BUGOUT_TIMEOUT_SECONDS, atoiErr)
+	} else {
+		NB_BUGOUT_TIMEOUT_SECONDS = bugoutTimeoutSeconds
 	}
-	NB_BUGOUT_TIMEOUT_SECONDS := time.Duration(bugoutTimeoutSeconds) * time.Second
+
+	NB_BUGOUT_TIMEOUT_SECONDS := time.Duration(NB_BUGOUT_TIMEOUT_SECONDS) * time.Second
 
 	bugoutClient := bugout.ClientBrood(BUGOUT_BROOD_URL, NB_BUGOUT_TIMEOUT_SECONDS)
 	return &bugoutClient, nil
@@ -95,6 +110,62 @@ func CheckEnvVarSet() {
 	for _, o := range strings.Split(MOONSTREAM_CORS_ALLOWED_ORIGINS, ",") {
 		CORS_WHITELIST_MAP[o] = true
 	}
+
+	// Health check variables
+	if NB_HEALTH_CHECK_INTERVAL_RAW != "" {
+		healthCheckInterval, atoiErr := strconv.Atoi(NB_HEALTH_CHECK_INTERVAL_RAW)
+		if atoiErr != nil {
+			log.Printf("Unable to parse environment variable NB_HEALTH_CHECK_INTERVAL as integer and set to default %d, err: %v", NB_HEALTH_CHECK_INTERVAL, atoiErr)
+		} else {
+			NB_HEALTH_CHECK_INTERVAL = healthCheckInterval
+		}
+	}
+
+	// Cache variables
+	if NB_CACHE_CLEANING_INTERVAL_RAW != "" {
+		nbCacheCleaningInterval, atoiErr := strconv.Atoi(NB_CACHE_CLEANING_INTERVAL_RAW)
+		if atoiErr != nil {
+			log.Printf("Unable to parse environment variable NB_CACHE_CLEANING_INTERVAL as integer and set to default %d, err: %v", NB_CACHE_CLEANING_INTERVAL, atoiErr)
+		} else {
+			NB_CACHE_CLEANING_INTERVAL = nbCacheCleaningInterval
+		}
+	}
+
+	if NB_CACHE_ACCESS_ID_LIFETIME_RAW != "" {
+		nbCacheAccessIdLifetime, atoiErr := strconv.Atoi(NB_CACHE_ACCESS_ID_LIFETIME_RAW)
+		if atoiErr != nil {
+			log.Printf("Unable to parse environment variable NB_CACHE_ACCESS_ID_LIFETIME as integer and set to default %d, err: %v", NB_CACHE_ACCESS_ID_LIFETIME, atoiErr)
+		} else {
+			NB_CACHE_ACCESS_ID_LIFETIME = int64(nbCacheAccessIdLifetime)
+		}
+	}
+
+	if NB_CACHE_ACCESS_ID_SESSION_LIFETIME_RAW != "" {
+		nbCacheAccessIdSessionLifetime, atoiErr := strconv.Atoi(NB_CACHE_ACCESS_ID_SESSION_LIFETIME_RAW)
+		if atoiErr != nil {
+			log.Printf("Unable to parse environment variable NB_CACHE_ACCESS_ID_SESSION_LIFETIME as integer and set to default %d, err: %v", NB_CACHE_ACCESS_ID_SESSION_LIFETIME, atoiErr)
+		} else {
+			NB_CACHE_ACCESS_ID_SESSION_LIFETIME = int64(nbCacheAccessIdSessionLifetime)
+		}
+	}
+
+	if NB_BALANCES_CACHE_EXPIRATION_RAW != "" {
+		nbBalancesCacheExpiration, atoiErr := strconv.Atoi(NB_BALANCES_CACHE_EXPIRATION_RAW)
+		if atoiErr != nil {
+			log.Printf("Unable to parse environment variable NB_BALANCES_CACHE_EXPIRATION as integer and set to default %d, err: %v", NB_BALANCES_CACHE_EXPIRATION, atoiErr)
+		} else {
+			NB_BALANCES_CACHE_EXPIRATION = nbBalancesCacheExpiration
+		}
+	}
+
+	if NB_BALANCES_CACHE_CLEANING_INTERVAL_RAW != "" {
+		nbBalancesCacheCleaningInterval, atoiErr := strconv.Atoi(NB_BALANCES_CACHE_CLEANING_INTERVAL_RAW)
+		if atoiErr != nil {
+			log.Printf("Unable to parse environment variable NB_BALANCES_CACHE_CLEANING_INTERVAL as integer and set to default %d, err: %v", NB_BALANCES_CACHE_CLEANING_INTERVAL, atoiErr)
+		} else {
+			NB_BALANCES_CACHE_CLEANING_INTERVAL = nbBalancesCacheCleaningInterval
+		}
+	}
 }
 
 // Nodes configuration
@@ -103,6 +174,19 @@ type NodeConfig struct {
 	Endpoint   string `json:"endpoint"`
 }
 
+type TokenConfig map[string]string
+
+type ChainConfig struct {
+	Multicall3  string      `json:"multicall3"`
+	Tokens      TokenConfig `json:"tokens"`
+	NativeToken string      `json:"native_token"`
+	ChainID     string      `json:"chain_id"`
+	Name        string      `json:"name"`
+	ImageURL    string      `json:"image_url"`
+}
+
+type ContractsConfig map[string]ChainConfig
+
 func LoadConfig(configPath string) error {
 	rawBytes, err := ioutil.ReadFile(configPath)
 	if err != nil {
@@ -117,6 +201,23 @@ func LoadConfig(configPath string) error {
 	return nil
 }
 
+func LoadContractsConfig(configPath string) error {
+
+	// Read config file
+	data, err := os.ReadFile(filepath.Join(configPath))
+	if err != nil {
+		return err
+	}
+
+	// Parse JSON
+	err = json.Unmarshal(data, &contractsConfig)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 type ConfigPlacement struct {
 	ConfigDirPath   string
 	ConfigDirExists bool

nodebalancer/cmd/nodebalancer/middleware.go

@@ -195,7 +195,7 @@ func (ac *AccessCache) Cleanup() (int64, int64) {
 }
 
 func initCacheCleaning(debug bool) {
-	t := time.NewTicker(NB_CACHE_CLEANING_INTERVAL)
+	t := time.NewTicker(time.Second * time.Duration(NB_CACHE_CLEANING_INTERVAL))
 	for {
 		select {
 		case <-t.C:

nodebalancer/cmd/nodebalancer/routes.go

@@ -11,6 +11,7 @@ import (
 	"log"
 	"net/http"
 	"strings"
+	"time"
 )
 
 type PingResponse struct {
@@ -123,3 +124,39 @@ func lbJSONRPCHandler(w http.ResponseWriter, r *http.Request, blockchain string,
 		return
 	}
 }
+
+// balancesRoute handles the /balances endpoint
+func balancesRoute(w http.ResponseWriter, r *http.Request) {
+	// Get address from query params
+	address := r.URL.Query().Get("address")
+
+	if address == "" {
+		http.Error(w, "Address is required", http.StatusBadRequest)
+		return
+	}
+
+	// Check cache first
+	cacheKey := fmt.Sprintf("balances:%s", address)
+	if cachedData, found := balancesCache.Get(cacheKey); found {
+		w.Header().Set("Content-Type", "application/json")
+		w.Write(cachedData.([]byte))
+		return
+	}
+
+	// Get balances
+	response, err := getBalances(r.Context(), address)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	// Cache the response
+	responseBytes, err := json.Marshal(response)
+	if err == nil {
+		balancesCache.Set(cacheKey, responseBytes, 10*time.Second)
+	}
+
+	// Send response
+	w.Header().Set("Content-Type", "application/json")
+	json.NewEncoder(w).Encode(response)
+}

nodebalancer/cmd/nodebalancer/server.go

@@ -10,12 +10,12 @@ import (
 	"net/http"
 	"net/http/httputil"
 	"net/url"
-	"strconv"
 	"strings"
 	"time"
 
 	humbug "github.com/bugout-dev/humbug/go/pkg"
 	"github.com/google/uuid"
+	"github.com/patrickmn/go-cache"
 )
 
 var (
@@ -23,15 +23,13 @@ var (
 
 	// Crash reporter
 	reporter *humbug.HumbugReporter
+	// Cache for balances
+	balancesCache *cache.Cache
 )
 
 // initHealthCheck runs a routine for check status of the nodes every 5 seconds
 func initHealthCheck(debug bool) {
-	healthCheckInterval, convErr := strconv.Atoi(NB_HEALTH_CHECK_INTERVAL)
-	if convErr != nil {
-		healthCheckInterval = 30
-	}
-	t := time.NewTicker(time.Second * time.Duration(healthCheckInterval))
+	t := time.NewTicker(time.Second * time.Duration(NB_HEALTH_CHECK_INTERVAL))
 	for {
 		select {
 		case <-t.C:
@@ -105,10 +103,18 @@ func proxyErrorHandler(proxy *httputil.ReverseProxy, url *url.URL) {
 	}
 }
 
-func Server(configPath, listeningHostAddr, listeningPort string, enableHealthCheck bool) error {
+func Server(nodesConfigPath, contractsConfigPath, listeningHostAddr, listeningPort string, enableHealthCheck bool) error {
+	// Initialize Balances cache
+	balancesCache = cache.New(time.Duration(NB_BALANCES_CACHE_EXPIRATION)*time.Second, time.Duration(NB_BALANCES_CACHE_CLEANING_INTERVAL)*time.Second)
+
 	// Create Access ID cache
 	CreateAccessCache()
 
+	// Load contracts configuration
+	if err := LoadContractsConfig(contractsConfigPath); err != nil {
+		return fmt.Errorf("failed to load contracts config: %v", err)
+	}
+
 	// Configure Humbug reporter to handle errors
 	var err error
 	sessionID := uuid.New().String()
@@ -125,6 +131,7 @@ func Server(configPath, listeningHostAddr, listeningPort string, enableHealthChe
 	if getErr != nil {
 		return fmt.Errorf("unable to get user with provided access identifier, err: %v", getErr)
 	}
+
 	if len(resources.Resources) == 1 {
 		clientAccess, parseErr := ParseResourceDataToClientAccess(resources.Resources[0])
 		if parseErr != nil {
@@ -154,7 +161,7 @@ func Server(configPath, listeningHostAddr, listeningPort string, enableHealthChe
 	}
 
 	// Fill NodeConfigList with initial nodes from environment variables
-	err = LoadConfig(configPath)
+	err = LoadConfig(nodesConfigPath)
 	if err != nil {
 		return err
 	}
@@ -216,6 +223,7 @@ func Server(configPath, listeningHostAddr, listeningPort string, enableHealthChe
 
 	serveMux := http.NewServeMux()
 	serveMux.Handle("/nb/", accessMiddleware(http.HandlerFunc(lbHandler)))
+	serveMux.HandleFunc("/balances", balancesRoute)
 	log.Println("Authentication middleware enabled")
 	serveMux.HandleFunc("/ping", pingRoute)
 

nodebalancer/cmd/nodebalancer/version.go

@@ -1,3 +1,3 @@
 package main
 
-var NB_VERSION = "0.2.8"
+var NB_VERSION = "0.2.9"

nodebalancer/configs/config.sample.json

@@ -0,0 +1,18 @@
+[
+  {
+    "blockchain": "ethereum",
+    "endpoint": "https://mainnet.infura.io",
+    "tags": [
+      "external",
+      "sample"
+    ]
+  },
+  {
+    "blockchain": "polygon",
+    "endpoint": "https://polygon-mainnet.infura.io",
+    "tags": [
+      "external",
+      "sample"
+    ]
+  }
+]

nodebalancer/configs/contractsConfig.sample.json

@@ -0,0 +1,23 @@
+{
+    "ethereum": {
+        "multicall3": "0xcA11bde05977b3631167028862bE2a173976CA11",
+        "native_token": "ETH",
+        "tokens": {
+            "WETH": "0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2",
+            "DAI": "0x6B175474E89094C44Da98b954EedeAC495271d0F",
+            "USDC": "0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48",
+            "USDT": "0xdAC17F958D2ee523a2206206994597C13D831ec7"
+        }
+    },
+    "polygon": {
+        "multicall3": "0xcA11bde05977b3631167028862bE2a173976CA11",
+        "native_token": "MATIC",
+        "tokens": {
+            "WMATIC": "0x0d500B1d8E8eF31E21C99d1Db9A6444d3ADf1270",
+            "USDC": "0x2791Bca1f2de4661ED88A30C99A7a9449Aa84174",
+            "DAI": "0x8f3Cf7ad23Cd3CaDbD9735AFf958023239c6A063",
+            "WETH": "0x7ceB23fD6bC0adD59E62ac25578270cFf1b9f619",
+            "USDT": "0xc2132D05D31c914a87C6611C10748AEb04B58e8F"
+        }
+    }
+}

nodebalancer/deploy/deploy.bash

@@ -17,6 +17,7 @@ PREFIX_CRIT="${C_RED}[CRIT]${C_RESET} [$(date +%d-%m\ %T)]"
 AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION:-us-east-1}"
 APP_DIR="${APP_DIR:-/home/ubuntu/api}"
 SECRETS_DIR="${SECRETS_DIR:-/home/ubuntu/nodebalancer-secrets}"
+NB_CONTRACTS_CONFIG_PATH="${SECRETS_DIR}/contractsConfig.json"
 PARAMETERS_ENV_PATH="${SECRETS_DIR}/app.env"
 SCRIPT_DIR="$(realpath $(dirname $0))"
 
@@ -30,6 +31,11 @@ echo
 echo -e "${PREFIX_INFO} Install checkenv"
 HOME=/home/ubuntu /usr/local/go/bin/go install github.com/bugout-dev/checkenv@v0.0.4
 
+if [ ! -d "${SECRETS_DIR}" ]; then
+  mkdir "${SECRETS_DIR}"
+  echo -e "${PREFIX_WARN} Created new secrets directory"
+fi
+
 echo
 echo
 echo -e "${PREFIX_INFO} Add instance local IP to parameters"
@@ -38,12 +44,13 @@ echo "AWS_LOCAL_IPV4=$(ec2metadata --local-ipv4)" > "${PARAMETERS_ENV_PATH}"
 echo
 echo
 echo -e "${PREFIX_INFO} Retrieving addition deployment parameters"
-if [ ! -d "${SECRETS_DIR}" ]; then
-  mkdir "${SECRETS_DIR}"
-  echo -e "${PREFIX_WARN} Created new secrets directory"
-fi
 AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION}" /home/ubuntu/go/bin/checkenv show aws_ssm+nodebalancer:true >> "${PARAMETERS_ENV_PATH}"
 
+echo
+echo
+echo -e "${PREFIX_INFO} Retrieve nodebalancer contracts config"
+AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION}" aws ssm get-parameter --name "NB_CONTRACTS_CONFIG_JSON" --output text --query Parameter.Value > "${NB_CONTRACTS_CONFIG_PATH}"
+
 echo
 echo
 echo -e "${PREFIX_INFO} Building executable load balancer for nodes script with Go"

nodebalancer/deploy/nodebalancer.service

@@ -13,7 +13,8 @@ ExecStart=/home/ubuntu/api/nodebalancer/nodebalancer server \
     --host "${AWS_LOCAL_IPV4}" \
     --port 8544 \
     --healthcheck \
-    --config /home/ubuntu/.nodebalancer/config.json
+    --config /home/ubuntu/.nodebalancer/config.json \
+    --contracts-config /home/ubuntu/nodebalancer-secrets/contractsConfig.json
 SyslogIdentifier=nodebalancer
 
 [Install]

nodebalancer/go.mod

@@ -5,12 +5,30 @@ go 1.17
 require (
 	github.com/bugout-dev/bugout-go v0.4.6
 	github.com/bugout-dev/humbug/go v0.0.0-20230713220619-2cd74a2b36d7
+	github.com/ethereum/go-ethereum v1.10.10
 	github.com/google/uuid v1.6.0
+	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/urfave/cli/v2 v2.27.5
 )
 
 require (
+	github.com/StackExchange/wmi v1.2.1 // indirect
+	github.com/btcsuite/btcd v0.20.1-beta // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.5 // indirect
+	github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea // indirect
+	github.com/go-ole/go-ole v1.3.0 // indirect
+	github.com/go-stack/stack v1.8.0 // indirect
+	github.com/gorilla/websocket v1.4.2 // indirect
+	github.com/holiman/uint256 v1.3.1 // indirect
+	github.com/rjeczalik/notify v0.9.1 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
+	github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible // indirect
+	github.com/stretchr/testify v1.8.2 // indirect
+	github.com/tklauser/go-sysconf v0.3.12 // indirect
+	github.com/tklauser/numcpus v0.6.1 // indirect
 	github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect
+	golang.org/x/crypto v0.23.0 // indirect
+	golang.org/x/sync v0.8.0 // indirect
+	golang.org/x/sys v0.25.0 // indirect
+	gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect
 )

nodebalancer/go.sum

@@ -1,85 +1,224 @@
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
+cloud.google.com/go v0.43.0/go.mod h1:BOSR3VbTLkk6FDC/TcffxP4NF/FFBGA5ku+jvKOP7pg=
 cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
 cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
 cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
 cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
+cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To=
+cloud.google.com/go v0.51.0/go.mod h1:hWtGJ6gnXH+KgDv+V0zFGDvpi07n3z8ZNj3T1RW0Gcw=
 cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
+cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
+cloud.google.com/go/bigtable v1.2.0/go.mod h1:JcVAOl45lrTmQfLj7T6TxyMzIN/3FGGcFm+2xVAli2o=
 cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
 cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk=
 cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
+cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
 cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
+cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
+collectd.org v0.3.0/go.mod h1:A/8DzQBkF6abtvrT2j/AU/4tiBgJWYyh0y/oB/4MlWE=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
+github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4=
+github.com/Azure/azure-pipeline-go v0.2.2/go.mod h1:4rQ/NZncSvGqNkkOsNpOU1tgoNuIlp9AfUH5G1tvCHc=
+github.com/Azure/azure-storage-blob-go v0.7.0/go.mod h1:f9YQKtsG1nMisotuTPpO0tjNuEjKRYAcJU8/ydDI++4=
+github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI=
+github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0=
+github.com/Azure/go-autorest/autorest/adal v0.8.0/go.mod h1:Z6vX6WXXuyieHAXwMj0S6HY6e6wcHn37qQMBQlvY3lc=
+github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA=
+github.com/Azure/go-autorest/autorest/date v0.2.0/go.mod h1:vcORJHLJEh643/Ioh9+vPmf1Ij9AEBM5FuBIXLmIy0g=
+github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0=
+github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0=
+github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM=
+github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc=
+github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
+github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
+github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
+github.com/StackExchange/wmi v1.2.1 h1:VIkavFPXSjcnS+O8yTq7NI32k0R5Aj+v39y29VYDOSA=
+github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8=
+github.com/VictoriaMetrics/fastcache v1.6.0 h1:C/3Oi3EiBCqufydp1neRZkqcwmEiuRT9c3fqvvgKm5o=
+github.com/VictoriaMetrics/fastcache v1.6.0/go.mod h1:0qHz5QP0GMX4pfmMA/zt5RgfNuXJrTP0zS7DqpHGGTw=
+github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII=
+github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM=
+github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
+github.com/apache/arrow/go/arrow v0.0.0-20191024131854-af6fa24be0db/go.mod h1:VTxUBvSJ3s3eHAg65PNgrsn5BtqCRPdmyXh6rAfdxN0=
 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
+github.com/aws/aws-sdk-go-v2 v1.2.0/go.mod h1:zEQs02YRBw1DjK0PoJv3ygDYOFTre1ejlJWl8FwAuQo=
+github.com/aws/aws-sdk-go-v2/config v1.1.1/go.mod h1:0XsVy9lBI/BCXm+2Tuvt39YmdHwS5unDQmxZOYe8F5Y=
+github.com/aws/aws-sdk-go-v2/credentials v1.1.1/go.mod h1:mM2iIjwl7LULWtS6JCACyInboHirisUUdkBPoTHMOUo=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.0.2/go.mod h1:3hGg3PpiEjHnrkrlasTfxFqUsZ2GCk/fMUn4CbKgSkM=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.0.2/go.mod h1:45MfaXZ0cNbeuT0KQ1XJylq8A6+OpVV2E5kvY/Kq+u8=
+github.com/aws/aws-sdk-go-v2/service/route53 v1.1.1/go.mod h1:rLiOUrPLW/Er5kRcQ7NkwbjlijluLsrIbu/iyl35RO4=
+github.com/aws/aws-sdk-go-v2/service/sso v1.1.1/go.mod h1:SuZJxklHxLAXgLTc1iFXbEWkXs7QRTQpCLGaKIprQW0=
+github.com/aws/aws-sdk-go-v2/service/sts v1.1.1/go.mod h1:Wi0EBZwiz/K44YliU0EKxqTCJGUfYTWXrrBwkq736bM=
+github.com/aws/smithy-go v1.1.0/go.mod h1:EzMw8dbp/YJL4A5/sbhGddag+NPT7q084agLbB9LgIw=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
 github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
+github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40/go.mod h1:8rLXio+WjiTceGBHIoTvn60HIbs7Hm7bcHjyrSqYB9c=
+github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
+github.com/btcsuite/btcd v0.20.1-beta h1:Ik4hyJqN8Jfyv3S4AGBOmyouMsYE3EdYODkMbQjwPGw=
+github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ=
+github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA=
+github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
+github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg=
+github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY=
+github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc=
+github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY=
+github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs=
 github.com/bugout-dev/bugout-go v0.4.6 h1:HaXoVNVZYqd6BaPwlQGhWKBYdGc2lhF3BRxIgyL+1SY=
 github.com/bugout-dev/bugout-go v0.4.6/go.mod h1:P4+788iHtt/32u2wIaRTaiXTWpvSVBYxZ01qQ8N7eB8=
 github.com/bugout-dev/humbug/go v0.0.0-20230713220619-2cd74a2b36d7 h1:Mn6t3HO056/++m5UESl/06FdSxz84S1p7pfQA+NZwVo=
 github.com/bugout-dev/humbug/go v0.0.0-20230713220619-2cd74a2b36d7/go.mod h1:U/NXHfc3tzGeQz+xVfpifXdPZi7p6VV8xdP/4ZKeWJU=
+github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOCSiVIqS34=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/cespare/cp v0.1.0 h1:SE+dxFebS7Iik5LK0tsi1k9ZCxEaFX4AjQmoyA+1dJk=
+github.com/cespare/cp v0.1.0/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s=
+github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
+github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
+github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
+github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
+github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cloudflare/cloudflare-go v0.14.0/go.mod h1:EnwdgGMaFOruiPZRFSgn+TsQ3hQ7C/YWzIGLeu5c304=
+github.com/consensys/bavard v0.1.8-0.20210406032232-f3452dc9b572/go.mod h1:Bpd0/3mZuaj6Sj+PqrmIquiOKy397AKGThQPaGzNXAQ=
+github.com/consensys/gnark-crypto v0.4.1-0.20210426202927-39ac3d4b3f1f/go.mod h1:815PAHg3wvysy0SyIqanF8gZ0Y1wjk/hrDHD/iT88+Q=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
+github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/cpuguy83/go-md2man/v2 v2.0.5 h1:ZtcqGrnekaHpVLArFSe4HK5DoKx1T0rq2DwVB0alcyc=
 github.com/cpuguy83/go-md2man/v2 v2.0.5/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4=
+github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg=
+github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea h1:j4317fAZh7X6GqbFowYdYdI0L9bwxL07jyPZIdepyZ0=
+github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ=
+github.com/deepmap/oapi-codegen v1.6.0/go.mod h1:ryDa9AgbELGeB+YEXE1dR53yAjHwFvE9iAUlWl9Al3M=
+github.com/deepmap/oapi-codegen v1.8.2/go.mod h1:YLgSKSDv/bZQB7N4ws6luhozi3cEdRktEqrX88CvjIw=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
+github.com/dgryski/go-bitstream v0.0.0-20180413035011-3522498ce2c8/go.mod h1:VMaSuZ+SZcx/wljOQKvp5srsbCiKDEb6K2wC4+PiBmQ=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
+github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=
+github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/dop251/goja v0.0.0-20211011172007-d99e4b8cbf48/go.mod h1:R9ET47fwRVRPZnOGvHxxhuZcbrMCuiqOz3Rlrh4KSnk=
+github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA7c8pLvoGndExHudxTDKZ84Pyvv+90pbBjbTz0Y=
+github.com/eclipse/paho.mqtt.golang v1.2.0/go.mod h1:H9keYFcgq3Qr5OUJm/JZI/i6U7joQ8SYLhZwfeOo6Ts=
+github.com/edsrzf/mmap-go v1.0.0 h1:CEBF7HpRnUCSJgGUb5h1Gm7e3VkmVDrR8lvWVLtrOFw=
+github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/ethereum/go-ethereum v1.10.10 h1:Ft2GcLQrr2M89l49g9NoqgNtJZ9AahzMb7N6VXKZy5U=
+github.com/ethereum/go-ethereum v1.10.10/go.mod h1:W3yfrFyL9C1pHcwY5hmRHVDaorTiQxhYBkKyu5mEDHw=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
+github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 h1:FtmdgXiUlNeRsoNMFlKLDt+S+6hbjVMEW6RGQ7aUf7c=
+github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0=
+github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
+github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff h1:tY80oXqGNY4FhTFhk+o9oFHGINQ/+vhlm8HFzi6znCI=
+github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww=
+github.com/getkin/kin-openapi v0.53.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4=
+github.com/getkin/kin-openapi v0.61.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
+github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE=
+github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24=
+github.com/go-chi/chi/v5 v5.0.0/go.mod h1:BBug9lr0cqtdAhsu6R4AAdvufI0/XBzAQSsUqJpoZOs=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
+github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
+github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8=
+github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
+github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE=
+github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78=
+github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
+github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
+github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg=
+github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
+github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
+github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
+github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k=
+github.com/golang/geo v0.0.0-20190916061304-5b978397cfec/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
+github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219/go.mod h1:/X8TswGSh1pIozq4ZwCfxS0WA5JGXguxk94ar/4c87Y=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/flatbuffers v1.11.0/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/gofuzz v1.1.1-0.20200604201612-c04b05f3adfa/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/google/uuid v1.1.5/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
+github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
+github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/graph-gophers/graphql-go v0.0.0-20201113091052-beb923fada29/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
 github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
 github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
 github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/go-bexpr v0.1.10 h1:9kuI5PFotCboP3dkDYFr/wi0gg0QVbSNz5oFRpxn4uE=
+github.com/hashicorp/go-bexpr v0.1.10/go.mod h1:oxlubA2vC/gFVfX1A6JGp7ls7uCDlfJn732ehYYg+g0=
 github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
 github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
 github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
@@ -92,27 +231,94 @@ github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/b
 github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d h1:dg1dEPuWpEqDnvIw251EVy4zlP8gWbsGj4BsUKCRpYs=
+github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
 github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
 github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
 github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
+github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZKVu0fao=
+github.com/holiman/bloomfilter/v2 v2.0.3/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iURXE7ZOP9L9hSkA=
+github.com/holiman/uint256 v1.2.0/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw=
+github.com/holiman/uint256 v1.3.1 h1:JfTzmih28bittyHM8z360dCjIA9dbPIBlcTI6lmctQs=
+github.com/holiman/uint256 v1.3.1/go.mod h1:EOMSn4q6Nyt9P6efbI3bueV4e1b3dGlUCXeiRV4ng7E=
+github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/huin/goupnp v1.0.2 h1:RfGLP+h3mvisuWEyybxNq5Eft3NWhHLPeUN72kpKZoI=
+github.com/huin/goupnp v1.0.2/go.mod h1:0dxJBVBHqTMjIUMkESDTNgOOx/Mw5wYIfyFmdzSamkM=
+github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o=
+github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/influxdata/flux v0.65.1/go.mod h1:J754/zds0vvpfwuq7Gc2wRdVwEodfpCFM7mYlOw2LqY=
+github.com/influxdata/influxdb v1.8.3/go.mod h1:JugdFhsvvI8gadxOI6noqNeeBHvWNTbfYGtiAn+2jhI=
+github.com/influxdata/influxdb-client-go/v2 v2.4.0/go.mod h1:vLNHdxTJkIf2mSLvGrpj8TCcISApPoXkaxP8g9uRlW8=
+github.com/influxdata/influxql v1.1.1-0.20200828144457-65d3ef77d385/go.mod h1:gHp9y86a/pxhjJ+zMjNXiQAA197Xk9wLxaz+fGG+kWk=
+github.com/influxdata/line-protocol v0.0.0-20180522152040-32c6aa80de5e/go.mod h1:4kt73NQhadE3daL3WhR5EJ/J2ocX0PZzwxQ0gXJ7oFE=
+github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo=
+github.com/influxdata/line-protocol v0.0.0-20210311194329-9aa0e372d097/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo=
+github.com/influxdata/promql/v2 v2.12.0/go.mod h1:fxOPu+DY0bqCTCECchSRtWfc+0X19ybifQhZoQNF5D8=
+github.com/influxdata/roaring v0.4.13-0.20180809181101-fc520f41fab6/go.mod h1:bSgUQ7q5ZLSO+bKBGqJiCBGAl+9DxyW63zLTujjUlOE=
+github.com/influxdata/tdigest v0.0.0-20181121200506-bf2b5ad3c0a9/go.mod h1:Js0mqiSBE6Ffsg94weZZ2c+v/ciT8QRHFOap7EKDrR0=
+github.com/influxdata/usage-client v0.0.0-20160829180054-6d3895376368/go.mod h1:Wbbw6tYNvwa5dlB6304Sd+82Z3f7PmVZHVKU637d4po=
+github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458 h1:6OvNmYgJyexcZ3pYbTI9jWx5tHo1Dee/tWbLMfPe2TA=
+github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc=
+github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e/go.mod h1:G1CVv03EnqU1wYL2dFwXxW2An0az9JTl/ZsqXQeBlkU=
+github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
+github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
+github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
+github.com/jsternberg/zap-logfmt v1.0.0/go.mod h1:uvPs/4X51zdkcm5jXl5SYoN+4RK21K8mysFmDaM/h+o=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes=
+github.com/jwilder/encoding v0.0.0-20170811194829-b4e1701a28ef/go.mod h1:Ct9fl0F6iIOGgxJ5npU/IUOhOhqlVrGjyIZc8/MagT0=
+github.com/karalabe/usb v0.0.0-20211005121534-4c5740d64559/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
+github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4=
+github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
+github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
+github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg=
+github.com/klauspost/pgzip v1.0.2-0.20170402124221-0bf5dcad4ada/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
+github.com/labstack/echo/v4 v4.2.1/go.mod h1:AA49e0DZ8kk5jTOOCKNuPR6oTnBS0dYiM4FW1e6jwpg=
+github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k=
+github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8=
+github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/matryer/moq v0.0.0-20190312154309-6cfb0558e1bd/go.mod h1:9ELz6aaclSIGnZBoaSLZ3NAl1VTufbOrXBPvtcy6WiQ=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
+github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
+github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
+github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8=
+github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
+github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc=
+github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc=
 github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
+github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
+github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
+github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
+github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
+github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
+github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0=
+github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
+github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
+github.com/mattn/go-tty v0.0.0-20180907095812-13ff1204f104/go.mod h1:XPvLUNfbS4fJH25nqRHfWLMa1ONC8Amw+mIA639KxkE=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
 github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
@@ -123,32 +329,79 @@ github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS4
 github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=
 github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag=
+github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
+github.com/mitchellh/pointerstructure v1.2.0 h1:O+i9nHnXS3l/9Wu7r4NrEdwA2VFTicjUEN1uBnDo34A=
+github.com/mitchellh/pointerstructure v1.2.0/go.mod h1:BRAsLI5zgXmw97Lf6s25bs8ohIXc3tViBH44KcwB2g4=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0=
+github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E=
+github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
+github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
+github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
+github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
+github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
+github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
+github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
+github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
+github.com/opentracing/opentracing-go v1.0.3-0.20180606204148-bd9c31933947/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
+github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
+github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
+github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
+github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChlfo5C6hzIHwPqfFE=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
+github.com/peterh/liner v1.0.1-0.20180619022028-8c1271fcf47f/go.mod h1:xIteQHvHuaLYG9IFj6mSxM0fCKrs34IrEQUhOYuGPHc=
+github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0=
+github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
+github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/term v0.0.0-20180730021639-bffc007b7fd5/go.mod h1:eCbImbZ95eXtAUIbLAuAVnBnwf83mjf6QIVH8SHYwqQ=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
+github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
 github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/prometheus/tsdb v0.7.1 h1:YZcsG11NqnK4czYLrWd9mpEuAJIHVQLwdrleYfszMAA=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
+github.com/retailnext/hllpp v1.0.1-0.20180308014038-101a6d2f8b52/go.mod h1:RDpi1RftBQPUCDRw6SmxeaREsAaRKnOclghuzp/WRzc=
+github.com/rjeczalik/notify v0.9.1 h1:CLCKso/QK1snAlnhNR/CNvNiFU2saUtjV0bx3EwNeCE=
+github.com/rjeczalik/notify v0.9.1/go.mod h1:rKwnCoCGeuQnwBtTSPL9Dad03Vh2n40ePRrjvIXnJho=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik=
+github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
+github.com/segmentio/kafka-go v0.1.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo=
+github.com/segmentio/kafka-go v0.2.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo=
+github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
+github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible h1:Bn1aCHHRnjv4Bl16T8rcaFjYSrGrIZvpiGO6P3Q4GpU=
+github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
@@ -157,38 +410,90 @@ github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4k
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
+github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
 github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
+github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4 h1:Gb2Tyox57NRNuZ2d3rmvB3pcmbu7O1RS3m8WRx7ilrg=
+github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4/go.mod h1:RZLeN1LMWmRsyYjvAu+I6Dm9QmlDaIIt+Y+4Kd7Tp+Q=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/testify v1.2.0/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
+github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
+github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY=
+github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc=
+github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
+github.com/tklauser/go-sysconf v0.3.5/go.mod h1:MkWzOF4RMCshBAMXuhXJs64Rte09mITnppBXY/rYEFI=
+github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU=
+github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI=
+github.com/tklauser/numcpus v0.2.2/go.mod h1:x3qojaO3uyYt0i56EW/VUYs7uBvdl2fkfZFu0T9wgjM=
+github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk=
+github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
+github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef h1:wHSqTBrZW24CsNJDfeh9Ex6Pm0Rcpc7qrgKBiL44vF4=
+github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs=
+github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI=
 github.com/urfave/cli/v2 v2.27.5 h1:WoHEJLdsXr6dDWoJgMq/CboDmyY/8HMMH1fTECbih+w=
 github.com/urfave/cli/v2 v2.27.5/go.mod h1:3Sevf16NykTbInEnD0yKkjDAeZDS0A6bzhBH5hrMvTQ=
+github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
+github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8=
+github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
+github.com/willf/bitset v1.1.3/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
+github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg=
 github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4=
 github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM=
+github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
+go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
+go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
+golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190909091759-094676da4a83/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
+golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
+golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI=
+golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
+golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
 golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
 golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
+golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -198,12 +503,19 @@ golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHl
 golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
 golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
 golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
 golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
 golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
+golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -215,36 +527,122 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn
 golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210220033124-5f55cee0dc0d/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
+golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
+golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200107162124-548cf772de50/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200826173525-f9321e4c35a6/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210316164454-77fc1eacc6aa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210420205809-ac73e9fd8988/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34=
+golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
+golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
+golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
+golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk=
+golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba h1:O8mE0/t419eoIwhTFpKVkHiTs/Igowgfkj25AcZrtiE=
+golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
@@ -252,6 +650,7 @@ golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3
 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
@@ -259,41 +658,94 @@ golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtn
 golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200108203644-89082a384178/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo=
+gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo=
+gonum.org/v1/gonum v0.6.0/go.mod h1:9mxDZsDKxgMAuccQkewq682L+0eCu4dCN2yonUJTCLU=
+gonum.org/v1/netlib v0.0.0-20181029234149-ec6d1f5cefe6/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw=
+gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw=
+gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc=
 google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
 google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
 google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
 google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
 google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
+google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190716160619-c506a9f90610/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
 google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
 google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
 google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
 google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200108215221-bd8f9a0ef82f/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
+gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce h1:+JknDZhAj8YMt7GC73Ei8pv4MzjDUNPHgQWJdtMAaDU=
+gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce/go.mod h1:5AcXVHNjg+BDxry382+8OKon8SEWiKktQR07RKPsv1c=
+gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200619000410-60c24ae608a6/go.mod h1:uAJfkITjFhyEEuUfm7bsmCZRbW5WRq8s9EY8HZ6hCns=
 gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/urfave/cli.v1 v1.20.0 h1:NdAVW6RYxDif9DhDHaAortIu956m2c0v+09AZBPTbE0=
+gopkg.in/urfave/cli.v1 v1.20.0/go.mod h1:vuBzUtMdQeixQj8LVd+/98pzhxNGQoyuPBlsXHOQNO0=
 gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
+honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
+rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=

nodebalancer/sample.env

@@ -6,6 +6,9 @@ export MOONSTREAM_CORS_ALLOWED_ORIGINS="http://localhost:3000,https://moonstream
 export NB_CONTROLLER_USER_ID="<bugout_id_of_nodebalancer_user>"
 export NB_CONTROLLER_TOKEN="<token_of_nodebalancer_user>"
 export NB_CONTROLLER_ACCESS_ID="<nodebalancer_access_id_for_internal_usage>"
+export NB_CACHE_CLEANING_INTERVAL=10
+export NB_CACHE_ACCESS_ID_LIFETIME=120
+export NB_CACHE_ACCESS_ID_SESSION_LIFETIME=900
 
 # Error humbug reporter
 export HUMBUG_REPORTER_NODE_BALANCER_TOKEN="<bugout_humbug_token_for_crash_reports>"