mirror
/
moonstream
kopia lustrzana https://github.com/bugout-dev/moonstream
1
0
Forkuj 0
Kod Zgłoszenia Projekty Wydania Wiki Aktywność

Merge branch 'main' into monitoring-systemd-init

pull/946/head
kompotkot 2023-11-09 15:03:08 +00:00
rodzic c196c9e0d2 1091ad699d
commit 1d2c418873
18 zmienionych plików z 59 dodań i 184 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

2
crawlers/deploy/ethereum-historical-crawl-events.service
Wyświetl plik

@ -11,7 +11,7 @@ Restart=on-failure
RestartSec=15s
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" historical-crawl --blockchain-type ethereum --find-deployed-blocks --end 0 --tasks-journal --only-events
CPUWeight=70
SyslogIdentifier=ethereum-historical-crawler-events
SyslogIdentifier=ethereum-historical-crawl-events
[Install]
WantedBy=multi-user.target

2
crawlers/deploy/ethereum-historical-crawl-transactions.service
Wyświetl plik

@ -11,7 +11,7 @@ Restart=on-failure
RestartSec=15s
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" historical-crawl --blockchain-type ethereum --find-deployed-blocks --end 0 --tasks-journal --only-functions
CPUWeight=70
SyslogIdentifier=ethereum-historical-crawler-transactions
SyslogIdentifier=ethereum-historical-crawl-transactions
[Install]
WantedBy=multi-user.target

17
crawlers/deploy/ethereum-txpool.service
Wyświetl plik

@ -1,17 +0,0 @@
[Unit]
Description=Ethereum txpool crawler
After=network.target
StartLimitIntervalSec=300
StartLimitBurst=3
[Service]
WorkingDirectory=/home/ubuntu/moonstream/crawlers/txpool
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
Restart=on-failure
RestartSec=15s
ExecStart=/home/ubuntu/moonstream/crawlers/txpool/txpool -blockchain ethereum -access-id "${NB_CONTROLLER_ACCESS_ID}"
CPUWeight=30
SyslogIdentifier=ethereum-txpool
[Install]
WantedBy=multi-user.target

17
crawlers/deploy/moonworm-unicorns-mainnet.service
Wyświetl plik

@ -1,17 +0,0 @@
[Unit]
Description=Moonworm CryptoUnicorns watch custom systemd service
StartLimitIntervalSec=300
StartLimitBurst=3
After=network.target
[Service]
Restart=on-failure
RestartSec=15s
WorkingDirectory=/home/ubuntu
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
ExecStart=/home/ubuntu/moonworm-env/bin/python -m moonworm.cli watch-cu -w "${MOONSTREAM_POLYGON_WEB3_PROVIDER_URI}?access_id=${NB_CONTROLLER_ACCESS_ID}&data_source=blockchain" -c 0xdC0479CC5BbA033B3e7De9F178607150B3AbCe1f -d 21418707 --confirmations 60
CPUWeight=70
SyslogIdentifier=moonworm-unicorns-mainnet
[Install]
WantedBy=multi-user.target

2
crawlers/deploy/mumbai-historical-crawl-events.service
Wyświetl plik

@ -11,7 +11,7 @@ Restart=on-failure
RestartSec=15s
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" historical-crawl --blockchain-type mumbai --find-deployed-blocks --end 0 --tasks-journal --only-events
CPUWeight=70
SyslogIdentifier=mumbai-historical-crawler-events
SyslogIdentifier=mumbai-historical-crawl-events
[Install]
WantedBy=multi-user.target

2
crawlers/deploy/mumbai-historical-crawl-transactions.service
Wyświetl plik

@ -11,7 +11,7 @@ Restart=on-failure
RestartSec=15s
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" historical-crawl --blockchain-type mumbai --find-deployed-blocks --end 0 --tasks-journal --only-functions
CPUWeight=70
SyslogIdentifier=mumbai-historical-crawler-transactions
SyslogIdentifier=mumbai-historical-crawl-transactions
[Install]
WantedBy=multi-user.target

102
crawlers/deploy/parameters.py
Wyświetl plik

@ -1,102 +0,0 @@
"""
Collect secrets from AWS SSM Parameter Store and output as environment variable exports.
"""
import argparse
from dataclasses import dataclass
import sys
from typing import Any, Dict, Iterable, List, Optional
import boto3
@dataclass
class EnvironmentVariable:
name: str
value: str
def get_parameters(path: str) -> List[Dict[str, Any]]:
"""
Retrieve parameters from AWS SSM Parameter Store. Decrypts any encrypted parameters.
Relies on the appropriate environment variables to authenticate against AWS:
https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html
"""
ssm = boto3.client("ssm")
next_token: Optional[bool] = True
parameters: List[Dict[str, Any]] = []
while next_token is not None:
kwargs = {"Path": path, "Recursive": False, "WithDecryption": True}
if next_token is not True:
kwargs["NextToken"] = next_token
response = ssm.get_parameters_by_path(**kwargs)
new_parameters = response.get("Parameters", [])
parameters.extend(new_parameters)
next_token = response.get("NextToken")
return parameters
def parameter_to_env(parameter_object: Dict[str, Any]) -> EnvironmentVariable:
"""
Transforms parameters returned by the AWS SSM API into EnvironmentVariables.
"""
parameter_path = parameter_object.get("Name")
if parameter_path is None:
raise ValueError('Did not find "Name" in parameter object')
name = parameter_path.split("/")[-1].upper()
value = parameter_object.get("Value")
if value is None:
raise ValueError('Did not find "Value" in parameter object')
return EnvironmentVariable(name, value)
def env_string(env_vars: Iterable[EnvironmentVariable], with_export: bool) -> str:
"""
Produces a string which, when executed in a shell, exports the desired environment variables as
specified by env_vars.
"""
prefix = "export " if with_export else ""
return "\n".join([f'{prefix}{var.name}="{var.value}"' for var in env_vars])
def extract_handler(args: argparse.Namespace) -> None:
"""
Save environment variables to file.
"""
result = env_string(map(parameter_to_env, get_parameters(args.path)), args.export)
with args.outfile as ofp:
print(result, file=ofp)
if __name__ == "__main__":
parser = argparse.ArgumentParser(
description="Materialize environment variables from AWS SSM Parameter Store"
)
parser.set_defaults(func=lambda _: parser.print_help())
subcommands = parser.add_subparsers(description="Parameters commands")
parser_extract = subcommands.add_parser(
"extract", description="Parameters extract commands"
)
parser_extract.set_defaults(func=lambda _: parser_extract.print_help())
parser_extract.add_argument(
"-o", "--outfile", type=argparse.FileType("w"), default=sys.stdout
)
parser_extract.add_argument(
"--export",
action="store_true",
help="Set to output environment strings with export statements",
)
parser_extract.add_argument(
"-p",
"--path",
default=None,
help="SSM path from which to pull environment variables (pull is NOT recursive)",
)
parser_extract.set_defaults(func=extract_handler)
args = parser.parse_args()
args.func(args)

2
crawlers/deploy/polygon-historical-crawl-events.service
Wyświetl plik

@ -11,7 +11,7 @@ Restart=on-failure
RestartSec=15s
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" historical-crawl --blockchain-type polygon --find-deployed-blocks --end 0 --tasks-journal --only-events
CPUWeight=70
SyslogIdentifier=polygon-historical-crawler-events
SyslogIdentifier=polygon-historical-crawl-events
[Install]
WantedBy=multi-user.target

2
crawlers/deploy/polygon-historical-crawl-transactions.service
Wyświetl plik

@ -11,7 +11,7 @@ Restart=on-failure
RestartSec=15s
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" historical-crawl --blockchain-type polygon --find-deployed-blocks --end 0 --tasks-journal --only-functions
CPUWeight=70
SyslogIdentifier=polygon-historical-crawler-transactions
SyslogIdentifier=polygon-historical-crawl-transactions
[Install]
WantedBy=multi-user.target

17
crawlers/deploy/polygon-txpool.service
Wyświetl plik

@ -1,17 +0,0 @@
[Unit]
Description=Polygon txpool crawler
After=network.target
StartLimitIntervalSec=300
StartLimitBurst=3
[Service]
WorkingDirectory=/home/ubuntu/moonstream/crawlers/txpool
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
Restart=on-failure
RestartSec=15s
ExecStart=/home/ubuntu/moonstream/crawlers/txpool/txpool -blockchain polygon -access-id "${NB_CONTROLLER_ACCESS_ID}"
CPUWeight=30
SyslogIdentifier=polygon-txpool
[Install]
WantedBy=multi-user.target

2
crawlers/deploy/wyrm-historical-crawl-events.service
Wyświetl plik

@ -11,7 +11,7 @@ Restart=on-failure
RestartSec=15s
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" historical-crawl --blockchain-type wyrm --find-deployed-blocks --end 0 --tasks-journal --only-events
CPUWeight=70
SyslogIdentifier=wyrm-historical-crawler-events
SyslogIdentifier=wyrm-historical-crawl-events
[Install]
WantedBy=multi-user.target

2
crawlers/deploy/wyrm-historical-crawl-transactions.service
Wyświetl plik

@ -11,7 +11,7 @@ Restart=on-failure
RestartSec=15s
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" historical-crawl --blockchain-type wyrm --find-deployed-blocks --end 0 --tasks-journal --only-functions
CPUWeight=70
SyslogIdentifier=wyrm-historical-crawler-transactions
SyslogIdentifier=wyrm-historical-crawl-transactions
[Install]
WantedBy=multi-user.target

2
crawlers/deploy/xdai-historical-crawl-events.service
Wyświetl plik

@ -11,7 +11,7 @@ Restart=on-failure
RestartSec=15s
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" historical-crawl --blockchain-type xdai --find-deployed-blocks --end 0 --tasks-journal --only-events
CPUWeight=70
SyslogIdentifier=xdai-historical-crawler-events
SyslogIdentifier=xdai-historical-crawl-events
[Install]
WantedBy=multi-user.target

2
crawlers/deploy/xdai-historical-crawl-transactions.service
Wyświetl plik

@ -11,7 +11,7 @@ Restart=on-failure
RestartSec=15s
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" historical-crawl --blockchain-type xdai --find-deployed-blocks --end 0 --tasks-journal --only-functions
CPUWeight=70
SyslogIdentifier=xdai-historical-crawler-transactions
SyslogIdentifier=xdai-historical-crawl-transactions
[Install]
WantedBy=multi-user.target

7
crawlers/mooncrawl/mooncrawl/leaderboards_generator/cli.py
Wyświetl plik

@ -97,6 +97,7 @@ def handle_leaderboards(args: argparse.Namespace) -> None:
MOONSTREAM_API_URL,
args.max_retries,
args.interval,
args.query_api_retries,
)
except Exception as e:
logger.error(f"Could not get results for query {query_name}: error: {e}")
@ -188,6 +189,12 @@ def main():
default=12,
help="Number of times to retry requests for Moonstream Query results",
)
leaderboard_generator_parser.add_argument(
"--query-api-retries",
type=int,
default=3,
help="Number of times to retry updating Moonstream Query data",
)
leaderboard_generator_parser.add_argument(
"--interval",
type=float,

5
crawlers/mooncrawl/mooncrawl/leaderboards_generator/utils.py
Wyświetl plik

@ -23,6 +23,7 @@ def get_results_for_moonstream_query(
api_url: str = MOONSTREAM_API_URL,
max_retries: int = 100,
interval: float = 30.0,
query_api_retries: int = 3,
) -> Optional[Dict[str, Any]]:
"""
@ -65,11 +66,11 @@ def get_results_for_moonstream_query(
success = False
attempts = 0
while not success and attempts < max_retries:
attempts += 1
while not success and attempts < query_api_retries:
response = requests.post(
request_url, json=request_body, headers=headers, timeout=10
)
attempts += 1
response.raise_for_status()
response_body = response.json()
data_url = response_body["url"]

2
crawlers/mooncrawl/mooncrawl/state_crawler/cli.py
Wyświetl plik

@ -469,7 +469,7 @@ def parse_jobs(
# run crawling of levels
try:
# initial call of level 0 all call without subcalls directly moved there
logger.info("Crawl level: 0. Jobs amount: {len(calls[0])}")
logger.info(f"Crawl level: 0. Jobs amount: {len(calls[0])}")
logger.info(f"call_tree_levels: {call_tree_levels}")
batch_size = crawl_calls_level(

56
crawlers/mooncrawl/mooncrawl/state_crawler/jobs/mumbai-jobs.json
Wyświetl plik

@ -66,10 +66,20 @@
{
"inputs": [
{
"internalType": "uint256",
"internalType": "uint16",
"name": "seasonId",
"type": "uint256",
"value": 1
"type": "uint16",
"value": {
"type": "queryAPI",
"query_url": "twilight_seasons",
"blockchain": "mumbai",
"params": {
"address": "0x665B8Db5b9E3b396e2Ccb0Bd768dc74fC47Ec20D"
},
"keys": [
"season_id"
]
}
},
{
"internalType": "address",
@ -89,48 +99,58 @@
}
],
"name": "twtGetSeasonalDominationPointsByAccount",
"address": "0xb93D53A10793C7EA7DaE973625C9BB3b18Eec2A7",
"address": "0x665B8Db5b9E3b396e2Ccb0Bd768dc74fC47Ec20D",
"outputs": [
{
"internalType": "uint256[5]",
"internalType": "uint56[5]",
"name": "shadowcornDominationPoints",
"type": "uint256[5]"
"type": "uint56[5]"
},
{
"internalType": "uint256[5]",
"internalType": "uint56[5]",
"name": "unicornDominationPoints",
"type": "uint256[5]"
"type": "uint56[5]"
}
],
"stateMutability": "view",
"type": "function",
"selector": "0x1c109952"
"selector": "0x0b4ef829"
},
{
"inputs": [
{
"internalType": "uint256",
"internalType": "uint16",
"name": "seasonId",
"type": "uint256",
"value": 1
"type": "uint16",
"value": {
"type": "queryAPI",
"query_url": "twilight_seasons",
"blockchain": "mumbai",
"params": {
"address": "0x665B8Db5b9E3b396e2Ccb0Bd768dc74fC47Ec20D"
},
"keys": [
"season_id"
]
}
}
],
"name": "twtGetSeasonalDominationPointsForAllRegions",
"address": "0xb93D53A10793C7EA7DaE973625C9BB3b18Eec2A7",
"address": "0x665B8Db5b9E3b396e2Ccb0Bd768dc74fC47Ec20D",
"outputs": [
{
"internalType": "uint256[5]",
"internalType": "uint56[5]",
"name": "shadowcornDominationPoints",
"type": "uint256[5]"
"type": "uint56[5]"
},
{
"internalType": "uint256[5]",
"internalType": "uint56[5]",
"name": "unicornDominationPoints",
"type": "uint256[5]"
"type": "uint56[5]"
}
],
"stateMutability": "view",
"type": "function",
"selector": "0xa491c0f6"
"selector": "0xbddb218c"
}
]