kopia lustrzana https://github.com/bugout-dev/moonstream
Merge pull request #939 from moonstream-to/removed-outdated-crawlers
Cleaned from unused crawler servicespull/963/head
Sergei Sumarokov
GitHub
commit
559c51b2d6
|
|
@ -34,7 +34,6 @@ LEADERBOARDS_WORKER_TIMER_FILE="leaderboards-worker.timer"
|
|||
ETHEREUM_SYNCHRONIZE_SERVICE_FILE="ethereum-synchronize.service"
|
||||
ETHEREUM_TRENDING_SERVICE_FILE="ethereum-trending.service"
|
||||
ETHEREUM_TRENDING_TIMER_FILE="ethereum-trending.timer"
|
||||
ETHEREUM_TXPOOL_SERVICE_FILE="ethereum-txpool.service"
|
||||
ETHEREUM_MISSING_SERVICE_FILE="ethereum-missing.service"
|
||||
ETHEREUM_MISSING_TIMER_FILE="ethereum-missing.timer"
|
||||
ETHEREUM_MOONWORM_CRAWLER_SERVICE_FILE="ethereum-moonworm-crawler.service"
|
||||
|
|
@ -51,7 +50,6 @@ POLYGON_MISSING_SERVICE_FILE="polygon-missing.service"
|
|||
POLYGON_MISSING_TIMER_FILE="polygon-missing.timer"
|
||||
POLYGON_STATISTICS_SERVICE_FILE="polygon-statistics.service"
|
||||
POLYGON_STATISTICS_TIMER_FILE="polygon-statistics.timer"
|
||||
POLYGON_TXPOOL_SERVICE_FILE="polygon-txpool.service"
|
||||
POLYGON_MOONWORM_CRAWLER_SERVICE_FILE="polygon-moonworm-crawler.service"
|
||||
POLYGON_STATE_SERVICE_FILE="polygon-state.service"
|
||||
POLYGON_STATE_TIMER_FILE="polygon-state.timer"
|
||||
|
|
@ -191,14 +189,6 @@ cp "${SCRIPT_DIR}/${ETHEREUM_TRENDING_TIMER_FILE}" "/home/ubuntu/.config/systemd
|
|||
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
|
||||
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ETHEREUM_TRENDING_TIMER_FILE}"
|
||||
|
||||
# echo
|
||||
# echo
|
||||
# echo -e "${PREFIX_INFO} Replacing existing Ethereum transaction pool crawler service definition with ${ETHEREUM_TXPOOL_SERVICE_FILE}"
|
||||
# chmod 644 "${SCRIPT_DIR}/${ETHEREUM_TXPOOL_SERVICE_FILE}"
|
||||
# cp "${SCRIPT_DIR}/${ETHEREUM_TXPOOL_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${ETHEREUM_TXPOOL_SERVICE_FILE}"
|
||||
# XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
|
||||
# XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ETHEREUM_TXPOOL_SERVICE_FILE}"
|
||||
|
||||
echo
|
||||
echo
|
||||
echo -e "${PREFIX_INFO} Replacing existing Ethereum missing service and timer with: ${ETHEREUM_MISSING_SERVICE_FILE}, ${ETHEREUM_MISSING_TIMER_FILE}"
|
||||
|
|
@ -270,14 +260,6 @@ cp "${SCRIPT_DIR}/${POLYGON_STATISTICS_TIMER_FILE}" "/home/ubuntu/.config/system
|
|||
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
|
||||
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${POLYGON_STATISTICS_TIMER_FILE}"
|
||||
|
||||
# echo
|
||||
# echo
|
||||
# echo -e "${PREFIX_INFO} Replacing existing Polygon transaction pool crawler service definition with ${POLYGON_TXPOOL_SERVICE_FILE}"
|
||||
# chmod 644 "${SCRIPT_DIR}/${POLYGON_TXPOOL_SERVICE_FILE}"
|
||||
# cp "${SCRIPT_DIR}/${POLYGON_TXPOOL_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${POLYGON_TXPOOL_SERVICE_FILE}"
|
||||
# XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
|
||||
# XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${POLYGON_TXPOOL_SERVICE_FILE}"
|
||||
|
||||
echo
|
||||
echo
|
||||
echo -e "${PREFIX_INFO} Replacing existing Polygon moonworm crawler service definition with ${POLYGON_MOONWORM_CRAWLER_SERVICE_FILE}"
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ Restart=on-failure
|
|||
RestartSec=15s
|
||||
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" historical-crawl --blockchain-type ethereum --find-deployed-blocks --end 0 --tasks-journal --only-events
|
||||
CPUWeight=70
|
||||
SyslogIdentifier=ethereum-historical-crawler-events
|
||||
SyslogIdentifier=ethereum-historical-crawl-events
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
|
|
@ -11,7 +11,7 @@ Restart=on-failure
|
|||
RestartSec=15s
|
||||
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" historical-crawl --blockchain-type ethereum --find-deployed-blocks --end 0 --tasks-journal --only-functions
|
||||
CPUWeight=70
|
||||
SyslogIdentifier=ethereum-historical-crawler-transactions
|
||||
SyslogIdentifier=ethereum-historical-crawl-transactions
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
|
|
@ -1,17 +0,0 @@
|
|||
[Unit]
|
||||
Description=Ethereum txpool crawler
|
||||
After=network.target
|
||||
StartLimitIntervalSec=300
|
||||
StartLimitBurst=3
|
||||
|
||||
[Service]
|
||||
WorkingDirectory=/home/ubuntu/moonstream/crawlers/txpool
|
||||
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
|
||||
Restart=on-failure
|
||||
RestartSec=15s
|
||||
ExecStart=/home/ubuntu/moonstream/crawlers/txpool/txpool -blockchain ethereum -access-id "${NB_CONTROLLER_ACCESS_ID}"
|
||||
CPUWeight=30
|
||||
SyslogIdentifier=ethereum-txpool
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
|
|
@ -1,17 +0,0 @@
|
|||
[Unit]
|
||||
Description=Moonworm CryptoUnicorns watch custom systemd service
|
||||
StartLimitIntervalSec=300
|
||||
StartLimitBurst=3
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
Restart=on-failure
|
||||
RestartSec=15s
|
||||
WorkingDirectory=/home/ubuntu
|
||||
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
|
||||
ExecStart=/home/ubuntu/moonworm-env/bin/python -m moonworm.cli watch-cu -w "${MOONSTREAM_POLYGON_WEB3_PROVIDER_URI}?access_id=${NB_CONTROLLER_ACCESS_ID}&data_source=blockchain" -c 0xdC0479CC5BbA033B3e7De9F178607150B3AbCe1f -d 21418707 --confirmations 60
|
||||
CPUWeight=70
|
||||
SyslogIdentifier=moonworm-unicorns-mainnet
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
|
|
@ -11,7 +11,7 @@ Restart=on-failure
|
|||
RestartSec=15s
|
||||
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" historical-crawl --blockchain-type mumbai --find-deployed-blocks --end 0 --tasks-journal --only-events
|
||||
CPUWeight=70
|
||||
SyslogIdentifier=mumbai-historical-crawler-events
|
||||
SyslogIdentifier=mumbai-historical-crawl-events
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
|
|
@ -11,7 +11,7 @@ Restart=on-failure
|
|||
RestartSec=15s
|
||||
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" historical-crawl --blockchain-type mumbai --find-deployed-blocks --end 0 --tasks-journal --only-functions
|
||||
CPUWeight=70
|
||||
SyslogIdentifier=mumbai-historical-crawler-transactions
|
||||
SyslogIdentifier=mumbai-historical-crawl-transactions
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
|
|
@ -1,102 +0,0 @@
|
|||
"""
|
||||
Collect secrets from AWS SSM Parameter Store and output as environment variable exports.
|
||||
"""
|
||||
import argparse
|
||||
from dataclasses import dataclass
|
||||
import sys
|
||||
from typing import Any, Dict, Iterable, List, Optional
|
||||
|
||||
import boto3
|
||||
|
||||
|
||||
@dataclass
|
||||
class EnvironmentVariable:
|
||||
name: str
|
||||
value: str
|
||||
|
||||
|
||||
def get_parameters(path: str) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Retrieve parameters from AWS SSM Parameter Store. Decrypts any encrypted parameters.
|
||||
|
||||
Relies on the appropriate environment variables to authenticate against AWS:
|
||||
https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html
|
||||
"""
|
||||
ssm = boto3.client("ssm")
|
||||
next_token: Optional[bool] = True
|
||||
parameters: List[Dict[str, Any]] = []
|
||||
while next_token is not None:
|
||||
kwargs = {"Path": path, "Recursive": False, "WithDecryption": True}
|
||||
if next_token is not True:
|
||||
kwargs["NextToken"] = next_token
|
||||
response = ssm.get_parameters_by_path(**kwargs)
|
||||
new_parameters = response.get("Parameters", [])
|
||||
parameters.extend(new_parameters)
|
||||
next_token = response.get("NextToken")
|
||||
|
||||
return parameters
|
||||
|
||||
|
||||
def parameter_to_env(parameter_object: Dict[str, Any]) -> EnvironmentVariable:
|
||||
"""
|
||||
Transforms parameters returned by the AWS SSM API into EnvironmentVariables.
|
||||
"""
|
||||
parameter_path = parameter_object.get("Name")
|
||||
if parameter_path is None:
|
||||
raise ValueError('Did not find "Name" in parameter object')
|
||||
name = parameter_path.split("/")[-1].upper()
|
||||
|
||||
value = parameter_object.get("Value")
|
||||
if value is None:
|
||||
raise ValueError('Did not find "Value" in parameter object')
|
||||
|
||||
return EnvironmentVariable(name, value)
|
||||
|
||||
|
||||
def env_string(env_vars: Iterable[EnvironmentVariable], with_export: bool) -> str:
|
||||
"""
|
||||
Produces a string which, when executed in a shell, exports the desired environment variables as
|
||||
specified by env_vars.
|
||||
"""
|
||||
prefix = "export " if with_export else ""
|
||||
return "\n".join([f'{prefix}{var.name}="{var.value}"' for var in env_vars])
|
||||
|
||||
|
||||
def extract_handler(args: argparse.Namespace) -> None:
|
||||
"""
|
||||
Save environment variables to file.
|
||||
"""
|
||||
result = env_string(map(parameter_to_env, get_parameters(args.path)), args.export)
|
||||
with args.outfile as ofp:
|
||||
print(result, file=ofp)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Materialize environment variables from AWS SSM Parameter Store"
|
||||
)
|
||||
parser.set_defaults(func=lambda _: parser.print_help())
|
||||
subcommands = parser.add_subparsers(description="Parameters commands")
|
||||
|
||||
parser_extract = subcommands.add_parser(
|
||||
"extract", description="Parameters extract commands"
|
||||
)
|
||||
parser_extract.set_defaults(func=lambda _: parser_extract.print_help())
|
||||
parser_extract.add_argument(
|
||||
"-o", "--outfile", type=argparse.FileType("w"), default=sys.stdout
|
||||
)
|
||||
parser_extract.add_argument(
|
||||
"--export",
|
||||
action="store_true",
|
||||
help="Set to output environment strings with export statements",
|
||||
)
|
||||
parser_extract.add_argument(
|
||||
"-p",
|
||||
"--path",
|
||||
default=None,
|
||||
help="SSM path from which to pull environment variables (pull is NOT recursive)",
|
||||
)
|
||||
parser_extract.set_defaults(func=extract_handler)
|
||||
|
||||
args = parser.parse_args()
|
||||
args.func(args)
|
||||
|
|
@ -11,7 +11,7 @@ Restart=on-failure
|
|||
RestartSec=15s
|
||||
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" historical-crawl --blockchain-type polygon --find-deployed-blocks --end 0 --tasks-journal --only-events
|
||||
CPUWeight=70
|
||||
SyslogIdentifier=polygon-historical-crawler-events
|
||||
SyslogIdentifier=polygon-historical-crawl-events
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
|
|
@ -11,7 +11,7 @@ Restart=on-failure
|
|||
RestartSec=15s
|
||||
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" historical-crawl --blockchain-type polygon --find-deployed-blocks --end 0 --tasks-journal --only-functions
|
||||
CPUWeight=70
|
||||
SyslogIdentifier=polygon-historical-crawler-transactions
|
||||
SyslogIdentifier=polygon-historical-crawl-transactions
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
|
|
@ -1,17 +0,0 @@
|
|||
[Unit]
|
||||
Description=Polygon txpool crawler
|
||||
After=network.target
|
||||
StartLimitIntervalSec=300
|
||||
StartLimitBurst=3
|
||||
|
||||
[Service]
|
||||
WorkingDirectory=/home/ubuntu/moonstream/crawlers/txpool
|
||||
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
|
||||
Restart=on-failure
|
||||
RestartSec=15s
|
||||
ExecStart=/home/ubuntu/moonstream/crawlers/txpool/txpool -blockchain polygon -access-id "${NB_CONTROLLER_ACCESS_ID}"
|
||||
CPUWeight=30
|
||||
SyslogIdentifier=polygon-txpool
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
|
|
@ -11,7 +11,7 @@ Restart=on-failure
|
|||
RestartSec=15s
|
||||
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" historical-crawl --blockchain-type wyrm --find-deployed-blocks --end 0 --tasks-journal --only-events
|
||||
CPUWeight=70
|
||||
SyslogIdentifier=wyrm-historical-crawler-events
|
||||
SyslogIdentifier=wyrm-historical-crawl-events
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
|
|
@ -11,7 +11,7 @@ Restart=on-failure
|
|||
RestartSec=15s
|
||||
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" historical-crawl --blockchain-type wyrm --find-deployed-blocks --end 0 --tasks-journal --only-functions
|
||||
CPUWeight=70
|
||||
SyslogIdentifier=wyrm-historical-crawler-transactions
|
||||
SyslogIdentifier=wyrm-historical-crawl-transactions
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
|
|
@ -11,7 +11,7 @@ Restart=on-failure
|
|||
RestartSec=15s
|
||||
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" historical-crawl --blockchain-type xdai --find-deployed-blocks --end 0 --tasks-journal --only-events
|
||||
CPUWeight=70
|
||||
SyslogIdentifier=xdai-historical-crawler-events
|
||||
SyslogIdentifier=xdai-historical-crawl-events
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
|
|
@ -11,7 +11,7 @@ Restart=on-failure
|
|||
RestartSec=15s
|
||||
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" historical-crawl --blockchain-type xdai --find-deployed-blocks --end 0 --tasks-journal --only-functions
|
||||
CPUWeight=70
|
||||
SyslogIdentifier=xdai-historical-crawler-transactions
|
||||
SyslogIdentifier=xdai-historical-crawl-transactions
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
Ładowanie…
Reference in New Issue