kopia lustrzana https://github.com/bugout-dev/moonstream
Add deploy services and fix if condition.
rodzic
f68ab988c9
commit
e64bb7fcad
|
@ -0,0 +1,13 @@
|
||||||
|
[Unit]
|
||||||
|
Description=Execute state crawler
|
||||||
|
After=network.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=oneshot
|
||||||
|
User=ubuntu
|
||||||
|
Group=www-data
|
||||||
|
WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
|
||||||
|
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
|
||||||
|
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl-jobs --blockchain mumbai --infura --jobs-file ./mooncrawl/state_crawler/jobs/mumbai-jobs.json
|
||||||
|
CPUWeight=60
|
||||||
|
SyslogIdentifier=polygon-state
|
|
@ -0,0 +1,9 @@
|
||||||
|
[Unit]
|
||||||
|
Description=Execute Polygon state crawler each 10m
|
||||||
|
|
||||||
|
[Timer]
|
||||||
|
OnBootSec=15s
|
||||||
|
OnUnitActiveSec=10m
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=timers.target
|
|
@ -8,6 +8,6 @@ User=ubuntu
|
||||||
Group=www-data
|
Group=www-data
|
||||||
WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
|
WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
|
||||||
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
|
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
|
||||||
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl-jobs --blockchain polygon
|
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl-jobs --blockchain polygon --infura --jobs-file ./mooncrawl/state_crawler/jobs/polygon-jobs.json
|
||||||
CPUWeight=60
|
CPUWeight=60
|
||||||
SyslogIdentifier=polygon-state
|
SyslogIdentifier=polygon-state
|
||||||
|
|
|
@ -210,3 +210,24 @@ if MOONSTREAM_S3_PUBLIC_DATA_BUCKET == "":
|
||||||
MOONSTREAM_S3_PUBLIC_DATA_BUCKET_PREFIX = os.environ.get(
|
MOONSTREAM_S3_PUBLIC_DATA_BUCKET_PREFIX = os.environ.get(
|
||||||
"MOONSTREAM_S3_PUBLIC_DATA_BUCKET_PREFIX", "dev"
|
"MOONSTREAM_S3_PUBLIC_DATA_BUCKET_PREFIX", "dev"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# infura config
|
||||||
|
|
||||||
|
|
||||||
|
INFURA_PROJECT_ID = os.environ.get("INFURA_PROJECT_ID")
|
||||||
|
|
||||||
|
infura_networks = {
|
||||||
|
AvailableBlockchainType.ETHEREUM: {
|
||||||
|
"name": "mainnet",
|
||||||
|
"url": f"https://mainnet.infura.io/v3/{INFURA_PROJECT_ID}",
|
||||||
|
},
|
||||||
|
AvailableBlockchainType.POLYGON: {
|
||||||
|
"name": "polygon",
|
||||||
|
"url": f"https://polygon-mainnet.infura.io/v3/{INFURA_PROJECT_ID}",
|
||||||
|
},
|
||||||
|
AvailableBlockchainType.MUMBAI: {
|
||||||
|
"name": "mumbai",
|
||||||
|
"url": f"https://polygon-mumbai.infura.io/v3/{INFURA_PROJECT_ID}",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
|
@ -28,9 +28,11 @@ from .Multicall2_interface import Contract as Multicall2
|
||||||
from ..settings import (
|
from ..settings import (
|
||||||
NB_CONTROLLER_ACCESS_ID,
|
NB_CONTROLLER_ACCESS_ID,
|
||||||
MOONSTREAM_STATE_CRAWLER_DB_STATEMENT_TIMEOUT_MILLIS,
|
MOONSTREAM_STATE_CRAWLER_DB_STATEMENT_TIMEOUT_MILLIS,
|
||||||
|
INFURA_PROJECT_ID,
|
||||||
multicall_contracts,
|
multicall_contracts,
|
||||||
|
infura_networks,
|
||||||
)
|
)
|
||||||
from .web3_util import FunctionSignature
|
from .web3_util import FunctionSignature, connect
|
||||||
|
|
||||||
logging.basicConfig(level=logging.INFO)
|
logging.basicConfig(level=logging.INFO)
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
@ -272,18 +274,13 @@ def parse_jobs(
|
||||||
contracts_methods: Dict[str, Any] = {}
|
contracts_methods: Dict[str, Any] = {}
|
||||||
calls: Dict[int, Any] = {0: []}
|
calls: Dict[int, Any] = {0: []}
|
||||||
|
|
||||||
if web3_provider_uri is None:
|
if web3_provider_uri is not None:
|
||||||
|
|
||||||
logger.info(f"Connecting to blockchain: {blockchain_type} with Node balancer.")
|
|
||||||
web3_client = _retry_connect_web3(
|
|
||||||
blockchain_type=blockchain_type, access_id=access_id
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
try:
|
try:
|
||||||
logger.info(
|
logger.info(
|
||||||
f"Connecting to blockchain: {blockchain_type} with custom provider!"
|
f"Connecting to blockchain: {blockchain_type} with custom provider!"
|
||||||
)
|
)
|
||||||
web3_client = Web3(HTTPProvider(web3_provider_uri))
|
|
||||||
|
web3_client = connect(web3_provider_uri)
|
||||||
|
|
||||||
if blockchain_type != AvailableBlockchainType.ETHEREUM:
|
if blockchain_type != AvailableBlockchainType.ETHEREUM:
|
||||||
web3_client.middleware_onion.inject(geth_poa_middleware, layer=0)
|
web3_client.middleware_onion.inject(geth_poa_middleware, layer=0)
|
||||||
|
@ -292,6 +289,11 @@ def parse_jobs(
|
||||||
f"Web3 connection to custom provider {web3_provider_uri} failed error: {e}"
|
f"Web3 connection to custom provider {web3_provider_uri} failed error: {e}"
|
||||||
)
|
)
|
||||||
raise (e)
|
raise (e)
|
||||||
|
else:
|
||||||
|
logger.info(f"Connecting to blockchain: {blockchain_type} with Node balancer.")
|
||||||
|
web3_client = _retry_connect_web3(
|
||||||
|
blockchain_type=blockchain_type, access_id=access_id
|
||||||
|
)
|
||||||
|
|
||||||
logger.info(f"Crawler started connected to blockchain: {blockchain_type}")
|
logger.info(f"Crawler started connected to blockchain: {blockchain_type}")
|
||||||
|
|
||||||
|
@ -459,10 +461,20 @@ def handle_crawl(args: argparse.Namespace) -> None:
|
||||||
|
|
||||||
blockchain_type = AvailableBlockchainType(args.blockchain)
|
blockchain_type = AvailableBlockchainType(args.blockchain)
|
||||||
|
|
||||||
|
custom_web3_provider = args.custom_web3_provider
|
||||||
|
|
||||||
|
if args.infura and INFURA_PROJECT_ID is not None:
|
||||||
|
if blockchain_type not in infura_networks:
|
||||||
|
raise ValueError(
|
||||||
|
f"Infura is not supported for {blockchain_type} blockchain type"
|
||||||
|
)
|
||||||
|
logger.info(f"Using Infura!")
|
||||||
|
custom_web3_provider = infura_networks[blockchain_type]["url"]
|
||||||
|
|
||||||
parse_jobs(
|
parse_jobs(
|
||||||
jobs,
|
jobs,
|
||||||
blockchain_type,
|
blockchain_type,
|
||||||
args.custom_web3_provider,
|
custom_web3_provider,
|
||||||
args.block_number,
|
args.block_number,
|
||||||
args.batch_size,
|
args.batch_size,
|
||||||
args.access_id,
|
args.access_id,
|
||||||
|
@ -539,6 +551,11 @@ def main() -> None:
|
||||||
help="Type of blovkchain wich writng in database",
|
help="Type of blovkchain wich writng in database",
|
||||||
required=True,
|
required=True,
|
||||||
)
|
)
|
||||||
|
view_state_crawler_parser.add_argument(
|
||||||
|
"--infura",
|
||||||
|
action="store_true",
|
||||||
|
help="Use infura as web3 provider",
|
||||||
|
)
|
||||||
view_state_crawler_parser.add_argument(
|
view_state_crawler_parser.add_argument(
|
||||||
"--custom-web3-provider",
|
"--custom-web3-provider",
|
||||||
"-w3",
|
"-w3",
|
||||||
|
|
|
@ -36,4 +36,5 @@ export COINMARKETCAP_API_KEY="<API_key_to_parse_conmarketcap>"
|
||||||
# Custom crawler
|
# Custom crawler
|
||||||
export MOONSTREAM_S3_PUBLIC_DATA_BUCKET="<public_bucket>"
|
export MOONSTREAM_S3_PUBLIC_DATA_BUCKET="<public_bucket>"
|
||||||
export MOONSTREAM_S3_PUBLIC_DATA_BUCKET_PREFIX="dev"
|
export MOONSTREAM_S3_PUBLIC_DATA_BUCKET_PREFIX="dev"
|
||||||
export MOONSTREAM_PUBLIC_QUERIES_DATA_ACCESS_TOKEN="<access token for run queries for public dashboards>"
|
export MOONSTREAM_PUBLIC_QUERIES_DATA_ACCESS_TOKEN="<access token for run queries for public dashboards>"
|
||||||
|
export INFURA_PROJECT_ID="<infura_project_id>"
|
Ładowanie…
Reference in New Issue