From 64e0bf264593f769df65e3ac3311632dc3310eda Mon Sep 17 00:00:00 2001 From: kompotkot Date: Wed, 22 Dec 2021 09:34:18 +0000 Subject: [PATCH] Mooncrawl ready to go with WEB3_URI instread of ips and ports --- crawlers/mooncrawl/mooncrawl/blockchain.py | 8 ++++---- crawlers/mooncrawl/mooncrawl/nft/cli.py | 10 ++++----- crawlers/mooncrawl/mooncrawl/settings.py | 24 ++++++++-------------- crawlers/mooncrawl/sample.env | 6 ++---- 4 files changed, 19 insertions(+), 29 deletions(-) diff --git a/crawlers/mooncrawl/mooncrawl/blockchain.py b/crawlers/mooncrawl/mooncrawl/blockchain.py index ded7a145..d4af9631 100644 --- a/crawlers/mooncrawl/mooncrawl/blockchain.py +++ b/crawlers/mooncrawl/mooncrawl/blockchain.py @@ -23,8 +23,8 @@ from web3.types import BlockData from .data import AvailableBlockchainType, DateRange from .settings import ( MOONSTREAM_CRAWL_WORKERS, - MOONSTREAM_ETHEREUM_IPC_PATH, - MOONSTREAM_POLYGON_IPC_PATH, + MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI, + MOONSTREAM_POLYGON_WEB3_PROVIDER_URI, ) logger = logging.getLogger(__name__) @@ -42,9 +42,9 @@ def connect(blockchain_type: AvailableBlockchainType, web3_uri: Optional[str] = if web3_uri is None: if blockchain_type == AvailableBlockchainType.ETHEREUM: - web3_uri = MOONSTREAM_ETHEREUM_IPC_PATH + web3_uri = MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI elif blockchain_type == AvailableBlockchainType.POLYGON: - web3_uri = MOONSTREAM_POLYGON_IPC_PATH + web3_uri = MOONSTREAM_POLYGON_WEB3_PROVIDER_URI else: raise Exception("Wrong blockchain type provided for web3 URI") diff --git a/crawlers/mooncrawl/mooncrawl/nft/cli.py b/crawlers/mooncrawl/mooncrawl/nft/cli.py index 0f06474f..2fe1c823 100644 --- a/crawlers/mooncrawl/mooncrawl/nft/cli.py +++ b/crawlers/mooncrawl/mooncrawl/nft/cli.py @@ -22,7 +22,7 @@ from ..publish import publish_json from ..settings import ( MOONSTREAM_ADMIN_ACCESS_TOKEN, MOONSTREAM_DATA_JOURNAL_ID, - MOONSTREAM_ETHEREUM_IPC_PATH, + MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI, NFT_HUMBUG_TOKEN, ) from ..version import MOONCRAWL_VERSION @@ -45,15 +45,15 @@ BLOCKS_PER_SUMMARY = 40 def web3_client_from_cli_or_env(args: argparse.Namespace) -> Web3: """ Returns a web3 client either by parsing "--web3" argument on the given arguments or by looking up - the MOONSTREAM_ETHEREUM_IPC_PATH environment variable. + the MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI environment variable. """ - web3_connection_string = MOONSTREAM_ETHEREUM_IPC_PATH + web3_connection_string = MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI args_web3 = vars(args).get("web3") if args_web3 is not None: web3_connection_string = cast(str, args_web3) if web3_connection_string is None: raise ValueError( - "Could not find Web3 connection information in arguments or in MOONSTREAM_ETHEREUM_IPC_PATH environment variable" + "Could not find Web3 connection information in arguments or in MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI environment variable" ) return connect(AvailableBlockchainType.ETHEREUM, web3_connection_string) @@ -294,7 +294,7 @@ def main() -> None: "--web3", type=str, default=None, - help="(Optional) Web3 connection string. If not provided, uses the value specified by MOONSTREAM_ETHEREUM_IPC_PATH environment variable.", + help="(Optional) Web3 connection string. If not provided, uses the value specified by MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI environment variable.", ) parser_ethereum_label.set_defaults(func=ethereum_label_handler) diff --git a/crawlers/mooncrawl/mooncrawl/settings.py b/crawlers/mooncrawl/mooncrawl/settings.py index 4484f993..f988b762 100644 --- a/crawlers/mooncrawl/mooncrawl/settings.py +++ b/crawlers/mooncrawl/mooncrawl/settings.py @@ -26,24 +26,16 @@ DOCS_TARGET_PATH = "docs" CRAWLER_LABEL = "moonworm-alpha" # Geth connection address -MOONSTREAM_NODE_ETHEREUM_IPC_ADDR = os.environ.get( - "MOONSTREAM_NODE_ETHEREUM_IPC_ADDR", "127.0.0.1" +MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI = os.environ.get( + "MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI" ) -MOONSTREAM_NODE_ETHEREUM_IPC_PORT = os.environ.get( - "MOONSTREAM_NODE_ETHEREUM_IPC_PORT", 8545 -) -MOONSTREAM_ETHEREUM_IPC_PATH = ( - f"http://{MOONSTREAM_NODE_ETHEREUM_IPC_ADDR}:{MOONSTREAM_NODE_ETHEREUM_IPC_PORT}" -) -MOONSTREAM_NODE_POLYGON_IPC_ADDR = os.environ.get( - "MOONSTREAM_NODE_POLYGON_IPC_ADDR", "127.0.0.1" -) -MOONSTREAM_NODE_POLYGON_IPC_PORT = os.environ.get( - "MOONSTREAM_NODE_POLYGON_IPC_PORT", 8545 -) -MOONSTREAM_POLYGON_IPC_PATH = ( - f"http://{MOONSTREAM_NODE_POLYGON_IPC_ADDR}:{MOONSTREAM_NODE_POLYGON_IPC_PORT}" +if MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI is None: + raise Exception("MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI env variable is not set") +MOONSTREAM_POLYGON_WEB3_PROVIDER_URI = os.environ.get( + "MOONSTREAM_POLYGON_WEB3_PROVIDER_URI" ) +if MOONSTREAM_POLYGON_WEB3_PROVIDER_URI is None: + raise Exception("MOONSTREAM_POLYGON_WEB3_PROVIDER_URI env variable is not set") MOONSTREAM_CRAWL_WORKERS = 4 MOONSTREAM_CRAWL_WORKERS_RAW = os.environ.get("MOONSTREAM_CRAWL_WORKERS") diff --git a/crawlers/mooncrawl/sample.env b/crawlers/mooncrawl/sample.env index e4219fd3..a22bb6b7 100644 --- a/crawlers/mooncrawl/sample.env +++ b/crawlers/mooncrawl/sample.env @@ -1,10 +1,8 @@ export BUGOUT_BROOD_URL="https://auth.bugout.dev" export BUGOUT_SPIRE_URL="https://spire.bugout.dev" export MOONSTREAM_CORS_ALLOWED_ORIGINS="http://localhost:3000,https://moonstream.to,https://www.moonstream.to" -export MOONSTREAM_NODE_ETHEREUM_IPC_ADDR="127.0.0.1" -export MOONSTREAM_NODE_ETHEREUM_IPC_PORT="8545" -export MOONSTREAM_NODE_POLYGON_IPC_ADDR="127.0.0.1" -export MOONSTREAM_NODE_POLYGON_IPC_PORT="8545" +export MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI="https://" +export MOONSTREAM_POLYGON_WEB3_PROVIDER_URI="https://" export MOONSTREAM_CRAWL_WORKERS=4 export MOONSTREAM_DB_URI="postgresql://:@:/" export MOONSTREAM_ETHERSCAN_TOKEN=""