From ce628b6e0b67ca000e7e9d808f7e1b31a71bfe5c Mon Sep 17 00:00:00 2001 From: Andrey Date: Wed, 17 Apr 2024 03:08:54 +0300 Subject: [PATCH] Add crawlers and api. --- crawlers/mooncrawl/mooncrawl/blockchain.py | 3 +++ crawlers/mooncrawl/mooncrawl/settings.py | 9 ++++++++ crawlers/mooncrawl/mooncrawl/version.py | 2 +- crawlers/mooncrawl/sample.env | 1 + crawlers/mooncrawl/setup.py | 2 +- moonstreamapi/configs/sample.env | 1 + .../moonstreamapi/admin/subscription_types.py | 22 +++++++++++++++++++ .../moonstreamapi/providers/__init__.py | 2 ++ .../providers/moonworm_provider.py | 8 +++++++ .../moonstreamapi/providers/transactions.py | 7 ++++++ moonstreamapi/moonstreamapi/settings.py | 8 +++++++ moonstreamapi/moonstreamapi/version.py | 2 +- moonstreamapi/moonstreamapi/web3_provider.py | 3 +++ moonstreamapi/requirements.txt | 2 +- moonstreamapi/setup.py | 2 +- 15 files changed, 69 insertions(+), 5 deletions(-) diff --git a/crawlers/mooncrawl/mooncrawl/blockchain.py b/crawlers/mooncrawl/mooncrawl/blockchain.py index 1551deb7..3f01fe3d 100644 --- a/crawlers/mooncrawl/mooncrawl/blockchain.py +++ b/crawlers/mooncrawl/mooncrawl/blockchain.py @@ -32,6 +32,7 @@ from .settings import ( MOONSTREAM_NODE_ETHEREUM_A_EXTERNAL_URI, MOONSTREAM_NODE_MUMBAI_A_EXTERNAL_URI, MOONSTREAM_NODE_POLYGON_A_EXTERNAL_URI, + MOONSTREAM_NODE_PROOFOFPLAY_APEX_A_EXTERNAL_URI, MOONSTREAM_NODE_XAI_A_EXTERNAL_URI, MOONSTREAM_NODE_XAI_SEPOLIA_A_EXTERNAL_URI, MOONSTREAM_NODE_XDAI_A_EXTERNAL_URI, @@ -91,6 +92,8 @@ def connect( web3_uri = MOONSTREAM_NODE_BLAST_A_EXTERNAL_URI elif blockchain_type == AvailableBlockchainType.BLAST_SEPOLIA: web3_uri = MOONSTREAM_NODE_BLAST_SEPOLIA_A_EXTERNAL_URI + elif blockchain_type == AvailableBlockchainType.PROOFOFPLAY_APEX: + web3_uri = MOONSTREAM_NODE_PROOFOFPLAY_APEX_A_EXTERNAL_URI else: raise Exception("Wrong blockchain type provided for web3 URI") diff --git a/crawlers/mooncrawl/mooncrawl/settings.py b/crawlers/mooncrawl/mooncrawl/settings.py index 35fe1764..f9a16b04 100644 --- a/crawlers/mooncrawl/mooncrawl/settings.py +++ b/crawlers/mooncrawl/mooncrawl/settings.py @@ -180,6 +180,15 @@ if MOONSTREAM_NODE_BLAST_SEPOLIA_A_EXTERNAL_URI == "": "MOONSTREAM_NODE_BLAST_SEPOLIA_A_EXTERNAL_URI env variable is not set" ) +MOONSTREAM_NODE_PROOFOFPLAY_APEX_A_EXTERNAL_URI = os.environ.get( + "MOONSTREAM_NODE_PROOFOFPLAY_APEX_A_EXTERNAL_URI", "" +) +if MOONSTREAM_NODE_PROOFOFPLAY_APEX_A_EXTERNAL_URI == "": + raise Exception( + "MOONSTREAM_NODE_PROOFOFPLAY_APEX_A_EXTERNAL_URI env variable is not set" + ) + + MOONSTREAM_CRAWL_WORKERS = 4 MOONSTREAM_CRAWL_WORKERS_RAW = os.environ.get("MOONSTREAM_CRAWL_WORKERS") try: diff --git a/crawlers/mooncrawl/mooncrawl/version.py b/crawlers/mooncrawl/mooncrawl/version.py index 6f915441..5cfca106 100644 --- a/crawlers/mooncrawl/mooncrawl/version.py +++ b/crawlers/mooncrawl/mooncrawl/version.py @@ -2,4 +2,4 @@ Moonstream crawlers version. """ -MOONCRAWL_VERSION = "0.4.3" +MOONCRAWL_VERSION = "0.4.4" diff --git a/crawlers/mooncrawl/sample.env b/crawlers/mooncrawl/sample.env index cce5e77f..88148683 100644 --- a/crawlers/mooncrawl/sample.env +++ b/crawlers/mooncrawl/sample.env @@ -23,6 +23,7 @@ export NFT_HUMBUG_TOKEN="" # Blockchain nodes environment variables export MOONSTREAM_NODE_ETHEREUM_A_EXTERNAL_URI="https://" export MOONSTREAM_NODE_POLYGON_A_EXTERNAL_URI="https://" +export MOONSTREAM_NODE_PROOFOFPLAY_APEX_A_EXTERNAL_URI="https://" export MOONSTREAM_NODE_MUMBAI_A_EXTERNAL_URI="https://" export MOONSTREAM_NODE_AMOY_A_EXTERNAL_URI="https://" export MOONSTREAM_NODE_XDAI_A_EXTERNAL_URI="https://" diff --git a/crawlers/mooncrawl/setup.py b/crawlers/mooncrawl/setup.py index b271e111..7c6c41f6 100644 --- a/crawlers/mooncrawl/setup.py +++ b/crawlers/mooncrawl/setup.py @@ -37,7 +37,7 @@ setup( "bugout>=0.2.13", "chardet", "fastapi", - "moonstreamdb>=0.4.1", + "moonstreamdb>=0.4.2", "moonstream>=0.1.1", "moonworm[moonstream]>=0.6.2", "humbug", diff --git a/moonstreamapi/configs/sample.env b/moonstreamapi/configs/sample.env index 3f17b0e4..e9d6e0c3 100644 --- a/moonstreamapi/configs/sample.env +++ b/moonstreamapi/configs/sample.env @@ -17,6 +17,7 @@ export MOONSTREAM_DATA_JOURNAL_ID="" export HUMBUG_TXPOOL_CLIENT_ID="" export MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI="https://" export MOONSTREAM_POLYGON_WEB3_PROVIDER_URI="https://" +export MOONSTREAM_PROOFOFPLAY_APEX_WEB3_PROVIDER_URI="https://" export MOONSTREAM_MUMBAI_WEB3_PROVIDER_URI="https://" export MOONSTREAM_AMOY_WEB3_PROVIDER_URI="https://" export MOONSTREAM_XDAI_WEB3_PROVIDER_URI="https://" diff --git a/moonstreamapi/moonstreamapi/admin/subscription_types.py b/moonstreamapi/moonstreamapi/admin/subscription_types.py index 71be9fe9..ca2f3d30 100644 --- a/moonstreamapi/moonstreamapi/admin/subscription_types.py +++ b/moonstreamapi/moonstreamapi/admin/subscription_types.py @@ -40,6 +40,17 @@ CANONICAL_SUBSCRIPTION_TYPES = { stripe_price_id=None, active=True, ), + "proofofplay_apex_smartcontract": SubscriptionTypeResourceData( + id="proofofplay_apex_smartcontract", + name="Proof of Play Apex smartcontracts", + blockchain="proofofplay_apex", + choices=["input:address", "tag:erc721"], + description="Contracts events and tx_calls of contract of Proof of Play Apex blockchain", + icon_url="https://static.simiotics.com/moonstream/assets/ethereum/eth-diamond-purple.png", + stripe_product_id=None, + stripe_price_id=None, + active=True, + ), "mumbai_smartcontract": SubscriptionTypeResourceData( id="mumbai_smartcontract", name="Mumbai smartcontracts", @@ -227,6 +238,17 @@ CANONICAL_SUBSCRIPTION_TYPES = { stripe_price_id=None, active=False, ), + "proofofplay_apex_blockchain": SubscriptionTypeResourceData( + id="proofofplay_apex_blockchain", + name="Proof of Play Apex transactions", + blockchain="proofofplay_apex", + choices=["input:address", "tag:erc721"], + description="Transactions that have been mined into the Proof of Play Apex blockchain", + icon_url="https://static.simiotics.com/moonstream/assets/ethereum/eth-diamond-purple.png", + stripe_product_id=None, + stripe_price_id=None, + active=False, + ), "mumbai_blockchain": SubscriptionTypeResourceData( id="mumbai_blockchain", name="Mumbai transactions", diff --git a/moonstreamapi/moonstreamapi/providers/__init__.py b/moonstreamapi/moonstreamapi/providers/__init__.py index 0068b2cd..5baf1461 100644 --- a/moonstreamapi/moonstreamapi/providers/__init__.py +++ b/moonstreamapi/moonstreamapi/providers/__init__.py @@ -49,6 +49,7 @@ class ReceivingEventsException(Exception): event_providers: Dict[str, Any] = { moonworm_provider.EthereumMoonwormProvider.event_type: moonworm_provider.EthereumMoonwormProvider, moonworm_provider.PolygonMoonwormProvider.event_type: moonworm_provider.PolygonMoonwormProvider, + moonworm_provider.ProofOfPlayApexMoonwormProvider.event_type: moonworm_provider.ProofOfPlayApexMoonwormProvider, moonworm_provider.MumbaiMoonwormProvider.event_type: moonworm_provider.MumbaiMoonwormProvider, moonworm_provider.AmoyMoonwormProvider.event_type: moonworm_provider.AmoyMoonwormProvider, moonworm_provider.XDaiMoonwormProvider.event_type: moonworm_provider.XDaiMoonwormProvider, @@ -65,6 +66,7 @@ event_providers: Dict[str, Any] = { moonworm_provider.BlastSepoliaMoonwormProvider.event_type: moonworm_provider.BlastSepoliaMoonwormProvider, transactions.EthereumTransactions.event_type: transactions.EthereumTransactions, transactions.PolygonTransactions.event_type: transactions.PolygonTransactions, + transactions.ProofOfPlayApexTransactions.event_type: transactions.ProofOfPlayApexTransactions, transactions.MumbaiTransactions.event_type: transactions.MumbaiTransactions, transactions.AmoyTransactions.event_type: transactions.AmoyTransactions, transactions.XDaiTransactions.event_type: transactions.XDaiTransactions, diff --git a/moonstreamapi/moonstreamapi/providers/moonworm_provider.py b/moonstreamapi/moonstreamapi/providers/moonworm_provider.py index 8c07c3ed..fa59237f 100644 --- a/moonstreamapi/moonstreamapi/providers/moonworm_provider.py +++ b/moonstreamapi/moonstreamapi/providers/moonworm_provider.py @@ -19,6 +19,7 @@ logger.setLevel(logging.WARN) ethereum_event_type = "ethereum_blockchain" polygon_event_type = "polygon_blockchain" +proofofplay_apex_event_type = "proofofplay_apex_blockchain" mumbai_event_type = "mumbai_blockchain" amoy_event_type = "amoy_blockchain" xdai_event_type = "xdai_blockchain" @@ -510,3 +511,10 @@ BlastSepoliaMoonwormProvider = MoonwormProvider( description="Provider for reviving transactions from Blast Sepolia tables.", streamboaundary_range_limit=2 * 60 * 60, ) + +ProofOfPlayApexMoonwormProvider = MoonwormProvider( + event_type="proofofplay_apex_smartcontract", + blockchain=AvailableBlockchainType("proofofplay_apex"), + description="Provider for reviving transactions from Proof of Play Apex tables.", + streamboaundary_range_limit=2 * 60 * 60, +) diff --git a/moonstreamapi/moonstreamapi/providers/transactions.py b/moonstreamapi/moonstreamapi/providers/transactions.py index 348286b5..3368f01c 100644 --- a/moonstreamapi/moonstreamapi/providers/transactions.py +++ b/moonstreamapi/moonstreamapi/providers/transactions.py @@ -559,3 +559,10 @@ BlastSepoliaTransactions = TransactionsProvider( description="Provider for resiving transactions from Blast Sepolia tables.", streamboaundary_range_limit=2 * 60 * 60, ) + +ProofOfPlayApexTransactions = TransactionsProvider( + event_type="proofofplay_apex_blockchain", + blockchain=AvailableBlockchainType("proofofplay_apex"), + description="Provider for resiving transactions from Proof of Play Apex tables.", + streamboaundary_range_limit=2 * 60 * 60, +) diff --git a/moonstreamapi/moonstreamapi/settings.py b/moonstreamapi/moonstreamapi/settings.py index 5f00a57a..d5921e63 100644 --- a/moonstreamapi/moonstreamapi/settings.py +++ b/moonstreamapi/moonstreamapi/settings.py @@ -214,6 +214,14 @@ if MOONSTREAM_BLAST_SEPOLIA_WEB3_PROVIDER_URI == "": "MOONSTREAM_BLAST_SEPOLIA_WEB3_PROVIDER_URI env variable is not set" ) +MOONSTREAM_PROOFOFPLAY_APEX_WEB3_PROVIDER_URI = os.environ.get( + "MOONSTREAM_PROOFOFPLAY_APEX_WEB3_PROVIDER_URI", "" +) +if MOONSTREAM_PROOFOFPLAY_APEX_WEB3_PROVIDER_URI == "": + raise Exception( + "MOONSTREAM_PROOFOFPLAY_APEX_WEB3_PROVIDER_URI env variable is not set" + ) + ## QueryAPI MOONSTREAM_S3_QUERIES_BUCKET = os.environ.get("MOONSTREAM_S3_QUERIES_BUCKET", "") diff --git a/moonstreamapi/moonstreamapi/version.py b/moonstreamapi/moonstreamapi/version.py index a621a692..39eec17e 100644 --- a/moonstreamapi/moonstreamapi/version.py +++ b/moonstreamapi/moonstreamapi/version.py @@ -2,4 +2,4 @@ Moonstream library and API version. """ -MOONSTREAMAPI_VERSION = "0.4.1" +MOONSTREAMAPI_VERSION = "0.4.2" diff --git a/moonstreamapi/moonstreamapi/web3_provider.py b/moonstreamapi/moonstreamapi/web3_provider.py index b444c4b3..cdcdbea2 100644 --- a/moonstreamapi/moonstreamapi/web3_provider.py +++ b/moonstreamapi/moonstreamapi/web3_provider.py @@ -22,6 +22,7 @@ from .settings import ( MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI, MOONSTREAM_MUMBAI_WEB3_PROVIDER_URI, MOONSTREAM_POLYGON_WEB3_PROVIDER_URI, + MOONSTREAM_PROOFOFPLAY_APEX_WEB3_PROVIDER_URI, MOONSTREAM_WYRM_WEB3_PROVIDER_URI, MOONSTREAM_XAI_SEPOLIA_WEB3_PROVIDER_URI, MOONSTREAM_XAI_WEB3_PROVIDER_URI, @@ -103,6 +104,8 @@ def connect( web3_uri = MOONSTREAM_BLAST_WEB3_PROVIDER_URI elif blockchain_type == AvailableBlockchainType.BLAST_SEPOLIA: web3_uri = MOONSTREAM_BLAST_SEPOLIA_WEB3_PROVIDER_URI + elif blockchain_type == AvailableBlockchainType.PROOFOFPLAY_APEX: + web3_uri = MOONSTREAM_PROOFOFPLAY_APEX_WEB3_PROVIDER_URI else: raise Exception("Wrong blockchain type provided for web3 URI") diff --git a/moonstreamapi/requirements.txt b/moonstreamapi/requirements.txt index cc509c0c..60915c97 100644 --- a/moonstreamapi/requirements.txt +++ b/moonstreamapi/requirements.txt @@ -37,7 +37,7 @@ lru-dict==1.1.8 Mako==1.2.3 MarkupSafe==2.1.1 moonstream==0.1.1 -moonstreamdb==0.4.1 +moonstreamdb==0.4.2 multiaddr==0.0.9 multidict==6.0.2 netaddr==0.8.0 diff --git a/moonstreamapi/setup.py b/moonstreamapi/setup.py index f998ee97..e3c14220 100644 --- a/moonstreamapi/setup.py +++ b/moonstreamapi/setup.py @@ -16,7 +16,7 @@ setup( "bugout>=0.2.15", "fastapi", "moonstream", - "moonstreamdb>=0.4.1", + "moonstreamdb>=0.4.2", "humbug", "pydantic==1.10.2", "pyevmasm",