mirror
/
moonstream
kopia lustrzana https://github.com/bugout-dev/moonstream
1
0
Forkuj 0
Kod Zgłoszenia Projekty Wydania Wiki Aktywność

Crawler can be executed with flag -t to chose what type of blockchain to use

pull/373/head
kompotkot 2021-11-08 15:08:57 +00:00
rodzic 8aa1734bdc
commit 725a1054a4
14 zmienionych plików z 191 dodań i 142 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

3
crawlers/mooncrawl/mooncrawl/.isort.cfg 100644
Wyświetl plik

@ -0,0 +1,3 @@
[settings]
profile = black
multi_line_output = 3

66
crawlers/mooncrawl/mooncrawl/ethereum.py → crawlers/mooncrawl/mooncrawl/blockchain.py
Wyświetl plik

@ -1,23 +1,27 @@
from concurrent.futures import Future, ProcessPoolExecutor, ThreadPoolExecutor, wait
from dataclasses import dataclass
from datetime import datetime
import logging
from concurrent.futures import Future, ProcessPoolExecutor, ThreadPoolExecutor, wait
from typing import Any, Callable, Dict, List, Optional, Tuple, Union
from psycopg2.errors import UniqueViolation # type: ignore
from sqlalchemy import desc, Column
from sqlalchemy import func
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session, Query
from tqdm import tqdm
from web3 import Web3, IPCProvider, HTTPProvider
from web3.types import BlockData
from .settings import MOONSTREAM_ETHEREUM_IPC_PATH, MOONSTREAM_CRAWL_WORKERS
from moonstreamdb.db import yield_db_session, yield_db_session_ctx
from moonstreamdb.models import (
EthereumBlock,
EthereumTransaction,
PolygonBlock,
PolygonTransactions,
)
from psycopg2.errors import UniqueViolation # type: ignore
from sqlalchemy import Column, desc, func
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Query, Session
from tqdm import tqdm
from web3 import HTTPProvider, IPCProvider, Web3
from web3.types import BlockData
from .data import AvailableBlockchainType, DateRange
from .settings import (
MOONSTREAM_CRAWL_WORKERS,
MOONSTREAM_ETHEREUM_IPC_PATH,
MOONSTREAM_POLYGON_IPC_PATH,
)
logger = logging.getLogger(__name__)
@ -30,15 +34,14 @@ class BlockCrawlError(Exception):
"""
@dataclass
class DateRange:
start_time: datetime
end_time: datetime
include_start: bool
include_end: bool
def connect(web3_uri: Optional[str] = MOONSTREAM_ETHEREUM_IPC_PATH):
def connect(blockchain_type: AvailableBlockchainType, web3_uri: Optional[str] = None):
if web3_uri is None:
if blockchain_type == AvailableBlockchainType.ETHEREUM:
web3_uri = MOONSTREAM_ETHEREUM_IPC_PATH
elif blockchain_type == AvailableBlockchainType.POLYGON:
web3_uri = MOONSTREAM_POLYGON_IPC_PATH
else:
raise Exception("Wrong blockchain type provided for web3 URI")
web3_provider: Union[IPCProvider, HTTPProvider] = Web3.IPCProvider()
if web3_uri is not None:
if web3_uri.startswith("http://") or web3_uri.startswith("https://"):
@ -103,14 +106,14 @@ def add_block_transactions(db_session, block: Any) -> None:
db_session.add(tx_obj)
def get_latest_blocks(confirmations: int = 0) -> Tuple[Optional[int], int]:
def get_latest_blocks(blockchain_type: AvailableBlockchainType,confirmations: int = 0) -> Tuple[Optional[int], int]:
"""
Retrieve the latest block from the connected node (connection is created by the connect() method).
Retrieve the latest block from the connected node (connection is created by the connect(AvailableBlockchainType) method).
If confirmations > 0, and the latest block on the node has block number N, this returns the block
with block_number (N - confirmations)
"""
web3_client = connect()
web3_client = connect(blockchain_type)
latest_block_number: int = web3_client.eth.block_number
if confirmations > 0:
latest_block_number -= confirmations
@ -128,11 +131,11 @@ def get_latest_blocks(confirmations: int = 0) -> Tuple[Optional[int], int]:
return latest_stored_block_number, latest_block_number
def crawl_blocks(blocks_numbers: List[int], with_transactions: bool = False) -> None:
def crawl_blocks(blockchain_type: AvailableBlockchainType, blocks_numbers: List[int], with_transactions: bool = False) -> None:
"""
Open database and geth sessions and fetch block data from blockchain.
"""
web3_client = connect()
web3_client = connect(blockchain_type)
with yield_db_session_ctx() as db_session:
pbar = tqdm(total=len(blocks_numbers))
for block_number in blocks_numbers:
@ -168,7 +171,7 @@ def crawl_blocks(blocks_numbers: List[int], with_transactions: bool = False) ->
pbar.close()
def check_missing_blocks(blocks_numbers: List[int], notransactions=False) -> List[int]:
def check_missing_blocks(blockchain_type: AvailableBlockchainType, blocks_numbers: List[int], notransactions=False) -> List[int]:
"""
Query block from postgres. If block does not presented in database,
add to missing blocks numbers list.
@ -204,7 +207,7 @@ def check_missing_blocks(blocks_numbers: List[int], notransactions=False) -> Lis
[block[0], block[1]] for block in blocks_exist_raw_query.all()
]
web3_client = connect()
web3_client = connect(blockchain_type)
blocks_exist_len = len(blocks_exist)
pbar = tqdm(total=blocks_exist_len)
@ -242,6 +245,7 @@ def check_missing_blocks(blocks_numbers: List[int], notransactions=False) -> Lis
def crawl_blocks_executor(
blockchain_type: AvailableBlockchainType,
block_numbers_list: List[int],
with_transactions: bool = False,
num_processes: int = MOONSTREAM_CRAWL_WORKERS,
@ -272,13 +276,13 @@ def crawl_blocks_executor(
results: List[Future] = []
if num_processes == 1:
logger.warning("Executing block crawler in lazy mod")
return crawl_blocks(block_numbers_list, with_transactions)
return crawl_blocks(blockchain_type, block_numbers_list, with_transactions)
else:
with ThreadPoolExecutor(max_workers=MOONSTREAM_CRAWL_WORKERS) as executor:
for worker in worker_indices:
block_chunk = worker_job_lists[worker]
logger.info(f"Spawned process for {len(block_chunk)} blocks")
result = executor.submit(crawl_blocks, block_chunk, with_transactions)
result = executor.submit(crawl_blocks, blockchain_type, block_chunk, with_transactions)
result.add_done_callback(record_error)
results.append(result)

125
crawlers/mooncrawl/mooncrawl/crawler.py
Wyświetl plik

@ -2,24 +2,25 @@
Moonstream crawlers CLI.
"""
import argparse
from datetime import datetime, timedelta, timezone
from enum import Enum
import json
import logging
import os
import sys
import time
from datetime import datetime, timedelta, timezone
from enum import Enum
from typing import Iterator, List
import dateutil.parser
from .ethereum import (
crawl_blocks_executor,
check_missing_blocks,
get_latest_blocks,
from .blockchain import (
DateRange,
check_missing_blocks,
crawl_blocks_executor,
get_latest_blocks,
trending,
)
from .data import AvailableBlockchainType
from .publish import publish_json
from .settings import MOONSTREAM_CRAWL_WORKERS
from .version import MOONCRAWL_VERSION
@ -86,13 +87,13 @@ def yield_blocks_numbers_lists(
current_block -= block_step
def ethcrawler_blocks_sync_handler(args: argparse.Namespace) -> None:
def crawler_blocks_sync_handler(args: argparse.Namespace) -> None:
"""
Synchronize latest Ethereum blocks with database.
Synchronize latest Blockchain blocks with database.
"""
while True:
latest_stored_block_number, latest_block_number = get_latest_blocks(
args.confirmations
args.blockchain_type, args.confirmations
)
if latest_stored_block_number is None:
latest_stored_block_number = 0
@ -132,6 +133,7 @@ def ethcrawler_blocks_sync_handler(args: argparse.Namespace) -> None:
)
# TODO(kompotkot): Set num_processes argument based on number of blocks to synchronize.
crawl_blocks_executor(
blockchain_type=args.blockchain_type,
block_numbers_list=blocks_numbers_list,
with_transactions=True,
num_processes=args.jobs,
@ -141,7 +143,7 @@ def ethcrawler_blocks_sync_handler(args: argparse.Namespace) -> None:
)
def ethcrawler_blocks_add_handler(args: argparse.Namespace) -> None:
def crawler_blocks_add_handler(args: argparse.Namespace) -> None:
"""
Add blocks to moonstream database.
"""
@ -150,7 +152,9 @@ def ethcrawler_blocks_add_handler(args: argparse.Namespace) -> None:
for blocks_numbers_list in yield_blocks_numbers_lists(args.blocks):
logger.info(f"Adding blocks {blocks_numbers_list[-1]}-{blocks_numbers_list[0]}")
crawl_blocks_executor(
block_numbers_list=blocks_numbers_list, with_transactions=True
blockchain_type=args.blockchain_type,
block_numbers_list=blocks_numbers_list,
with_transactions=True,
)
logger.info(
@ -158,7 +162,7 @@ def ethcrawler_blocks_add_handler(args: argparse.Namespace) -> None:
)
def ethcrawler_blocks_missing_handler(args: argparse.Namespace) -> None:
def crawler_blocks_missing_handler(args: argparse.Namespace) -> None:
"""
Check missing blocks and missing transactions in each block.
"""
@ -171,6 +175,7 @@ def ethcrawler_blocks_missing_handler(args: argparse.Namespace) -> None:
f"with comparing transactions: {not args.notransactions}"
)
missing_blocks_numbers = check_missing_blocks(
blockchain_type=args.blockchain_type,
blocks_numbers=blocks_numbers_list,
notransactions=args.notransactions,
)
@ -185,7 +190,8 @@ def ethcrawler_blocks_missing_handler(args: argparse.Namespace) -> None:
if (len(missing_blocks_numbers_total)) > 0:
time.sleep(5)
crawl_blocks_executor(
missing_blocks_numbers_total,
blockchain_type=args.blockchain_type,
block_numbers_list=missing_blocks_numbers_total,
with_transactions=True,
num_processes=1 if args.lazy else MOONSTREAM_CRAWL_WORKERS,
)
@ -195,7 +201,7 @@ def ethcrawler_blocks_missing_handler(args: argparse.Namespace) -> None:
)
def ethcrawler_trending_handler(args: argparse.Namespace) -> None:
def crawler_trending_handler(args: argparse.Namespace) -> None:
date_range = DateRange(
start_time=args.start,
end_time=args.end,
@ -228,15 +234,15 @@ def main() -> None:
time_now = datetime.now(timezone.utc)
# Ethereum blocks parser
parser_ethcrawler_blocks = subcommands.add_parser(
"blocks", description="Ethereum blocks commands"
# Blockchain blocks parser
parser_crawler_blocks = subcommands.add_parser(
"blocks", description="Blockchain blocks commands"
)
parser_ethcrawler_blocks.set_defaults(
func=lambda _: parser_ethcrawler_blocks.print_help()
parser_crawler_blocks.set_defaults(
func=lambda _: parser_crawler_blocks.print_help()
)
subcommands_ethcrawler_blocks = parser_ethcrawler_blocks.add_subparsers(
description="Ethereum blocks commands"
subcommands_crawler_blocks = parser_crawler_blocks.add_subparsers(
description="Blockchain blocks commands"
)
valid_processing_orders = {
@ -251,29 +257,29 @@ def main() -> None:
f"Invalid processing order ({raw_order}). Valid choices: {valid_processing_orders.keys()}"
)
parser_ethcrawler_blocks_sync = subcommands_ethcrawler_blocks.add_parser(
"synchronize", description="Synchronize to latest ethereum block commands"
parser_crawler_blocks_sync = subcommands_crawler_blocks.add_parser(
"synchronize", description="Synchronize to latest blockchain block commands"
)
parser_ethcrawler_blocks_sync.add_argument(
parser_crawler_blocks_sync.add_argument(
"-s",
"--start",
type=int,
help="(Optional) Block to start synchronization from. Default: None - current Ethereum block minus confirmations ",
help="(Optional) Block to start synchronization from. Default: None - current Blockchain block minus confirmations ",
)
parser_ethcrawler_blocks_sync.add_argument(
parser_crawler_blocks_sync.add_argument(
"-c",
"--confirmations",
type=int,
default=0,
help="Number of confirmations we require before storing a block in the database. (Default: 0)",
)
parser_ethcrawler_blocks_sync.add_argument(
parser_crawler_blocks_sync.add_argument(
"--order",
type=processing_order,
default=ProcessingOrder.ASCENDING,
help="Order in which to process blocks (choices: desc, asc; default: asc)",
)
parser_ethcrawler_blocks_sync.add_argument(
parser_crawler_blocks_sync.add_argument(
"-j",
"--jobs",
type=int,
@ -283,72 +289,89 @@ def main() -> None:
" If you set to 1, the main process handles synchronization without spawning subprocesses."
),
)
parser_ethcrawler_blocks_sync.set_defaults(func=ethcrawler_blocks_sync_handler)
parser_ethcrawler_blocks_add = subcommands_ethcrawler_blocks.add_parser(
"add", description="Add ethereum blocks commands"
parser_crawler_blocks_sync.add_argument(
"-t",
"--blockchain-type",
required=True,
help=f"Available blockchain types: {[member for member in AvailableBlockchainType.__members__]}",
)
parser_ethcrawler_blocks_add.add_argument(
parser_crawler_blocks_sync.set_defaults(func=crawler_blocks_sync_handler)
parser_crawler_blocks_add = subcommands_crawler_blocks.add_parser(
"add", description="Add blockchain blocks commands"
)
parser_crawler_blocks_add.add_argument(
"-b",
"--blocks",
required=True,
help="List of blocks range in format {bottom_block}-{top_block}",
)
parser_ethcrawler_blocks_add.set_defaults(func=ethcrawler_blocks_add_handler)
parser_ethcrawler_blocks_missing = subcommands_ethcrawler_blocks.add_parser(
"missing", description="Add missing ethereum blocks with transactions commands"
parser_crawler_blocks_add.add_argument(
"-t",
"--blockchain-type",
required=True,
help=f"Available blockchain types: {[member for member in AvailableBlockchainType.__members__]}",
)
parser_ethcrawler_blocks_missing.add_argument(
parser_crawler_blocks_add.set_defaults(func=crawler_blocks_add_handler)
parser_crawler_blocks_missing = subcommands_crawler_blocks.add_parser(
"missing",
description="Add missing Blockchain blocks with transactions commands",
)
parser_crawler_blocks_missing.add_argument(
"-b",
"--blocks",
required=True,
help="List of blocks range in format {bottom_block}-{top_block}",
)
parser_ethcrawler_blocks_missing.add_argument(
parser_crawler_blocks_missing.add_argument(
"-n",
"--notransactions",
action="store_true",
help="Skip crawling block transactions",
)
parser_ethcrawler_blocks_missing.add_argument(
parser_crawler_blocks_missing.add_argument(
"-l",
"--lazy",
action="store_true",
help="Lazy block adding one by one",
)
parser_ethcrawler_blocks_missing.set_defaults(
func=ethcrawler_blocks_missing_handler
parser_crawler_blocks_missing.add_argument(
"-t",
"--blockchain-type",
required=True,
help=f"Available blockchain types: {[member for member in AvailableBlockchainType.__members__]}",
)
parser_crawler_blocks_missing.set_defaults(func=crawler_blocks_missing_handler)
parser_ethcrawler_trending = subcommands.add_parser(
"trending", description="Trending addresses on the Ethereum blockchain"
parser_crawler_trending = subcommands.add_parser(
"trending", description="Trending addresses on the Blockchain blockchain"
)
parser_ethcrawler_trending.add_argument(
parser_crawler_trending.add_argument(
"-s",
"--start",
type=dateutil.parser.parse,
default=(time_now - timedelta(hours=1, minutes=0)).isoformat(),
help=f"Start time for window to calculate trending addresses in (default: {(time_now - timedelta(hours=1,minutes=0)).isoformat()})",
)
parser_ethcrawler_trending.add_argument(
parser_crawler_trending.add_argument(
"--include-start",
action="store_true",
help="Set this flag if range should include start time",
)
parser_ethcrawler_trending.add_argument(
parser_crawler_trending.add_argument(
"-e",
"--end",
type=dateutil.parser.parse,
default=time_now.isoformat(),
help=f"End time for window to calculate trending addresses in (default: {time_now.isoformat()})",
)
parser_ethcrawler_trending.add_argument(
parser_crawler_trending.add_argument(
"--include-end",
action="store_true",
help="Set this flag if range should include end time",
)
parser_ethcrawler_trending.add_argument(
parser_crawler_trending.add_argument(
"--humbug",
default=None,
help=(
@ -357,14 +380,14 @@ def main() -> None:
"MOONSTREAM_HUMBUG_TOKEN environment variable)"
),
)
parser_ethcrawler_trending.add_argument(
parser_crawler_trending.add_argument(
"-o",
"--outfile",
type=argparse.FileType("w"),
default=sys.stdout,
help="Optional file to write output to. By default, prints to stdout.",
)
parser_ethcrawler_trending.set_defaults(func=ethcrawler_trending_handler)
parser_crawler_trending.set_defaults(func=crawler_trending_handler)
args = parser.parse_args()
args.func(args)

16
crawlers/mooncrawl/mooncrawl/data.py 100644
Wyświetl plik

@ -0,0 +1,16 @@
from dataclasses import dataclass
from datetime import datetime
from enum import Enum
class AvailableBlockchainType(Enum):
ETHEREUM = "ethereum"
POLYGON = "polygon"
@dataclass
class DateRange:
start_time: datetime
end_time: datetime
include_start: bool
include_end: bool

2
crawlers/mooncrawl/mooncrawl/esd.py
Wyświetl plik

@ -3,10 +3,10 @@ import sys
import time
from typing import Optional, Union
import requests
from moonstreamdb.db import yield_db_session_ctx
from moonstreamdb.models import ESDEventSignature, ESDFunctionSignature
from sqlalchemy.orm import Session
import requests
CRAWL_URLS = {
"functions": "https://www.4byte.directory/api/v1/signatures/",

10
crawlers/mooncrawl/mooncrawl/etherscan.py
Wyświetl plik

@ -1,23 +1,23 @@
import argparse
import codecs
import csv
from dataclasses import dataclass
from datetime import datetime
import json
import logging
import os
import sys
import time
from typing import Any, List, Optional, Dict
from dataclasses import dataclass
from datetime import datetime
from typing import Any, Dict, List, Optional
import boto3 # type: ignore
import requests
from moonstreamdb.db import yield_db_session_ctx
from moonstreamdb.models import EthereumLabel
import requests
from sqlalchemy.orm import Session
from .version import MOONCRAWL_VERSION
from .settings import MOONSTREAM_ETHERSCAN_TOKEN
from .version import MOONCRAWL_VERSION
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)

3
crawlers/mooncrawl/mooncrawl/identity.py
Wyświetl plik

@ -4,10 +4,9 @@ import os
import time
import requests
from sqlalchemy import text
from moonstreamdb.db import yield_db_session_ctx
from moonstreamdb.models import EthereumLabel
from sqlalchemy import text
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

33
crawlers/mooncrawl/mooncrawl/nft/cli.py
Wyświetl plik

@ -2,12 +2,12 @@
A command line tool to crawl information about NFTs from various sources.
"""
import argparse
from datetime import datetime, timezone
import json
import logging
import sys
import time
from typing import Any, cast, Dict, Optional
from datetime import datetime, timezone
from typing import Any, Dict, Optional, cast
from bugout.app import Bugout
from bugout.journal import SearchOrder
@ -16,26 +16,25 @@ from moonstreamdb.models import EthereumBlock
from sqlalchemy.orm.session import Session
from web3 import Web3
from ..ethereum import connect
from ..blockchain import connect
from ..data import AvailableBlockchainType
from ..publish import publish_json
from ..settings import (
MOONSTREAM_ADMIN_ACCESS_TOKEN,
MOONSTREAM_DATA_JOURNAL_ID,
MOONSTREAM_ETHEREUM_IPC_PATH,
NFT_HUMBUG_TOKEN,
)
from ..version import MOONCRAWL_VERSION
from .ethereum import (
summary as ethereum_summary,
add_labels,
MINT_LABEL,
TRANSFER_LABEL,
SUMMARY_KEY_ARGS,
SUMMARY_KEY_END_BLOCK,
SUMMARY_KEY_ID,
SUMMARY_KEY_NUM_BLOCKS,
SUMMARY_KEY_START_BLOCK,
SUMMARY_KEY_END_BLOCK,
add_labels,
)
from ..publish import publish_json
from ..settings import (
MOONSTREAM_ETHEREUM_IPC_PATH,
MOONSTREAM_ADMIN_ACCESS_TOKEN,
NFT_HUMBUG_TOKEN,
MOONSTREAM_DATA_JOURNAL_ID,
)
from ..version import MOONCRAWL_VERSION
from .ethereum import summary as ethereum_summary
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
@ -56,7 +55,7 @@ def web3_client_from_cli_or_env(args: argparse.Namespace) -> Web3:
raise ValueError(
"Could not find Web3 connection information in arguments or in MOONSTREAM_ETHEREUM_IPC_PATH environment variable"
)
return connect(web3_connection_string)
return connect(AvailableBlockchainType.ETHEREUM, web3_connection_string)
def get_latest_block_from_node(web3_client: Web3):

19
crawlers/mooncrawl/mooncrawl/nft/ethereum.py
Wyświetl plik

@ -1,23 +1,18 @@
from dataclasses import dataclass, asdict
from datetime import datetime, timedelta, timezone
import json
import logging
from hexbytes.main import HexBytes
from typing import Any, cast, Dict, List, Optional, Set, Tuple
from dataclasses import asdict, dataclass
from datetime import datetime, timezone
from typing import Any, Dict, List, Optional, Set, Tuple, cast
from eth_typing.encoding import HexStr
from moonstreamdb.models import (
EthereumBlock,
EthereumLabel,
EthereumTransaction,
)
from hexbytes.main import HexBytes
from moonstreamdb.models import EthereumBlock, EthereumLabel, EthereumTransaction
from sqlalchemy import and_, func, text
from sqlalchemy.dialects.postgresql import insert
from sqlalchemy.orm import Session, Query
from sqlalchemy.orm import Query, Session
from tqdm import tqdm
from web3 import Web3
from web3.types import FilterParams, LogReceipt
from web3._utils.events import get_event_data
from web3.types import FilterParams, LogReceipt
from ..reporter import reporter

2
crawlers/mooncrawl/mooncrawl/publish.py
Wyświetl plik

@ -1,6 +1,6 @@
from datetime import datetime
import json
import os
from datetime import datetime
from typing import Any, Dict, List, Optional
import requests

9
crawlers/mooncrawl/mooncrawl/settings.py
Wyświetl plik

@ -14,6 +14,15 @@ MOONSTREAM_NODE_ETHEREUM_IPC_PORT = os.environ.get(
MOONSTREAM_ETHEREUM_IPC_PATH = (
f"http://{MOONSTREAM_NODE_ETHEREUM_IPC_ADDR}:{MOONSTREAM_NODE_ETHEREUM_IPC_PORT}"
)
MOONSTREAM_NODE_POLYGON_IPC_ADDR = os.environ.get(
"MOONSTREAM_NODE_POLYGON_IPC_ADDR", "127.0.0.1"
)
MOONSTREAM_NODE_POLYGON_IPC_PORT = os.environ.get(
"MOONSTREAM_NODE_POLYGON_IPC_PORT", 8545
)
MOONSTREAM_POLYGON_IPC_PATH = (
f"http://{MOONSTREAM_NODE_POLYGON_IPC_ADDR}:{MOONSTREAM_NODE_POLYGON_IPC_PORT}"
)
MOONSTREAM_CRAWL_WORKERS = 4
MOONSTREAM_CRAWL_WORKERS_RAW = os.environ.get("MOONSTREAM_CRAWL_WORKERS")

42
crawlers/mooncrawl/mooncrawl/test_cli.py
Wyświetl plik

@ -1,13 +1,13 @@
import unittest
from . import ethcrawler
from . import crawler
class TestYieldBlockNumbersLists(unittest.TestCase):
def test_yield_descending_10_6_step_4(self):
partition = [
block_numbers_list
for block_numbers_list in ethcrawler.yield_blocks_numbers_lists(
for block_numbers_list in crawler.yield_blocks_numbers_lists(
"10-6", block_step=4
)
]
@ -16,7 +16,7 @@ class TestYieldBlockNumbersLists(unittest.TestCase):
def test_yield_descending_10_6_step_3(self):
partition = [
block_numbers_list
for block_numbers_list in ethcrawler.yield_blocks_numbers_lists(
for block_numbers_list in crawler.yield_blocks_numbers_lists(
"10-6", block_step=3
)
]
@ -25,8 +25,8 @@ class TestYieldBlockNumbersLists(unittest.TestCase):
def test_yield_descending_10_6_descending_step_3(self):
partition = [
block_numbers_list
for block_numbers_list in ethcrawler.yield_blocks_numbers_lists(
"10-6", ethcrawler.ProcessingOrder.DESCENDING, 3
for block_numbers_list in crawler.yield_blocks_numbers_lists(
"10-6", crawler.ProcessingOrder.DESCENDING, 3
)
]
self.assertListEqual(partition, [[10, 9, 8], [7, 6]])
@ -34,8 +34,8 @@ class TestYieldBlockNumbersLists(unittest.TestCase):
def test_yield_descending_10_6_descending_step_10(self):
partition = [
block_numbers_list
for block_numbers_list in ethcrawler.yield_blocks_numbers_lists(
"10-6", ethcrawler.ProcessingOrder.DESCENDING, 10
for block_numbers_list in crawler.yield_blocks_numbers_lists(
"10-6", crawler.ProcessingOrder.DESCENDING, 10
)
]
self.assertListEqual(partition, [[10, 9, 8, 7, 6]])
@ -43,7 +43,7 @@ class TestYieldBlockNumbersLists(unittest.TestCase):
def test_yield_descending_6_10_step_4(self):
partition = [
block_numbers_list
for block_numbers_list in ethcrawler.yield_blocks_numbers_lists(
for block_numbers_list in crawler.yield_blocks_numbers_lists(
"6-10", block_step=4
)
]
@ -52,7 +52,7 @@ class TestYieldBlockNumbersLists(unittest.TestCase):
def test_yield_descending_6_10_step_3(self):
partition = [
block_numbers_list
for block_numbers_list in ethcrawler.yield_blocks_numbers_lists(
for block_numbers_list in crawler.yield_blocks_numbers_lists(
"6-10", block_step=3
)
]
@ -61,8 +61,8 @@ class TestYieldBlockNumbersLists(unittest.TestCase):
def test_yield_descending_6_10_descending_step_3(self):
partition = [
block_numbers_list
for block_numbers_list in ethcrawler.yield_blocks_numbers_lists(
"6-10", ethcrawler.ProcessingOrder.DESCENDING, 3
for block_numbers_list in crawler.yield_blocks_numbers_lists(
"6-10", crawler.ProcessingOrder.DESCENDING, 3
)
]
self.assertListEqual(partition, [[10, 9, 8], [7, 6]])
@ -70,8 +70,8 @@ class TestYieldBlockNumbersLists(unittest.TestCase):
def test_yield_descending_6_10_descending_step_10(self):
partition = [
block_numbers_list
for block_numbers_list in ethcrawler.yield_blocks_numbers_lists(
"6-10", ethcrawler.ProcessingOrder.DESCENDING, 10
for block_numbers_list in crawler.yield_blocks_numbers_lists(
"6-10", crawler.ProcessingOrder.DESCENDING, 10
)
]
self.assertListEqual(partition, [[10, 9, 8, 7, 6]])
@ -79,8 +79,8 @@ class TestYieldBlockNumbersLists(unittest.TestCase):
def test_yield_ascending_10_6_ascending_step_3(self):
partition = [
block_numbers_list
for block_numbers_list in ethcrawler.yield_blocks_numbers_lists(
"10-6", ethcrawler.ProcessingOrder.ASCENDING, 3
for block_numbers_list in crawler.yield_blocks_numbers_lists(
"10-6", crawler.ProcessingOrder.ASCENDING, 3
)
]
self.assertListEqual(partition, [[6, 7, 8], [9, 10]])
@ -88,8 +88,8 @@ class TestYieldBlockNumbersLists(unittest.TestCase):
def test_yield_ascending_10_6_ascending_step_10(self):
partition = [
block_numbers_list
for block_numbers_list in ethcrawler.yield_blocks_numbers_lists(
"10-6", ethcrawler.ProcessingOrder.ASCENDING, 10
for block_numbers_list in crawler.yield_blocks_numbers_lists(
"10-6", crawler.ProcessingOrder.ASCENDING, 10
)
]
self.assertListEqual(partition, [[6, 7, 8, 9, 10]])
@ -97,8 +97,8 @@ class TestYieldBlockNumbersLists(unittest.TestCase):
def test_yield_ascending_6_10_ascending_step_4(self):
partition = [
block_numbers_list
for block_numbers_list in ethcrawler.yield_blocks_numbers_lists(
"6-10", ethcrawler.ProcessingOrder.ASCENDING, 4
for block_numbers_list in crawler.yield_blocks_numbers_lists(
"6-10", crawler.ProcessingOrder.ASCENDING, 4
)
]
self.assertListEqual(partition, [[6, 7, 8, 9], [10]])
@ -106,8 +106,8 @@ class TestYieldBlockNumbersLists(unittest.TestCase):
def test_yield_ascending_6_10_ascending_step_10(self):
partition = [
block_numbers_list
for block_numbers_list in ethcrawler.yield_blocks_numbers_lists(
"6-10", ethcrawler.ProcessingOrder.ASCENDING, 10
for block_numbers_list in crawler.yield_blocks_numbers_lists(
"6-10", crawler.ProcessingOrder.ASCENDING, 10
)
]
self.assertListEqual(partition, [[6, 7, 8, 9, 10]])

1
crawlers/mooncrawl/requirements.txt
Wyświetl plik

@ -27,6 +27,7 @@ idna==3.2
ipfshttpclient==0.8.0a2
jmespath==0.10.0
jsonschema==3.2.0
isort==5.10.0
lru-dict==1.1.7
multiaddr==0.0.9
multidict==5.1.0

2
crawlers/mooncrawl/setup.py
Wyświetl plik

@ -42,7 +42,7 @@ setup(
"web3",
],
extras_require={
"dev": ["black", "mypy", "types-requests", "types-python-dateutil"]
"dev": ["black", "isort", "mypy", "types-requests", "types-python-dateutil"]
},
entry_points={
"console_scripts": [