Added --start argument to "blocks synchronize"

Also black formatting
pull/39/head
Neeraj Kashyap 2021-08-02 06:15:33 -07:00
rodzic 167de38dc5
commit 7f3fb3b8f9
2 zmienionych plików z 43 dodań i 8 usunięć

Wyświetl plik

@ -54,11 +54,12 @@ def ethcrawler_blocks_sync_handler(args: argparse.Namespace) -> None:
"""
Synchronize latest Ethereum blocks with database.
"""
starting_block: int = args.start
while True:
bottom_block_number, top_block_number = get_latest_blocks(
bool(strtobool(args.transactions))
)
bottom_block_number = bottom_block_number + 1
bottom_block_number = max(bottom_block_number + 1, starting_block)
if bottom_block_number >= top_block_number:
print(
f"Synchronization is unnecessary for blocks {bottom_block_number}-{top_block_number - 1}"
@ -178,6 +179,13 @@ def main() -> None:
default="True",
help="Add or not block transactions",
)
parser_ethcrawler_blocks_sync.add_argument(
"-s",
"--start",
type=int,
default=0,
help="(Optional) Block to start synchronization from. Default: 0",
)
parser_ethcrawler_blocks_sync.set_defaults(func=ethcrawler_blocks_sync_handler)
parser_ethcrawler_blocks_add = subcommands_ethcrawler_blocks.add_parser(

Wyświetl plik

@ -18,13 +18,18 @@ DB_MODELS = {
"events": ESDEventSignature,
}
def crawl_step(db_session: Session, crawl_url: str, db_model: Union[ESDEventSignature, ESDFunctionSignature]) -> Optional[str]:
def crawl_step(
db_session: Session,
crawl_url: str,
db_model: Union[ESDEventSignature, ESDFunctionSignature],
) -> Optional[str]:
attempt = 0
current_interval = 2
success = False
response: Optional[requests.Response] = None
while (not success) and attempt < 3:
while (not success) and attempt < 3:
attempt += 1
try:
response = requests.get(crawl_url)
@ -41,12 +46,21 @@ def crawl_step(db_session: Session, crawl_url: str, db_model: Union[ESDEventSign
page = response.json()
results = page.get("results", [])
rows = [db_model(id=row.get("id"), text_signature=row.get("text_signature"), hex_signature=row.get("hex_signature"), created_at=row.get("created_at")) for row in results]
rows = [
db_model(
id=row.get("id"),
text_signature=row.get("text_signature"),
hex_signature=row.get("hex_signature"),
created_at=row.get("created_at"),
)
for row in results
]
db_session.bulk_save_objects(rows)
db_session.commit()
return page.get("next")
def crawl(crawl_type: str, interval: float) -> None:
crawl_url: Optional[str] = CRAWL_URLS[crawl_type]
db_model = DB_MODELS[crawl_type]
@ -56,13 +70,26 @@ def crawl(crawl_type: str, interval: float) -> None:
crawl_url = crawl_step(db_session, crawl_url, db_model)
time.sleep(interval)
def main():
parser = argparse.ArgumentParser(description="Crawls function and event signatures from the Ethereum Signature Database (https://www.4byte.directory/)")
parser.add_argument("crawl_type", choices=CRAWL_URLS, help="Specifies whether to crawl function signatures or event signatures")
parser.add_argument("--interval", type=float, default=0.1, help="Number of seconds to wait between requests to the Ethereum Signature Database API")
parser = argparse.ArgumentParser(
description="Crawls function and event signatures from the Ethereum Signature Database (https://www.4byte.directory/)"
)
parser.add_argument(
"crawl_type",
choices=CRAWL_URLS,
help="Specifies whether to crawl function signatures or event signatures",
)
parser.add_argument(
"--interval",
type=float,
default=0.1,
help="Number of seconds to wait between requests to the Ethereum Signature Database API",
)
args = parser.parse_args()
crawl(args.crawl_type, args.interval)
if __name__ == "__main__":
main()
main()