Fix Etherscan crawler so that we don't manually specify autogenerated id

pull/239/head
Neeraj Kashyap 2021-09-06 09:46:50 -07:00
rodzic c12d4f9a36
commit 741383b503
1 zmienionych plików z 23 dodań i 23 usunięć

Wyświetl plik

@ -66,21 +66,16 @@ def get_address_id(db_session: Session, contract_address: str) -> int:
if id is not None:
return id[0]
latest_address_id = (
db_session.query(EthereumAddress.id).order_by(text("id desc")).limit(1).one()
)[0]
id = latest_address_id + 1
smart_contract = EthereumAddress(
id=id,
address=contract_address,
)
try:
db_session.add(smart_contract)
db_session.commit()
except:
return smart_contract.id
except Exception as e:
db_session.rollback()
return id
raise e
def crawl_step(db_session: Session, contract: VerifiedSmartContract, crawl_url: str):
@ -112,22 +107,27 @@ def crawl_step(db_session: Session, contract: VerifiedSmartContract, crawl_url:
object_key = f"/etherscan/v1/{contract.address}.json"
push_to_bucket(contract_info, object_key)
eth_address_id = get_address_id(db_session, contract.address)
eth_label = EthereumLabel(
label=ETHERSCAN_SMARTCONTRACTS_LABEL_NAME,
address_id=eth_address_id,
label_data={
"object_uri": f"s3://{bucket}/{object_key}",
"name": contract.name,
"tx_hash": contract.tx_hash,
},
)
try:
db_session.add(eth_label)
db_session.commit()
except:
db_session.rollback()
eth_address_id = get_address_id(db_session, contract.address)
eth_label = EthereumLabel(
label=ETHERSCAN_SMARTCONTRACTS_LABEL_NAME,
address_id=eth_address_id,
label_data={
"object_uri": f"s3://{bucket}/{object_key}",
"name": contract.name,
"tx_hash": contract.tx_hash,
},
)
try:
db_session.add(eth_label)
db_session.commit()
except Exception as e:
db_session.rollback()
raise e
except Exception as e:
logger.error(
f"Failed to add addresss label ${contract.address} to database\n{str(e)}"
)
def crawl(