kopia lustrzana https://github.com/bugout-dev/moonstream
Merge pull request #761 from bugout-dev/add-coldera-blockcahin
Add Wyrm blockchain supportpull/763/head mooncrawl/v0.2.7
Andrey Dolgolev
GitHub
commit
e8a97406d3
|
|
@ -78,8 +78,14 @@ def decode_abi(source: str, session: Optional[Session] = None) -> ContractABI:
|
|||
session.close()
|
||||
|
||||
abi = ContractABI(
|
||||
functions=cast(EVMFunctionSignature, function_signatures),
|
||||
events=cast(EVMEventSignature, event_signatures),
|
||||
functions=[
|
||||
cast(EVMFunctionSignature, function_signature)
|
||||
for function_signature in function_signatures
|
||||
],
|
||||
events=[
|
||||
cast(EVMEventSignature, event_signature)
|
||||
for event_signature in event_signatures
|
||||
],
|
||||
)
|
||||
return abi
|
||||
|
||||
|
|
|
|||
|
|
@ -48,10 +48,12 @@ blockchain_by_subscription_id = {
|
|||
"polygon_blockchain": "polygon",
|
||||
"mumbai_blockchain": "mumbai",
|
||||
"xdai_blockchain": "xdai",
|
||||
"wyrm_blockchain": "wyrm",
|
||||
"ethereum_smartcontract": "ethereum",
|
||||
"polygon_smartcontract": "polygon",
|
||||
"mumbai_smartcontract": "mumbai",
|
||||
"xdai_smartcontract": "xdai",
|
||||
"wyrm_smartcontract": "wyrm",
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -129,7 +131,6 @@ def get_ens_name(web3: Web3, address: str) -> Optional[str]:
|
|||
|
||||
|
||||
def get_ens_address(web3: Web3, name: str) -> Optional[str]:
|
||||
|
||||
if not is_valid_ens_name(name):
|
||||
raise ValueError(f"{name} is not valid ens name")
|
||||
|
||||
|
|
@ -148,7 +149,6 @@ def get_ens_address(web3: Web3, name: str) -> Optional[str]:
|
|||
def get_ethereum_address_info(
|
||||
db_session: Session, web3: Web3, address: str
|
||||
) -> Optional[data.EthereumAddressInfo]:
|
||||
|
||||
if not is_address(address):
|
||||
raise ValueError(f"Invalid ethereum address : {address}")
|
||||
|
||||
|
|
@ -265,7 +265,6 @@ def create_onboarding_resource(
|
|||
"is_complete": False,
|
||||
},
|
||||
) -> BugoutResource:
|
||||
|
||||
resource = bc.create_resource(
|
||||
token=token,
|
||||
application_id=MOONSTREAM_APPLICATION_ID,
|
||||
|
|
@ -317,7 +316,6 @@ def dashboards_abi_validation(
|
|||
abi: Any,
|
||||
s3_path: str,
|
||||
):
|
||||
|
||||
"""
|
||||
Validate current dashboard subscription : https://github.com/bugout-dev/moonstream/issues/345#issuecomment-953052444
|
||||
with contract abi on S3
|
||||
|
|
@ -332,7 +330,6 @@ def dashboards_abi_validation(
|
|||
}
|
||||
if not dashboard_subscription.all_methods:
|
||||
for method in dashboard_subscription.methods:
|
||||
|
||||
if method["name"] not in abi_functions:
|
||||
# Method not exists
|
||||
logger.error(
|
||||
|
|
@ -342,9 +339,7 @@ def dashboards_abi_validation(
|
|||
)
|
||||
raise MoonstreamHTTPException(status_code=400)
|
||||
if method.get("filters") and isinstance(method["filters"], dict):
|
||||
|
||||
for input_argument_name, value in method["filters"].items():
|
||||
|
||||
if input_argument_name not in abi_functions[method["name"]]:
|
||||
# Argument not exists
|
||||
logger.error(
|
||||
|
|
@ -373,7 +368,6 @@ def dashboards_abi_validation(
|
|||
|
||||
if not dashboard_subscription.all_events:
|
||||
for event in dashboard_subscription.events:
|
||||
|
||||
if event["name"] not in abi_events:
|
||||
logger.error(
|
||||
f"Error on dashboard resource validation event:{event['name']}"
|
||||
|
|
@ -383,9 +377,7 @@ def dashboards_abi_validation(
|
|||
raise MoonstreamHTTPException(status_code=400)
|
||||
|
||||
if event.get("filters") and isinstance(event["filters"], dict):
|
||||
|
||||
for input_argument_name, value in event["filters"].items():
|
||||
|
||||
if input_argument_name not in abi_events[event["name"]]:
|
||||
# Argument not exists
|
||||
logger.error(
|
||||
|
|
@ -482,7 +474,6 @@ def get_all_entries_from_search(
|
|||
reporter.error_report(e)
|
||||
|
||||
if len(results) != existing_metods.total_results:
|
||||
|
||||
for offset in range(limit, existing_metods.total_results, limit):
|
||||
existing_metods = bc.search(
|
||||
token=token,
|
||||
|
|
|
|||
|
|
@ -81,7 +81,6 @@ def migrations_run(args: argparse.Namespace) -> None:
|
|||
drop_keys = []
|
||||
|
||||
if args.file is not None:
|
||||
|
||||
with open(args.file) as migration_json_file:
|
||||
migration_json = json.load(migration_json_file)
|
||||
|
||||
|
|
@ -119,12 +118,10 @@ def migrations_run(args: argparse.Namespace) -> None:
|
|||
|
||||
|
||||
def moonworm_tasks_list_handler(args: argparse.Namespace) -> None:
|
||||
|
||||
moonworm_tasks.get_list_of_addresses()
|
||||
|
||||
|
||||
def moonworm_tasks_add_subscription_handler(args: argparse.Namespace) -> None:
|
||||
|
||||
moonworm_tasks.add_subscription(args.id)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -28,7 +28,6 @@ def get_list_of_addresses():
|
|||
addresses = set()
|
||||
|
||||
for entry in entries:
|
||||
|
||||
addresses.add(entry.title)
|
||||
|
||||
print(addresses)
|
||||
|
|
@ -53,7 +52,6 @@ def add_subscription(id: str):
|
|||
s3_client = boto3.client("s3")
|
||||
|
||||
if subscription_resource.resource_data["abi"] is not None:
|
||||
|
||||
bucket = subscription_resource.resource_data["bucket"]
|
||||
key = subscription_resource.resource_data["s3_path"]
|
||||
|
||||
|
|
@ -63,7 +61,6 @@ def add_subscription(id: str):
|
|||
s3_path = f"s3://{bucket}/{key}"
|
||||
|
||||
try:
|
||||
|
||||
response = s3_client.get_object(
|
||||
Bucket=bucket,
|
||||
Key=key,
|
||||
|
|
|
|||
|
|
@ -57,6 +57,16 @@ CANONICAL_SUBSCRIPTION_TYPES = {
|
|||
stripe_price_id=None,
|
||||
active=True,
|
||||
),
|
||||
"wyrm_smartcontract": SubscriptionTypeResourceData(
|
||||
id="wyrm_smartcontract",
|
||||
name="Wyrm smartcontract",
|
||||
choices=["input:address", "tag:erc721"],
|
||||
description="Contracts events and tx_calls of contract of Wyrm blockchain.",
|
||||
icon_url="https://s3.amazonaws.com/static.simiotics.com/moonstream/assets/great-wyrm-network-logo.png",
|
||||
stripe_product_id=None,
|
||||
stripe_price_id=None,
|
||||
active=True,
|
||||
),
|
||||
"ethereum_blockchain": SubscriptionTypeResourceData(
|
||||
id="ethereum_blockchain",
|
||||
name="Ethereum transactions",
|
||||
|
|
@ -97,6 +107,16 @@ CANONICAL_SUBSCRIPTION_TYPES = {
|
|||
stripe_price_id=None,
|
||||
active=True,
|
||||
),
|
||||
"wyrm_blockchain": SubscriptionTypeResourceData(
|
||||
id="wyrm_blockchain",
|
||||
name="Wyrm transactions",
|
||||
choices=["input:address", "tag:erc721"],
|
||||
description="Wyrm chain transactions subscription.",
|
||||
icon_url="https://s3.amazonaws.com/static.simiotics.com/moonstream/assets/great-wyrm-network-logo.png",
|
||||
stripe_product_id=None,
|
||||
stripe_price_id=None,
|
||||
active=True,
|
||||
),
|
||||
"ethereum_whalewatch": SubscriptionTypeResourceData(
|
||||
id="ethereum_whalewatch",
|
||||
name="Ethereum whale watch",
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ class MoonstreamHTTPException(HTTPException):
|
|||
status_code: int,
|
||||
detail: Any = None,
|
||||
headers: Optional[Dict[str, Any]] = None,
|
||||
internal_error: Exception = None,
|
||||
internal_error: Optional[Exception] = None,
|
||||
):
|
||||
super().__init__(status_code, detail, headers)
|
||||
if internal_error is not None:
|
||||
|
|
|
|||
|
|
@ -288,7 +288,6 @@ class EthereumTXPoolProvider(BugoutEventProvider):
|
|||
batch_size: int = 100,
|
||||
timeout: float = 30.0,
|
||||
):
|
||||
|
||||
super().__init__(
|
||||
event_type=event_type,
|
||||
description=description,
|
||||
|
|
@ -302,7 +301,6 @@ class EthereumTXPoolProvider(BugoutEventProvider):
|
|||
def parse_filters(
|
||||
self, query: StreamQuery, user_subscriptions: Dict[str, List[BugoutResource]]
|
||||
) -> Optional[List[str]]:
|
||||
|
||||
is_query_constrained = query.subscription_types or query.subscriptions
|
||||
relevant_subscriptions = user_subscriptions.get(self.event_type)
|
||||
|
||||
|
|
|
|||
|
|
@ -42,7 +42,6 @@ class ArgsFilters:
|
|||
|
||||
@dataclass
|
||||
class LabelsFilters:
|
||||
|
||||
name: str
|
||||
type: str
|
||||
args: List[ArgsFilters] = field(default_factory=list)
|
||||
|
|
@ -50,14 +49,12 @@ class LabelsFilters:
|
|||
|
||||
@dataclass
|
||||
class AddressFilters:
|
||||
|
||||
address: str
|
||||
label_filters: List[LabelsFilters] = field(default_factory=list)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Filters:
|
||||
|
||||
addresses: List[AddressFilters] = field(default_factory=list)
|
||||
|
||||
|
||||
|
|
@ -84,7 +81,6 @@ class MoonwormProvider:
|
|||
Optional[str], subscription.resource_data.get("address")
|
||||
)
|
||||
if subscription_address is not None:
|
||||
|
||||
# How apply labels?
|
||||
filters.addresses.append(
|
||||
AddressFilters(address=subscription_address, label_filters=[])
|
||||
|
|
@ -213,7 +209,6 @@ class MoonwormProvider:
|
|||
for address_filter in parsed_filters.addresses:
|
||||
labels_filters = []
|
||||
for label_filter in address_filter.label_filters:
|
||||
|
||||
labels_filters.append(
|
||||
and_(
|
||||
*(
|
||||
|
|
|
|||
|
|
@ -75,7 +75,6 @@ async def add_dashboard_handler(
|
|||
|
||||
for dashboard_subscription in subscription_settings:
|
||||
if dashboard_subscription.subscription_id in available_subscriptions.keys():
|
||||
|
||||
# TODO(Andrey): Add some dedublication for get object from s3 for repeated subscription_id
|
||||
|
||||
bucket = available_subscriptions[dashboard_subscription.subscription_id][
|
||||
|
|
@ -96,7 +95,6 @@ async def add_dashboard_handler(
|
|||
s3_path = f"s3://{bucket}/{key}"
|
||||
|
||||
try:
|
||||
|
||||
response = s3_client.get_object(
|
||||
Bucket=bucket,
|
||||
Key=key,
|
||||
|
|
@ -263,9 +261,7 @@ async def update_dashboard_handler(
|
|||
}
|
||||
|
||||
for dashboard_subscription in subscription_settings:
|
||||
|
||||
if dashboard_subscription.subscription_id in available_subscriptions:
|
||||
|
||||
# TODO(Andrey): Add some dedublication for get object from s3 for repeated subscription_id
|
||||
|
||||
bucket = available_subscriptions[dashboard_subscription.subscription_id][
|
||||
|
|
@ -286,7 +282,6 @@ async def update_dashboard_handler(
|
|||
s3_path = f"s3://{bucket}/{abi_path}"
|
||||
|
||||
try:
|
||||
|
||||
response = s3_client.get_object(
|
||||
Bucket=bucket,
|
||||
Key=abi_path,
|
||||
|
|
@ -316,7 +311,6 @@ async def update_dashboard_handler(
|
|||
dashboard_resource: Dict[str, Any] = {}
|
||||
|
||||
if subscription_settings:
|
||||
|
||||
dashboard_resource["subscription_settings"] = json.loads(dashboard.json())[
|
||||
"subscription_settings"
|
||||
]
|
||||
|
|
@ -406,7 +400,6 @@ async def get_dashboard_data_links_handler(
|
|||
stats: Dict[UUID, Any] = {}
|
||||
|
||||
for subscription in dashboard_subscriptions:
|
||||
|
||||
available_timescales = [timescale.value for timescale in data.TimeScale]
|
||||
stats[subscription.id] = {}
|
||||
for timescale in available_timescales:
|
||||
|
|
|
|||
|
|
@ -36,7 +36,6 @@ router = APIRouter(
|
|||
|
||||
@router.get("/list", tags=["queries"])
|
||||
async def get_list_of_queries_handler(request: Request) -> List[Dict[str, Any]]:
|
||||
|
||||
token = request.state.token
|
||||
|
||||
# Check already existed queries
|
||||
|
|
@ -93,7 +92,6 @@ async def create_query_handler(
|
|||
)
|
||||
|
||||
if query_name in used_queries:
|
||||
|
||||
raise MoonstreamHTTPException(
|
||||
status_code=404,
|
||||
detail=f"Provided query name already use. Please remove it or use PUT /{query_name} for update query",
|
||||
|
|
@ -133,7 +131,6 @@ async def create_query_handler(
|
|||
raise MoonstreamHTTPException(status_code=500, internal_error=e)
|
||||
|
||||
try:
|
||||
|
||||
bc.update_tags(
|
||||
token=MOONSTREAM_ADMIN_ACCESS_TOKEN,
|
||||
journal_id=MOONSTREAM_QUERIES_JOURNAL_ID,
|
||||
|
|
@ -152,7 +149,6 @@ async def create_query_handler(
|
|||
|
||||
@router.get("/{query_name}/query", tags=["queries"])
|
||||
async def get_query_handler(request: Request, query_name: str) -> BugoutJournalEntry:
|
||||
|
||||
token = request.state.token
|
||||
|
||||
try:
|
||||
|
|
@ -166,7 +162,6 @@ async def get_query_handler(request: Request, query_name: str) -> BugoutJournalE
|
|||
raise MoonstreamHTTPException(status_code=500, internal_error=e)
|
||||
|
||||
try:
|
||||
|
||||
entry = bc.get_entry(
|
||||
token=MOONSTREAM_ADMIN_ACCESS_TOKEN,
|
||||
journal_id=MOONSTREAM_QUERIES_JOURNAL_ID,
|
||||
|
|
@ -188,7 +183,6 @@ async def update_query_handler(
|
|||
query_name: str,
|
||||
request_update: data.UpdateQueryRequest = Body(...),
|
||||
) -> BugoutJournalEntryContent:
|
||||
|
||||
token = request.state.token
|
||||
|
||||
try:
|
||||
|
|
@ -202,7 +196,6 @@ async def update_query_handler(
|
|||
raise MoonstreamHTTPException(status_code=500, internal_error=e)
|
||||
|
||||
try:
|
||||
|
||||
entry = bc.update_entry_content(
|
||||
token=MOONSTREAM_ADMIN_ACCESS_TOKEN,
|
||||
journal_id=MOONSTREAM_QUERIES_JOURNAL_ID,
|
||||
|
|
@ -335,7 +328,6 @@ async def get_access_link_handler(
|
|||
s3_response = None
|
||||
|
||||
if entries.results and entries.results[0].content:
|
||||
|
||||
tags = entries.results[0].tags
|
||||
|
||||
file_type = "json"
|
||||
|
|
|
|||
|
|
@ -111,7 +111,6 @@ async def add_subscription_handler(
|
|||
raise MoonstreamHTTPException(status_code=500, internal_error=e)
|
||||
|
||||
if abi:
|
||||
|
||||
try:
|
||||
json_abi = json.loads(abi)
|
||||
except json.JSONDecodeError:
|
||||
|
|
@ -259,7 +258,6 @@ async def update_subscriptions_handler(
|
|||
update["label"] = label
|
||||
|
||||
if abi:
|
||||
|
||||
try:
|
||||
json_abi = json.loads(abi)
|
||||
except json.JSONDecodeError:
|
||||
|
|
@ -338,7 +336,6 @@ async def get_subscription_abi_handler(
|
|||
request: Request,
|
||||
subscription_id: str,
|
||||
) -> data.SubdcriptionsAbiResponse:
|
||||
|
||||
token = request.state.token
|
||||
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ logger = logging.getLogger(__name__)
|
|||
|
||||
router = APIRouter(prefix="/txinfo")
|
||||
|
||||
|
||||
# TODO(zomglings): Factor out the enrichment logic into a separate action, because it may be useful
|
||||
# independently from serving API calls (e.g. data processing).
|
||||
# TODO(kompotkot): Re-organize function to be able handle each steps with exceptions.
|
||||
|
|
|
|||
|
|
@ -182,7 +182,6 @@ async def set_onboarding_state(
|
|||
request: Request,
|
||||
onboarding_data: data.OnboardingState = Body(...),
|
||||
) -> data.OnboardingState:
|
||||
|
||||
token = request.state.token
|
||||
try:
|
||||
response = bc.list_resources(
|
||||
|
|
@ -240,7 +239,6 @@ async def get_onboarding_state(request: Request) -> data.OnboardingState:
|
|||
except BugoutResponseException as e:
|
||||
raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail)
|
||||
except Exception as e:
|
||||
|
||||
raise MoonstreamHTTPException(status_code=500)
|
||||
|
||||
if (
|
||||
|
|
|
|||
|
|
@ -343,7 +343,6 @@ subscription = {
|
|||
|
||||
class TestValidateStreamBoundary(unittest.TestCase):
|
||||
def test_valid_functions_and_event_subscription(self):
|
||||
|
||||
subscription = DashboardMeta(
|
||||
**{
|
||||
"subscription_id": "0869942d-3cd2-4ea5-9424-d0f3e1653173",
|
||||
|
|
@ -394,7 +393,6 @@ class TestValidateStreamBoundary(unittest.TestCase):
|
|||
)
|
||||
|
||||
with self.assertRaises(MoonstreamHTTPException):
|
||||
|
||||
dashboards_abi_validation(subscription, abi_example, s3_path="")
|
||||
|
||||
def test_invalid_function_arg_name_subscription(self):
|
||||
|
|
@ -421,7 +419,6 @@ class TestValidateStreamBoundary(unittest.TestCase):
|
|||
)
|
||||
|
||||
with self.assertRaises(MoonstreamHTTPException):
|
||||
|
||||
dashboards_abi_validation(subscription, abi_example, s3_path="")
|
||||
|
||||
def test_invalid_function_arg_type_subscription(self):
|
||||
|
|
@ -448,7 +445,6 @@ class TestValidateStreamBoundary(unittest.TestCase):
|
|||
)
|
||||
|
||||
with self.assertRaises(MoonstreamHTTPException):
|
||||
|
||||
dashboards_abi_validation(subscription, abi_example, s3_path="")
|
||||
|
||||
def test_invalid_event_name_subscription(self):
|
||||
|
|
@ -471,7 +467,6 @@ class TestValidateStreamBoundary(unittest.TestCase):
|
|||
}
|
||||
)
|
||||
with self.assertRaises(MoonstreamHTTPException):
|
||||
|
||||
dashboards_abi_validation(subscription, abi_example, s3_path="")
|
||||
|
||||
def test_invalid_event_arg_name_subscription(self):
|
||||
|
|
@ -498,7 +493,6 @@ class TestValidateStreamBoundary(unittest.TestCase):
|
|||
)
|
||||
|
||||
with self.assertRaises(MoonstreamHTTPException):
|
||||
|
||||
dashboards_abi_validation(subscription, abi_example, s3_path="")
|
||||
|
||||
def test_invalid_event_arg_type_subscription(self):
|
||||
|
|
@ -522,7 +516,6 @@ class TestValidateStreamBoundary(unittest.TestCase):
|
|||
)
|
||||
|
||||
with self.assertRaises(MoonstreamHTTPException):
|
||||
|
||||
dashboards_abi_validation(subscription, abi_example, s3_path="")
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -2,4 +2,4 @@
|
|||
Moonstream library and API version.
|
||||
"""
|
||||
|
||||
MOONSTREAMAPI_VERSION = "0.2.1"
|
||||
MOONSTREAMAPI_VERSION = "0.2.2"
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ setup(
|
|||
"boto3",
|
||||
"bugout>=0.1.19",
|
||||
"fastapi",
|
||||
"moonstreamdb>=0.3.1",
|
||||
"moonstreamdb>=0.3.3",
|
||||
"humbug",
|
||||
"pydantic",
|
||||
"pyevmasm",
|
||||
|
|
|
|||
|
|
@ -75,6 +75,9 @@ XDAI_STATISTICS_SERVICE_FILE="xdai-statistics.service"
|
|||
XDAI_STATISTICS_TIMER_FILE="xdai-statistics.timer"
|
||||
XDAI_MOONWORM_CRAWLER_SERVICE_FILE="xdai-moonworm-crawler.service"
|
||||
|
||||
# Wyrm service files
|
||||
WYRM_MOONWORM_CRAWLER_SERVICE_FILE="wyrm-moonworm-crawler.service"
|
||||
|
||||
set -eu
|
||||
|
||||
echo
|
||||
|
|
@ -334,3 +337,11 @@ chmod 644 "${SCRIPT_DIR}/${XDAI_MOONWORM_CRAWLER_SERVICE_FILE}"
|
|||
cp "${SCRIPT_DIR}/${XDAI_MOONWORM_CRAWLER_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${XDAI_MOONWORM_CRAWLER_SERVICE_FILE}"
|
||||
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
|
||||
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${XDAI_MOONWORM_CRAWLER_SERVICE_FILE}"
|
||||
|
||||
echo
|
||||
echo
|
||||
echo -e "${PREFIX_INFO} Replacing existing Wyrm moonworm crawler service definition with ${WYRM_MOONWORM_CRAWLER_SERVICE_FILE}"
|
||||
chmod 644 "${SCRIPT_DIR}/${WYR_MOONWORM_CRAWLER_SERVICE_FILE}"
|
||||
cp "${SCRIPT_DIR}/${WYRM_MOONWORM_CRAWLER_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${WYRM_MOONWORM_CRAWLER_SERVICE_FILE}"
|
||||
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
|
||||
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${WYRM_MOONWORM_CRAWLER_SERVICE_FILE}"
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
[Unit]
|
||||
Description=Wyrm moonworm crawler
|
||||
After=network.target
|
||||
StartLimitIntervalSec=300
|
||||
StartLimitBurst=3
|
||||
|
||||
[Service]
|
||||
WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
|
||||
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
|
||||
Restart=on-failure
|
||||
RestartSec=15s
|
||||
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl -b wyrm --poa --no-confirmations
|
||||
CPUWeight=70
|
||||
SyslogIdentifier=wyrm-moonworm-crawler
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
|
|
@ -29,6 +29,7 @@ from .settings import (
|
|||
MOONSTREAM_POLYGON_WEB3_PROVIDER_URI,
|
||||
MOONSTREAM_MUMBAI_WEB3_PROVIDER_URI,
|
||||
MOONSTREAM_XDAI_WEB3_PROVIDER_URI,
|
||||
MOONSTREAM_WYRM_WEB3_PROVIDER_URI,
|
||||
NB_ACCESS_ID_HEADER,
|
||||
NB_DATA_SOURCE_HEADER,
|
||||
WEB3_CLIENT_REQUEST_TIMEOUT_SECONDS,
|
||||
|
|
@ -70,6 +71,8 @@ def connect(
|
|||
web3_uri = MOONSTREAM_MUMBAI_WEB3_PROVIDER_URI
|
||||
elif blockchain_type == AvailableBlockchainType.XDAI:
|
||||
web3_uri = MOONSTREAM_XDAI_WEB3_PROVIDER_URI
|
||||
elif blockchain_type == AvailableBlockchainType.WYRM:
|
||||
web3_uri = MOONSTREAM_WYRM_WEB3_PROVIDER_URI
|
||||
else:
|
||||
raise Exception("Wrong blockchain type provided for web3 URI")
|
||||
|
||||
|
|
|
|||
|
|
@ -97,6 +97,13 @@ def handle_crawl(args: argparse.Namespace) -> None:
|
|||
else:
|
||||
logger.info(f"Using start block: {start_block}")
|
||||
|
||||
confirmations = args.confirmations
|
||||
|
||||
if not args.no_confirmations:
|
||||
assert confirmations > 0, "confirmations must be greater than 0"
|
||||
else:
|
||||
confirmations = 0
|
||||
|
||||
continuous_crawler(
|
||||
db_session,
|
||||
blockchain_type,
|
||||
|
|
@ -106,7 +113,7 @@ def handle_crawl(args: argparse.Namespace) -> None:
|
|||
start_block,
|
||||
args.max_blocks_batch,
|
||||
args.min_blocks_batch,
|
||||
args.confirmations,
|
||||
confirmations,
|
||||
args.min_sleep_time,
|
||||
args.heartbeat_interval,
|
||||
args.new_jobs_refetch_interval,
|
||||
|
|
@ -305,6 +312,13 @@ def main() -> None:
|
|||
help="Number of confirmations to wait for",
|
||||
)
|
||||
|
||||
crawl_parser.add_argument(
|
||||
"--no-confirmations",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Do not wait for confirmations explicitly set confirmations to 0",
|
||||
)
|
||||
|
||||
crawl_parser.add_argument(
|
||||
"--min-sleep-time",
|
||||
"-t",
|
||||
|
|
|
|||
|
|
@ -103,7 +103,6 @@ def continuous_crawler(
|
|||
), "min_blocks_batch must be less than max_blocks_batch"
|
||||
assert min_blocks_batch > 0, "min_blocks_batch must be greater than 0"
|
||||
assert max_blocks_batch > 0, "max_blocks_batch must be greater than 0"
|
||||
assert confirmations > 0, "confirmations must be greater than 0"
|
||||
assert min_sleep_time > 0, "min_sleep_time must be greater than 0"
|
||||
assert heartbeat_interval > 0, "heartbeat_interval must be greater than 0"
|
||||
assert (
|
||||
|
|
@ -124,6 +123,8 @@ def continuous_crawler(
|
|||
network = Network.mumbai
|
||||
elif blockchain_type == AvailableBlockchainType.XDAI:
|
||||
network = Network.xdai
|
||||
elif blockchain_type == AvailableBlockchainType.WYRM:
|
||||
network = Network.wyrm
|
||||
else:
|
||||
raise ValueError(f"Unknown blockchain type: {blockchain_type}")
|
||||
|
||||
|
|
|
|||
|
|
@ -31,6 +31,7 @@ class SubscriptionTypes(Enum):
|
|||
ETHEREUM_BLOCKCHAIN = "ethereum_smartcontract"
|
||||
MUMBAI_BLOCKCHAIN = "mumbai_smartcontract"
|
||||
XDAI_BLOCKCHAIN = "xdai_smartcontract"
|
||||
WYRM_BLOCKCHAIN = "wyrm_smartcontract"
|
||||
|
||||
|
||||
def abi_input_signature(input_abi: Dict[str, Any]) -> str:
|
||||
|
|
@ -133,6 +134,8 @@ def blockchain_type_to_subscription_type(
|
|||
return SubscriptionTypes.MUMBAI_BLOCKCHAIN
|
||||
elif blockchain_type == AvailableBlockchainType.XDAI:
|
||||
return SubscriptionTypes.XDAI_BLOCKCHAIN
|
||||
elif blockchain_type == AvailableBlockchainType.WYRM:
|
||||
return SubscriptionTypes.WYRM_BLOCKCHAIN
|
||||
else:
|
||||
raise ValueError(f"Unknown blockchain type: {blockchain_type}")
|
||||
|
||||
|
|
|
|||
|
|
@ -66,6 +66,8 @@ def function_call_crawler(
|
|||
network = Network.mumbai
|
||||
elif blockchain_type == AvailableBlockchainType.XDAI:
|
||||
network = Network.xdai
|
||||
elif blockchain_type == AvailableBlockchainType.WYRM:
|
||||
network = Network.wyrm
|
||||
else:
|
||||
raise ValueError(f"Unknown blockchain type: {blockchain_type}")
|
||||
|
||||
|
|
|
|||
|
|
@ -99,6 +99,12 @@ MOONSTREAM_XDAI_WEB3_PROVIDER_URI = os.environ.get(
|
|||
if MOONSTREAM_XDAI_WEB3_PROVIDER_URI == "":
|
||||
raise Exception("MOONSTREAM_XDAI_WEB3_PROVIDER_URI env variable is not set")
|
||||
|
||||
MOONSTREAM_WYRM_WEB3_PROVIDER_URI = os.environ.get(
|
||||
"MOONSTREAM_WYRM_WEB3_PROVIDER_URI", ""
|
||||
)
|
||||
if MOONSTREAM_WYRM_WEB3_PROVIDER_URI == "":
|
||||
raise Exception("MOONSTREAM_WYRM_WEB3_PROVIDER_URI env variable is not set")
|
||||
|
||||
MOONSTREAM_CRAWL_WORKERS = 4
|
||||
MOONSTREAM_CRAWL_WORKERS_RAW = os.environ.get("MOONSTREAM_CRAWL_WORKERS")
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -45,6 +45,7 @@ subscription_ids_by_blockchain = {
|
|||
"polygon": ["polygon_blockchain", "polygon_smartcontract"],
|
||||
"mumbai": ["mumbai_blockchain", "mumbai_smartcontract"],
|
||||
"xdai": ["xdai_blockchain", "xdai_smartcontract"],
|
||||
"wyrm": ["wyrm_blockchain", "wyrm_smartcontract"],
|
||||
}
|
||||
|
||||
blockchain_by_subscription_id = {
|
||||
|
|
@ -52,10 +53,12 @@ blockchain_by_subscription_id = {
|
|||
"polygon_blockchain": "polygon",
|
||||
"mumbai_blockchain": "mumbai",
|
||||
"xdai_blockchain": "xdai",
|
||||
"wyrm_blockchain": "wyrm",
|
||||
"ethereum_smartcontract": "ethereum",
|
||||
"polygon_smartcontract": "polygon",
|
||||
"mumbai_smartcontract": "mumbai",
|
||||
"xdai_smartcontract": "xdai",
|
||||
"wyrm_smartcontract": "wyrm",
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -2,4 +2,4 @@
|
|||
Moonstream crawlers version.
|
||||
"""
|
||||
|
||||
MOONCRAWL_VERSION = "0.2.6"
|
||||
MOONCRAWL_VERSION = "0.2.7"
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ export MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI="https://<connection_path_uri_to_et
|
|||
export MOONSTREAM_POLYGON_WEB3_PROVIDER_URI="https://<connection_path_uri_to_polygon_node>"
|
||||
export MOONSTREAM_MUMBAI_WEB3_PROVIDER_URI="https://<connection_path_uri_to_mumbai_node>"
|
||||
export MOONSTREAM_XDAI_WEB3_PROVIDER_URI="https://<connection_path_uri_to_xdai_node>"
|
||||
export MOONSTREAM_WYRM_WEB3_PROVIDER_URI="https://<connection_path_uri_to_wyrm_node>"
|
||||
export NB_CONTROLLER_ACCESS_ID="<access_uuid_for_moonstream_nodebalancer>"
|
||||
|
||||
# AWS environment variables
|
||||
|
|
@ -37,4 +38,4 @@ export COINMARKETCAP_API_KEY="<API_key_to_parse_conmarketcap>"
|
|||
export MOONSTREAM_S3_PUBLIC_DATA_BUCKET="<public_bucket>"
|
||||
export MOONSTREAM_S3_PUBLIC_DATA_BUCKET_PREFIX="dev"
|
||||
export MOONSTREAM_PUBLIC_QUERIES_DATA_ACCESS_TOKEN="<access token for run queries for public dashboards>"
|
||||
export INFURA_PROJECT_ID="<infura_project_id>"
|
||||
export INFURA_PROJECT_ID="<infura_project_id>"
|
||||
|
|
|
|||
|
|
@ -37,16 +37,16 @@ setup(
|
|||
"bugout>=0.1.19",
|
||||
"chardet",
|
||||
"fastapi",
|
||||
"moonstreamdb>=0.3.2",
|
||||
"moonstreamdb>=0.3.3",
|
||||
"moonstream>=0.1.1",
|
||||
"moonworm[moonstream]>=0.5.2",
|
||||
"moonworm[moonstream]>=0.6.1",
|
||||
"humbug",
|
||||
"pydantic==1.9.2",
|
||||
"python-dateutil",
|
||||
"requests",
|
||||
"tqdm",
|
||||
"uvicorn",
|
||||
"web3[tester]",
|
||||
"web3",
|
||||
],
|
||||
extras_require={
|
||||
"dev": ["black", "isort", "mypy", "types-requests", "types-python-dateutil"],
|
||||
|
|
|
|||
|
|
@ -0,0 +1,116 @@
|
|||
"""add-wyrm-chain
|
||||
|
||||
Revision ID: c413d5265f76
|
||||
Revises: 11233cf42d62
|
||||
Create Date: 2023-03-02 18:02:58.934085
|
||||
|
||||
"""
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy.dialects import postgresql
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = 'c413d5265f76'
|
||||
down_revision = '11233cf42d62'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.create_table('wyrm_blocks',
|
||||
sa.Column('block_number', sa.BigInteger(), nullable=False),
|
||||
sa.Column('difficulty', sa.BigInteger(), nullable=True),
|
||||
sa.Column('extra_data', sa.VARCHAR(length=128), nullable=True),
|
||||
sa.Column('gas_limit', sa.BigInteger(), nullable=True),
|
||||
sa.Column('gas_used', sa.BigInteger(), nullable=True),
|
||||
sa.Column('base_fee_per_gas', sa.Numeric(precision=78, scale=0), nullable=True),
|
||||
sa.Column('hash', sa.VARCHAR(length=256), nullable=True),
|
||||
sa.Column('logs_bloom', sa.VARCHAR(length=1024), nullable=True),
|
||||
sa.Column('miner', sa.VARCHAR(length=256), nullable=True),
|
||||
sa.Column('nonce', sa.VARCHAR(length=256), nullable=True),
|
||||
sa.Column('parent_hash', sa.VARCHAR(length=256), nullable=True),
|
||||
sa.Column('receipt_root', sa.VARCHAR(length=256), nullable=True),
|
||||
sa.Column('uncles', sa.VARCHAR(length=256), nullable=True),
|
||||
sa.Column('size', sa.Integer(), nullable=True),
|
||||
sa.Column('state_root', sa.VARCHAR(length=256), nullable=True),
|
||||
sa.Column('timestamp', sa.BigInteger(), nullable=True),
|
||||
sa.Column('total_difficulty', sa.VARCHAR(length=256), nullable=True),
|
||||
sa.Column('transactions_root', sa.VARCHAR(length=256), nullable=True),
|
||||
sa.Column('indexed_at', sa.DateTime(timezone=True), server_default=sa.text("TIMEZONE('utc', statement_timestamp())"), nullable=False),
|
||||
sa.PrimaryKeyConstraint('block_number', name=op.f('pk_wyrm_blocks'))
|
||||
)
|
||||
op.create_index(op.f('ix_wyrm_blocks_block_number'), 'wyrm_blocks', ['block_number'], unique=True)
|
||||
op.create_index(op.f('ix_wyrm_blocks_hash'), 'wyrm_blocks', ['hash'], unique=False)
|
||||
op.create_index(op.f('ix_wyrm_blocks_timestamp'), 'wyrm_blocks', ['timestamp'], unique=False)
|
||||
op.create_table('wyrm_labels',
|
||||
sa.Column('id', sa.UUID(), nullable=False),
|
||||
sa.Column('label', sa.VARCHAR(length=256), nullable=False),
|
||||
sa.Column('block_number', sa.BigInteger(), nullable=True),
|
||||
sa.Column('address', sa.VARCHAR(length=256), nullable=True),
|
||||
sa.Column('transaction_hash', sa.VARCHAR(length=256), nullable=True),
|
||||
sa.Column('label_data', postgresql.JSONB(astext_type=sa.Text()), nullable=True),
|
||||
sa.Column('block_timestamp', sa.BigInteger(), nullable=True),
|
||||
sa.Column('log_index', sa.Integer(), nullable=True),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text("TIMEZONE('utc', statement_timestamp())"), nullable=False),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_wyrm_labels')),
|
||||
sa.UniqueConstraint('id', name=op.f('uq_wyrm_labels_id'))
|
||||
)
|
||||
op.create_index(op.f('ix_wyrm_labels_address'), 'wyrm_labels', ['address'], unique=False)
|
||||
op.create_index('ix_wyrm_labels_address_block_number', 'wyrm_labels', ['address', 'block_number'], unique=False)
|
||||
op.create_index('ix_wyrm_labels_address_block_timestamp', 'wyrm_labels', ['address', 'block_timestamp'], unique=False)
|
||||
op.create_index(op.f('ix_wyrm_labels_block_number'), 'wyrm_labels', ['block_number'], unique=False)
|
||||
op.create_index(op.f('ix_wyrm_labels_block_timestamp'), 'wyrm_labels', ['block_timestamp'], unique=False)
|
||||
op.create_index(op.f('ix_wyrm_labels_label'), 'wyrm_labels', ['label'], unique=False)
|
||||
op.create_index(op.f('ix_wyrm_labels_transaction_hash'), 'wyrm_labels', ['transaction_hash'], unique=False)
|
||||
op.create_table('wyrm_transactions',
|
||||
sa.Column('hash', sa.VARCHAR(length=256), nullable=False),
|
||||
sa.Column('block_number', sa.BigInteger(), nullable=False),
|
||||
sa.Column('from_address', sa.VARCHAR(length=256), nullable=True),
|
||||
sa.Column('to_address', sa.VARCHAR(length=256), nullable=True),
|
||||
sa.Column('gas', sa.Numeric(precision=78, scale=0), nullable=True),
|
||||
sa.Column('gas_price', sa.Numeric(precision=78, scale=0), nullable=True),
|
||||
sa.Column('max_fee_per_gas', sa.Numeric(precision=78, scale=0), nullable=True),
|
||||
sa.Column('max_priority_fee_per_gas', sa.Numeric(precision=78, scale=0), nullable=True),
|
||||
sa.Column('input', sa.Text(), nullable=True),
|
||||
sa.Column('nonce', sa.VARCHAR(length=256), nullable=True),
|
||||
sa.Column('transaction_index', sa.BigInteger(), nullable=True),
|
||||
sa.Column('transaction_type', sa.Integer(), nullable=True),
|
||||
sa.Column('value', sa.Numeric(precision=78, scale=0), nullable=True),
|
||||
sa.Column('indexed_at', sa.DateTime(timezone=True), server_default=sa.text("TIMEZONE('utc', statement_timestamp())"), nullable=False),
|
||||
sa.ForeignKeyConstraint(['block_number'], ['wyrm_blocks.block_number'], name=op.f('fk_wyrm_transactions_block_number_wyrm_blocks'), ondelete='CASCADE'),
|
||||
sa.PrimaryKeyConstraint('hash', name=op.f('pk_wyrm_transactions'))
|
||||
)
|
||||
op.create_index(op.f('ix_wyrm_transactions_block_number'), 'wyrm_transactions', ['block_number'], unique=False)
|
||||
op.create_index(op.f('ix_wyrm_transactions_from_address'), 'wyrm_transactions', ['from_address'], unique=False)
|
||||
op.create_index(op.f('ix_wyrm_transactions_gas'), 'wyrm_transactions', ['gas'], unique=False)
|
||||
op.create_index(op.f('ix_wyrm_transactions_gas_price'), 'wyrm_transactions', ['gas_price'], unique=False)
|
||||
op.create_index(op.f('ix_wyrm_transactions_hash'), 'wyrm_transactions', ['hash'], unique=True)
|
||||
op.create_index(op.f('ix_wyrm_transactions_to_address'), 'wyrm_transactions', ['to_address'], unique=False)
|
||||
op.create_index(op.f('ix_wyrm_transactions_value'), 'wyrm_transactions', ['value'], unique=False)
|
||||
# ### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade():
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.drop_index(op.f('ix_wyrm_transactions_value'), table_name='wyrm_transactions')
|
||||
op.drop_index(op.f('ix_wyrm_transactions_to_address'), table_name='wyrm_transactions')
|
||||
op.drop_index(op.f('ix_wyrm_transactions_hash'), table_name='wyrm_transactions')
|
||||
op.drop_index(op.f('ix_wyrm_transactions_gas_price'), table_name='wyrm_transactions')
|
||||
op.drop_index(op.f('ix_wyrm_transactions_gas'), table_name='wyrm_transactions')
|
||||
op.drop_index(op.f('ix_wyrm_transactions_from_address'), table_name='wyrm_transactions')
|
||||
op.drop_index(op.f('ix_wyrm_transactions_block_number'), table_name='wyrm_transactions')
|
||||
op.drop_table('wyrm_transactions')
|
||||
op.drop_index(op.f('ix_wyrm_labels_transaction_hash'), table_name='wyrm_labels')
|
||||
op.drop_index(op.f('ix_wyrm_labels_label'), table_name='wyrm_labels')
|
||||
op.drop_index(op.f('ix_wyrm_labels_block_timestamp'), table_name='wyrm_labels')
|
||||
op.drop_index(op.f('ix_wyrm_labels_block_number'), table_name='wyrm_labels')
|
||||
op.drop_index('ix_wyrm_labels_address_block_timestamp', table_name='wyrm_labels')
|
||||
op.drop_index('ix_wyrm_labels_address_block_number', table_name='wyrm_labels')
|
||||
op.drop_index(op.f('ix_wyrm_labels_address'), table_name='wyrm_labels')
|
||||
op.drop_table('wyrm_labels')
|
||||
op.drop_index(op.f('ix_wyrm_blocks_timestamp'), table_name='wyrm_blocks')
|
||||
op.drop_index(op.f('ix_wyrm_blocks_hash'), table_name='wyrm_blocks')
|
||||
op.drop_index(op.f('ix_wyrm_blocks_block_number'), table_name='wyrm_blocks')
|
||||
op.drop_table('wyrm_blocks')
|
||||
# ### end Alembic commands ###
|
||||
|
|
@ -12,6 +12,9 @@ from .models import (
|
|||
XDaiBlock,
|
||||
XDaiLabel,
|
||||
XDaiTransaction,
|
||||
WyrmBlock,
|
||||
WyrmLabel,
|
||||
WyrmTransaction,
|
||||
)
|
||||
|
||||
from enum import Enum
|
||||
|
|
@ -24,16 +27,19 @@ class AvailableBlockchainType(Enum):
|
|||
POLYGON = "polygon"
|
||||
MUMBAI = "mumbai"
|
||||
XDAI = "xdai"
|
||||
WYRM = "wyrm"
|
||||
|
||||
|
||||
def get_block_model(
|
||||
blockchain_type: AvailableBlockchainType,
|
||||
) -> Type[Union[EthereumBlock, PolygonBlock, MumbaiBlock, XDaiBlock]]:
|
||||
) -> Type[Union[EthereumBlock, PolygonBlock, MumbaiBlock, XDaiBlock, WyrmBlock]]:
|
||||
"""
|
||||
Depends on provided blockchain type: Ethereum, Polygon, Mumbai or XDai,
|
||||
Depends on provided blockchain type: Ethereum, Polygon, Mumbai, XDai, Wyrm
|
||||
set proper blocks model.
|
||||
"""
|
||||
block_model: Type[Union[EthereumBlock, PolygonBlock, MumbaiBlock, XDaiBlock]]
|
||||
block_model: Type[
|
||||
Union[EthereumBlock, PolygonBlock, MumbaiBlock, XDaiBlock, WyrmBlock]
|
||||
]
|
||||
if blockchain_type == AvailableBlockchainType.ETHEREUM:
|
||||
block_model = EthereumBlock
|
||||
elif blockchain_type == AvailableBlockchainType.POLYGON:
|
||||
|
|
@ -42,6 +48,8 @@ def get_block_model(
|
|||
block_model = MumbaiBlock
|
||||
elif blockchain_type == AvailableBlockchainType.XDAI:
|
||||
block_model = XDaiBlock
|
||||
elif blockchain_type == AvailableBlockchainType.WYRM:
|
||||
block_model = WyrmBlock
|
||||
else:
|
||||
raise Exception("Unsupported blockchain type provided")
|
||||
|
||||
|
|
@ -50,12 +58,14 @@ def get_block_model(
|
|||
|
||||
def get_label_model(
|
||||
blockchain_type: AvailableBlockchainType,
|
||||
) -> Type[Union[EthereumLabel, PolygonLabel, MumbaiLabel, XDaiLabel]]:
|
||||
) -> Type[Union[EthereumLabel, PolygonLabel, MumbaiLabel, XDaiLabel, WyrmLabel]]:
|
||||
"""
|
||||
Depends on provided blockchain type: Ethereum, Polygon, Mumbai or XDai,
|
||||
Depends on provided blockchain type: Ethereum, Polygon, Mumbai, XDai, Wyrm
|
||||
set proper block label model.
|
||||
"""
|
||||
label_model: Type[Union[EthereumLabel, PolygonLabel, MumbaiLabel, XDaiLabel]]
|
||||
label_model: Type[
|
||||
Union[EthereumLabel, PolygonLabel, MumbaiLabel, XDaiLabel, WyrmLabel]
|
||||
]
|
||||
if blockchain_type == AvailableBlockchainType.ETHEREUM:
|
||||
label_model = EthereumLabel
|
||||
elif blockchain_type == AvailableBlockchainType.POLYGON:
|
||||
|
|
@ -64,6 +74,8 @@ def get_label_model(
|
|||
label_model = MumbaiLabel
|
||||
elif blockchain_type == AvailableBlockchainType.XDAI:
|
||||
label_model = XDaiLabel
|
||||
elif blockchain_type == AvailableBlockchainType.WYRM:
|
||||
label_model = WyrmLabel
|
||||
else:
|
||||
raise Exception("Unsupported blockchain type provided")
|
||||
|
||||
|
|
@ -73,15 +85,25 @@ def get_label_model(
|
|||
def get_transaction_model(
|
||||
blockchain_type: AvailableBlockchainType,
|
||||
) -> Type[
|
||||
Union[EthereumTransaction, PolygonTransaction, MumbaiTransaction, XDaiTransaction]
|
||||
Union[
|
||||
EthereumTransaction,
|
||||
PolygonTransaction,
|
||||
MumbaiTransaction,
|
||||
XDaiTransaction,
|
||||
WyrmTransaction,
|
||||
]
|
||||
]:
|
||||
"""
|
||||
Depends on provided blockchain type: Ethereum, Polygon, Mumbai or XDai,
|
||||
Depends on provided blockchain type: Ethereum, Polygon, Mumbai, XDai, Wyrm
|
||||
set proper block transactions model.
|
||||
"""
|
||||
transaction_model: Type[
|
||||
Union[
|
||||
EthereumTransaction, PolygonTransaction, MumbaiTransaction, XDaiTransaction
|
||||
EthereumTransaction,
|
||||
PolygonTransaction,
|
||||
MumbaiTransaction,
|
||||
XDaiTransaction,
|
||||
WyrmTransaction,
|
||||
]
|
||||
]
|
||||
if blockchain_type == AvailableBlockchainType.ETHEREUM:
|
||||
|
|
@ -92,6 +114,8 @@ def get_transaction_model(
|
|||
transaction_model = MumbaiTransaction
|
||||
elif blockchain_type == AvailableBlockchainType.XDAI:
|
||||
transaction_model = XDaiTransaction
|
||||
elif blockchain_type == AvailableBlockchainType.WYRM:
|
||||
transaction_model = WyrmTransaction
|
||||
else:
|
||||
raise Exception("Unsupported blockchain type provided")
|
||||
|
||||
|
|
|
|||
|
|
@ -16,7 +16,6 @@ def labels_add_handler(args: argparse.Namespace) -> None:
|
|||
raise ValueError("Unable to parse data as dictionary")
|
||||
|
||||
with yield_db_session_ctx() as db_session:
|
||||
|
||||
label = EthereumLabel(
|
||||
label=args.label, address=str(args.address), label_data=label_data
|
||||
)
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ Moonstream database connection.
|
|||
"""
|
||||
from contextlib import contextmanager
|
||||
import os
|
||||
from typing import Generator
|
||||
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import sessionmaker, Session
|
||||
|
|
@ -60,7 +61,7 @@ engine = create_moonstream_engine(
|
|||
SessionLocal = sessionmaker(bind=engine)
|
||||
|
||||
|
||||
def yield_db_session() -> Session:
|
||||
def yield_db_session() -> Generator[Session, None, None]:
|
||||
"""
|
||||
Yields a database connection (created using environment variables).
|
||||
As per FastAPI docs:
|
||||
|
|
@ -84,7 +85,7 @@ RO_engine = create_moonstream_engine(
|
|||
RO_SessionLocal = sessionmaker(bind=RO_engine)
|
||||
|
||||
|
||||
def yield_db_read_only_session() -> Session:
|
||||
def yield_db_read_only_session() -> Generator[Session, None, None]:
|
||||
"""
|
||||
Yields a database connection (created using environment variables).
|
||||
As per FastAPI docs:
|
||||
|
|
|
|||
|
|
@ -42,7 +42,7 @@ Following:
|
|||
|
||||
|
||||
class utcnow(expression.FunctionElement):
|
||||
type = DateTime
|
||||
type = DateTime # type: ignore
|
||||
|
||||
|
||||
@compiles(utcnow, "postgresql")
|
||||
|
|
@ -493,6 +493,127 @@ class XDaiLabel(Base): # type: ignore
|
|||
)
|
||||
|
||||
|
||||
class WyrmBlock(Base): # type: ignore
|
||||
__tablename__ = "wyrm_blocks"
|
||||
|
||||
block_number = Column(
|
||||
BigInteger, primary_key=True, unique=True, nullable=False, index=True
|
||||
)
|
||||
difficulty = Column(BigInteger)
|
||||
extra_data = Column(VARCHAR(128))
|
||||
gas_limit = Column(BigInteger)
|
||||
gas_used = Column(BigInteger)
|
||||
base_fee_per_gas = Column(Numeric(precision=78, scale=0), nullable=True)
|
||||
hash = Column(VARCHAR(256), index=True)
|
||||
logs_bloom = Column(VARCHAR(1024))
|
||||
miner = Column(VARCHAR(256))
|
||||
nonce = Column(VARCHAR(256))
|
||||
parent_hash = Column(VARCHAR(256))
|
||||
receipt_root = Column(VARCHAR(256))
|
||||
uncles = Column(VARCHAR(256))
|
||||
size = Column(Integer)
|
||||
state_root = Column(VARCHAR(256))
|
||||
timestamp = Column(BigInteger, index=True)
|
||||
total_difficulty = Column(VARCHAR(256))
|
||||
transactions_root = Column(VARCHAR(256))
|
||||
indexed_at = Column(
|
||||
DateTime(timezone=True), server_default=utcnow(), nullable=False
|
||||
)
|
||||
|
||||
|
||||
class WyrmTransaction(Base): # type: ignore
|
||||
__tablename__ = "wyrm_transactions"
|
||||
|
||||
hash = Column(
|
||||
VARCHAR(256), primary_key=True, unique=True, nullable=False, index=True
|
||||
)
|
||||
block_number = Column(
|
||||
BigInteger,
|
||||
ForeignKey("wyrm_blocks.block_number", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
index=True,
|
||||
)
|
||||
from_address = Column(VARCHAR(256), index=True)
|
||||
to_address = Column(VARCHAR(256), index=True)
|
||||
gas = Column(Numeric(precision=78, scale=0), index=True)
|
||||
gas_price = Column(Numeric(precision=78, scale=0), index=True)
|
||||
max_fee_per_gas = Column(Numeric(precision=78, scale=0), nullable=True)
|
||||
max_priority_fee_per_gas = Column(Numeric(precision=78, scale=0), nullable=True)
|
||||
input = Column(Text)
|
||||
nonce = Column(VARCHAR(256))
|
||||
transaction_index = Column(BigInteger)
|
||||
transaction_type = Column(Integer, nullable=True)
|
||||
value = Column(Numeric(precision=78, scale=0), index=True)
|
||||
|
||||
indexed_at = Column(
|
||||
DateTime(timezone=True), server_default=utcnow(), nullable=False
|
||||
)
|
||||
|
||||
|
||||
class WyrmLabel(Base): # type: ignore
|
||||
"""
|
||||
Example of label_data:
|
||||
{
|
||||
"label": "ERC20",
|
||||
"label_data": {
|
||||
"name": "Uniswap",
|
||||
"symbol": "UNI"
|
||||
}
|
||||
},
|
||||
{
|
||||
"label": "Exchange"
|
||||
"label_data": {...}
|
||||
}
|
||||
"""
|
||||
|
||||
__tablename__ = "wyrm_labels"
|
||||
|
||||
__table_args__ = (
|
||||
Index(
|
||||
"ix_wyrm_labels_address_block_number",
|
||||
"address",
|
||||
"block_number",
|
||||
unique=False,
|
||||
),
|
||||
Index(
|
||||
"ix_wyrm_labels_address_block_timestamp",
|
||||
"address",
|
||||
"block_timestamp",
|
||||
unique=False,
|
||||
),
|
||||
)
|
||||
|
||||
id = Column(
|
||||
UUID(as_uuid=True),
|
||||
primary_key=True,
|
||||
default=uuid.uuid4,
|
||||
unique=True,
|
||||
nullable=False,
|
||||
)
|
||||
label = Column(VARCHAR(256), nullable=False, index=True)
|
||||
block_number = Column(
|
||||
BigInteger,
|
||||
nullable=True,
|
||||
index=True,
|
||||
)
|
||||
address = Column(
|
||||
VARCHAR(256),
|
||||
nullable=True,
|
||||
index=True,
|
||||
)
|
||||
transaction_hash = Column(
|
||||
VARCHAR(256),
|
||||
nullable=True,
|
||||
index=True,
|
||||
)
|
||||
label_data = Column(JSONB, nullable=True)
|
||||
block_timestamp = Column(BigInteger, index=True)
|
||||
log_index = Column(Integer, nullable=True)
|
||||
created_at = Column(
|
||||
DateTime(timezone=True), server_default=utcnow(), nullable=False
|
||||
)
|
||||
|
||||
|
||||
class ESDFunctionSignature(Base): # type: ignore
|
||||
"""
|
||||
Function signature from blockchain (Ethereum/Polygon) Signature Database.
|
||||
|
|
|
|||
|
|
@ -0,0 +1,66 @@
|
|||
from enum import Enum
|
||||
from typing import Dict, Union
|
||||
|
||||
from .models import (
|
||||
Base,
|
||||
EthereumBlock,
|
||||
EthereumLabel,
|
||||
EthereumTransaction,
|
||||
MumbaiBlock,
|
||||
MumbaiLabel,
|
||||
MumbaiTransaction,
|
||||
PolygonBlock,
|
||||
PolygonLabel,
|
||||
PolygonTransaction,
|
||||
WyrmBlock,
|
||||
WyrmLabel,
|
||||
WyrmTransaction,
|
||||
XDaiBlock,
|
||||
XDaiLabel,
|
||||
XDaiTransaction,
|
||||
)
|
||||
|
||||
|
||||
class Network(Enum):
|
||||
ethereum = "ethereum"
|
||||
polygon = "polygon"
|
||||
mumbai = "mumbai"
|
||||
xdai = "xdai"
|
||||
wyrm = "wyrm"
|
||||
|
||||
|
||||
tx_raw_types = Union[
|
||||
EthereumTransaction,
|
||||
MumbaiTransaction,
|
||||
PolygonTransaction,
|
||||
WyrmTransaction,
|
||||
XDaiTransaction,
|
||||
]
|
||||
|
||||
MODELS: Dict[Network, Dict[str, Base]] = {
|
||||
Network.ethereum: {
|
||||
"blocks": EthereumBlock,
|
||||
"labels": EthereumLabel,
|
||||
"transactions": EthereumTransaction,
|
||||
},
|
||||
Network.mumbai: {
|
||||
"blocks": MumbaiBlock,
|
||||
"labels": MumbaiLabel,
|
||||
"transactions": MumbaiTransaction,
|
||||
},
|
||||
Network.polygon: {
|
||||
"blocks": PolygonBlock,
|
||||
"labels": PolygonLabel,
|
||||
"transactions": PolygonTransaction,
|
||||
},
|
||||
Network.xdai: {
|
||||
"blocks": XDaiBlock,
|
||||
"labels": XDaiLabel,
|
||||
"transactions": XDaiTransaction,
|
||||
},
|
||||
Network.wyrm: {
|
||||
"blocks": WyrmBlock,
|
||||
"labels": WyrmLabel,
|
||||
"transactions": WyrmTransaction,
|
||||
},
|
||||
}
|
||||
|
|
@ -2,4 +2,4 @@
|
|||
Moonstream database version.
|
||||
"""
|
||||
|
||||
MOONSTREAMDB_VERSION = "0.3.2"
|
||||
MOONSTREAMDB_VERSION = "0.3.3"
|
||||
|
|
|
|||
|
|
@ -32,7 +32,7 @@ setup(
|
|||
packages=find_packages(),
|
||||
package_data={"moonstreamdb": ["py.typed"]},
|
||||
zip_safe=False,
|
||||
install_requires=["alembic", "psycopg2-binary", "sqlalchemy"],
|
||||
install_requires=["alembic", "psycopg2-binary", "sqlalchemy>=2.0.4"],
|
||||
extras_require={
|
||||
"dev": ["black", "isort", "mypy"],
|
||||
"distribute": ["setuptools", "twine", "wheel"],
|
||||
|
|
|
|||
Ładowanie…
Reference in New Issue