Merge branch 'main' into add-moonworm-v3-db-crawler

pull/1085/head
kompotkot 2024-06-11 18:43:31 +00:00
commit 5a021d51ff
9 zmienionych plików z 707 dodań i 4 usunięć

Wyświetl plik

@ -6,6 +6,7 @@ import argparse
import json
import logging
import os
import uuid
from posix import listdir
from typing import Any, Callable, Dict, List, Optional, Union
@ -35,6 +36,13 @@ logger = logging.getLogger(__name__)
MIGRATIONS_FOLDER = "./moonstreamapi/admin/migrations"
def uuid_type(value):
try:
return uuid.UUID(value)
except ValueError:
raise argparse.ArgumentTypeError(f"{value} is not a valid UUID.")
def parse_boolean_arg(raw_arg: Optional[str]) -> Optional[bool]:
if raw_arg is None:
return None
@ -285,6 +293,28 @@ def generate_usage_handler(args: argparse.Namespace) -> None:
output_file.write(json.dumps(usage_info, indent=4))
def moonworm_tasks_v3_migrate(args: argparse.Namespace) -> None:
"""
Read users subsriptions and rewrite them to v3 jobs table
"""
### Request user resources from brood
moonworm_tasks.migrate_v3_tasks(
user_id=args.user_id, customer_id=args.customer_id, blockchain=args.blockchain
)
def create_v3_task_handler(args: argparse.Namespace) -> None:
moonworm_tasks.create_v3_task(
user_id=args.user_id,
customer_id=args.customer_id,
blockchain=args.blockchain,
address=args.address,
abi=json.loads(args.abi.read()),
)
def main() -> None:
cli_description = f"""Moonstream Admin CLI
@ -539,6 +569,76 @@ This CLI is configured to work with the following API URLs:
parser_moonworm_tasks_add.set_defaults(func=moonworm_tasks_add_subscription_handler)
parser_moonworm_tasks_migrate = subcommands_moonworm_tasks.add_parser(
"migrate-v2-tasks",
description="Migrate moonworm tasks to abi_jobs of moonstream index",
)
parser_moonworm_tasks_migrate.add_argument(
"--user-id",
required=True,
type=uuid_type,
help="user-id of which we want see subscription.",
)
parser_moonworm_tasks_migrate.add_argument(
"--customer-id",
required=True,
type=uuid_type,
help="customer-id of which we want see subscription.",
)
parser_moonworm_tasks_migrate.add_argument(
"--blockchain",
required=False,
type=str,
help="Blockchain of which we want see subscription.",
)
parser_moonworm_tasks_migrate.set_defaults(func=moonworm_tasks_v3_migrate)
parser_moonworm_tasks_v3_create = subcommands_moonworm_tasks.add_parser(
"create_v3_tasks",
description="Create v3 tasks from v2 tasks",
)
parser_moonworm_tasks_v3_create.add_argument(
"--user-id",
required=True,
type=uuid_type,
help="user-id of which we want see subscription.",
)
parser_moonworm_tasks_v3_create.add_argument(
"--customer-id",
required=True,
type=uuid_type,
help="customer-id of which we want see subscription.",
)
parser_moonworm_tasks_v3_create.add_argument(
"--blockchain",
required=True,
type=str,
help="Blockchain of which we want see subscription.",
)
parser_moonworm_tasks_v3_create.add_argument(
"--address",
required=True,
type=str,
help="Address of which we want see subscription.",
)
parser_moonworm_tasks_v3_create.add_argument(
"--abi",
required=True,
type=argparse.FileType("r"),
help="ABI of which we want see subscription.",
)
parser_moonworm_tasks_v3_create.set_defaults(func=create_v3_task_handler)
queries_parser = subcommands.add_parser(
"queries", description="Manage Moonstream queries"
)
@ -610,6 +710,44 @@ This CLI is configured to work with the following API URLs:
generate_usage_parser.set_defaults(func=generate_usage_handler)
### databases commands
databases_parser = subcommands.add_parser(
"databases", description="Manage Moonstream databases"
)
databases_parser.set_defaults(func=lambda _: databases_parser.print_help())
databases_subcommands = databases_parser.add_subparsers(
description="Database commands"
)
database_labels_migration_parser = databases_subcommands.add_parser(
"v2-to-v3-labels-migration",
description="Migrate labels in database",
)
database_labels_migration_parser.add_argument(
"--user-id",
type=uuid_type,
help="User ID for which to migrate labels",
)
database_labels_migration_parser.add_argument(
"--customer-id",
type=uuid_type,
help="Customer ID for which to migrate labels",
)
database_labels_migration_parser.add_argument(
"--blockchain",
type=str,
help="Blockchain for which to migrate labels",
)
database_labels_migration_parser.set_defaults(
func=lambda args: print("Not implemented yet")
)
args = parser.parse_args()
args.func(args)

Wyświetl plik

@ -1,13 +1,25 @@
import json
import logging
import os
from typing import Any, Dict, List, Optional, Union
from uuid import UUID
import boto3 # type: ignore
from bugout.data import BugoutResource, BugoutResources
from bugout.data import BugoutResource, BugoutResources, BugoutSearchResult
from bugout.exceptions import BugoutResponseException
from moonstreamdbv3.db import MoonstreamDBIndexesEngine
from moonstreamdbv3.models_indexes import AbiJobs
from sqlalchemy.dialects.postgresql import insert
from web3 import Web3
from ..actions import apply_moonworm_tasks, get_all_entries_from_search
from ..settings import MOONSTREAM_ADMIN_ACCESS_TOKEN, MOONSTREAM_MOONWORM_TASKS_JOURNAL
from ..settings import (
BUGOUT_REQUEST_TIMEOUT_SECONDS,
MOONSTREAM_ADMIN_ACCESS_TOKEN,
MOONSTREAM_MOONWORM_TASKS_JOURNAL,
)
from ..settings import bugout_client as bc
from .subscription_types import CANONICAL_SUBSCRIPTION_TYPES
logger = logging.getLogger(__name__)
@ -81,3 +93,251 @@ def add_subscription(id: str):
)
else:
logging.info("For apply to moonworm tasks subscriptions must have an abi.")
def create_v3_task(
customer_id: str,
user_id: str,
abi: Dict[str, Any],
address: str,
blockchain: str,
):
"""
Create moonworm task for v3
"""
abi_tasks = []
db_engine = MoonstreamDBIndexesEngine()
with db_engine.yield_db_session_ctx() as db_session_v3:
for abi_task in abi:
if abi_task["type"] != "event" and abi_task["type"] != "function":
continue
abi_selector = Web3.keccak(
text=abi_task["name"]
+ "("
+ ",".join(map(lambda x: x["type"], abi_task["inputs"]))
+ ")"
)
if abi_task["type"] == "function":
abi_selector = abi_selector[:4]
abi_selector = abi_selector.hex()
try:
abi_tasks.append(
{
"address": bytes.fromhex(address[2:]),
"user_id": user_id,
"customer_id": customer_id,
"abi_selector": abi_selector,
"chain": blockchain,
"abi_name": abi_task["name"],
"status": "active",
"historical_crawl_status": "pending",
"progress": 0,
"moonworm_task_pickedup": False,
"abi": json.dumps(abi_task),
}
)
except Exception as e:
logger.error(
f"Error creating subscription for subscription for abi {abi_task['name']}: {str(e)}"
)
db_session_v3.rollback()
raise e
insert_statement = insert(AbiJobs).values(abi_tasks)
result_stmt = insert_statement.on_conflict_do_nothing(
index_elements=[
AbiJobs.chain,
AbiJobs.address,
AbiJobs.abi_selector,
AbiJobs.customer_id,
]
)
try:
db_session_v3.execute(result_stmt)
db_session_v3.commit()
except Exception as e:
logger.error(f"Error inserting subscriptions: {str(e)}")
db_session_v3.rollback()
return None
def migrate_v3_tasks(
user_id: UUID, customer_id: UUID, blockchain: Optional[str] = None
):
"""
Migrate moonworm tasks
"""
### get user subscription entity journal id
subscription_resources = bc.list_resources(
token=MOONSTREAM_ADMIN_ACCESS_TOKEN, # type: ignore
params={
"user_id": user_id,
"type": "entity_subscription",
},
timeout=BUGOUT_REQUEST_TIMEOUT_SECONDS,
)
chain_to_subscription_type = {
CANONICAL_SUBSCRIPTION_TYPES[key].blockchain: key
for key in CANONICAL_SUBSCRIPTION_TYPES.keys()
if key.endswith("smartcontract")
}
logger.info(
"Found users collection resources: %s", len(subscription_resources.resources)
)
db_engine = MoonstreamDBIndexesEngine()
if len(subscription_resources.resources) == 0:
raise Exception("User has no subscriptions")
collection_id = subscription_resources.resources[0].resource_data["collection_id"]
query = f"tag:type:subscription"
if blockchain is not None:
query += f" tag:subscription_type_id:{chain_to_subscription_type[blockchain]}"
subscriptions: List[BugoutSearchResult] = get_all_entries_from_search(
journal_id=collection_id,
search_query=query,
token=os.environ.get("SPECIFIC_ACCESS_TOKEN"),
limit=100,
content=True,
)
logger.info("Found users subscriptions: %s", len(subscriptions))
if len(subscriptions) == 0:
raise Exception("User has no subscriptions")
with db_engine.yield_db_session_ctx() as session:
user_subscriptions = []
for index, subscription in enumerate(subscriptions):
abis = None
address = None
subscription_type_id = None
if subscription.content is None:
continue
subscription_data = json.loads(subscription.content)
if "abi" in subscription_data:
abis_container = subscription_data["abi"]
for tag in subscription.tags:
if tag.startswith("subscription_type_id:"):
subscription_type_id = tag.split(":")[1]
if tag.startswith("address:"):
address = tag.split(":")[1]
if subscription_type_id is None:
continue
if abis_container is not None:
try:
abis = json.loads(abis_container)
except Exception as e:
logger.error(
f"Error loading abi for subscription {subscription.id}: {str(e)}"
)
continue
### reformat abi to separate abi tasks
chain = CANONICAL_SUBSCRIPTION_TYPES[subscription_type_id].blockchain
for abi_task in abis:
if abi_task["type"] not in ("event", "function"):
continue
abi_selector = Web3.keccak(
text=abi_task["name"]
+ "("
+ ",".join(map(lambda x: x["type"], abi_task["inputs"]))
+ ")"
)
if abi_task["type"] == "function":
abi_selector = abi_selector[:4]
abi_selector = abi_selector.hex()
try:
abi_job = {
"address": (
bytes.fromhex(address[2:]) if address is not None else None
),
"user_id": user_id,
"customer_id": customer_id,
"abi_selector": abi_selector,
"chain": chain,
"abi_name": abi_task["name"],
"status": "active",
"historical_crawl_status": "pending",
"progress": 0,
"moonworm_task_pickedup": False,
"abi": json.dumps(abi_task),
}
try:
AbiJobs(**abi_job)
except Exception as e:
logger.error(
f"Error creating subscription for subscription {subscription.id}: {str(e)}"
)
continue
user_subscriptions.append(abi_job)
except Exception as e:
logger.error(
f"Error creating subscription for subscription {subscription.id}: {str(e)}"
)
session.rollback()
continue
insert_statement = insert(AbiJobs).values(user_subscriptions)
result_stmt = insert_statement.on_conflict_do_nothing(
index_elements=[
AbiJobs.chain,
AbiJobs.address,
AbiJobs.abi_selector,
AbiJobs.customer_id,
]
)
try:
session.execute(result_stmt)
session.commit()
except Exception as e:
logger.error(f"Error inserting subscriptions: {str(e)}")
session.rollback()
logger.info(f"Processed {index} subscriptions")
return None

Wyświetl plik

@ -2,4 +2,4 @@
Moonstream library and API version.
"""
MOONSTREAMAPI_VERSION = "0.4.2"
MOONSTREAMAPI_VERSION = "0.4.3"

Wyświetl plik

@ -38,6 +38,7 @@ Mako==1.2.3
MarkupSafe==2.1.1
moonstream==0.1.1
moonstreamdb==0.4.4
moonstreamdb-v3==0.0.9
multiaddr==0.0.9
multidict==6.0.2
netaddr==0.8.0

Wyświetl plik

@ -17,6 +17,7 @@ setup(
"fastapi",
"moonstream",
"moonstreamdb>=0.4.4",
"moonstreamdb-v3>=0.0.9",
"humbug",
"pydantic==1.10.2",
"pyevmasm",

Wyświetl plik

@ -0,0 +1,103 @@
"""Logs address selector index
Revision ID: e02c90ea67bb
Revises: a4ef4f9031e4
Create Date: 2024-06-06 13:12:14.594600
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = "e02c90ea67bb"
down_revision: Union[str, None] = "a4ef4f9031e4"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.create_index(
"idx_arbitrum_one_logs_address_selector",
"arbitrum_one_logs",
["address", "selector"],
unique=False,
)
op.create_index(
"idx_arbitrum_sepolia_logs_address_selector",
"arbitrum_sepolia_logs",
["address", "selector"],
unique=False,
)
op.create_index(
"idx_ethereum_logs_address_selector",
"ethereum_logs",
["address", "selector"],
unique=False,
)
op.create_index(
"idx_game7_orbit_arbitrum_sepolia_logs_address_selector",
"game7_orbit_arbitrum_sepolia_logs",
["address", "selector"],
unique=False,
)
op.create_index(
"idx_mantle_logs_address_selector",
"mantle_logs",
["address", "selector"],
unique=False,
)
op.create_index(
"idx_mantle_sepolia_logs_address_selector",
"mantle_sepolia_logs",
["address", "selector"],
unique=False,
)
op.create_index(
"idx_polygon_logs_address_selector",
"polygon_logs",
["address", "selector"],
unique=False,
)
op.create_index(
"idx_xai_logs_address_selector",
"xai_logs",
["address", "selector"],
unique=False,
)
op.create_index(
"idx_xai_sepolia_logs_address_selector",
"xai_sepolia_logs",
["address", "selector"],
unique=False,
)
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.drop_index(
"idx_xai_sepolia_logs_address_selector", table_name="xai_sepolia_logs"
)
op.drop_index("idx_xai_logs_address_selector", table_name="xai_logs")
op.drop_index("idx_polygon_logs_address_selector", table_name="polygon_logs")
op.drop_index(
"idx_mantle_sepolia_logs_address_selector", table_name="mantle_sepolia_logs"
)
op.drop_index("idx_mantle_logs_address_selector", table_name="mantle_logs")
op.drop_index(
"idx_game7_orbit_arbitrum_sepolia_logs_address_selector",
table_name="game7_orbit_arbitrum_sepolia_logs",
)
op.drop_index("idx_ethereum_logs_address_selector", table_name="ethereum_logs")
op.drop_index(
"idx_arbitrum_sepolia_logs_address_selector", table_name="arbitrum_sepolia_logs"
)
op.drop_index(
"idx_arbitrum_one_logs_address_selector", table_name="arbitrum_one_logs"
)
# ### end Alembic commands ###

Wyświetl plik

@ -0,0 +1,167 @@
from enum import Enum
from typing import Type, Union
from .models import (
EthereumLabel,
SepoliaLabel,
PolygonLabel,
MumbaiLabel,
AmoyLabel,
XDaiLabel,
ZkSyncEraLabel,
ZkSyncEraSepoliaLabel,
BaseLabel,
ArbitrumNovaLabel,
ArbitrumOneLabel,
ArbitrumSepoliaLabel,
Game7OrbitArbitrumSepoliaLabel,
XaiLabel,
XaiSepoliaLabel,
AvalancheLabel,
AvalancheFujiLabel,
BlastLabel,
BlastSepoliaLabel,
ProofOfPlayApexLabel,
StarknetLabel,
StarknetSepoliaLabel,
MantleLabel,
MantleSepoliaLabel,
)
class AvailableBlockchainType(Enum):
ETHEREUM = "ethereum"
POLYGON = "polygon"
MUMBAI = "mumbai"
AMOY = "amoy"
XDAI = "xdai"
ZKSYNC_ERA = "zksync_era"
ZKSYNC_ERA_TESTNET = "zksync_era_testnet"
ZKSYNC_ERA_SEPOLIA = "zksync_era_sepolia"
ARBITRUM_ONE = "arbitrum_one"
ARBITRUM_NOVA = "arbitrum_nova"
ARBITRUM_SEPOLIA = "arbitrum_sepolia"
XAI = "xai"
XAI_SEPOLIA = "xai_sepolia"
AVALANCHE = "avalanche"
AVALANCHE_FUJI = "avalanche_fuji"
BLAST = "blast"
BLAST_SEPOLIA = "blast_sepolia"
PROOFOFPLAY_APEX = "proofofplay_apex"
STARKNET = "starknet"
STARKNET_SEPOLIA = "starknet_sepolia"
MANTLE = "mantle"
MANTLE_SEPOLIA = "mantle_sepolia"
GAME7_ORBIT_ARBITRUM_SEPOLIA = "game7_orbit_arbitrum_sepolia"
def get_label_model(blockchain_type: AvailableBlockchainType) -> Type[
Union[
EthereumLabel,
SepoliaLabel,
PolygonLabel,
MumbaiLabel,
AmoyLabel,
XDaiLabel,
ZkSyncEraLabel,
ZkSyncEraSepoliaLabel,
BaseLabel,
ArbitrumNovaLabel,
ArbitrumOneLabel,
ArbitrumSepoliaLabel,
Game7OrbitArbitrumSepoliaLabel,
XaiLabel,
XaiSepoliaLabel,
AvalancheLabel,
AvalancheFujiLabel,
BlastLabel,
BlastSepoliaLabel,
ProofOfPlayApexLabel,
StarknetLabel,
StarknetSepoliaLabel,
MantleLabel,
MantleSepoliaLabel,
]
]:
"""
Depends on provided blockchain type set proper blocks model.
"""
label_model: Type[
Union[
EthereumLabel,
SepoliaLabel,
PolygonLabel,
MumbaiLabel,
AmoyLabel,
XDaiLabel,
ZkSyncEraLabel,
ZkSyncEraSepoliaLabel,
BaseLabel,
ArbitrumNovaLabel,
ArbitrumOneLabel,
ArbitrumSepoliaLabel,
Game7OrbitArbitrumSepoliaLabel,
XaiLabel,
XaiSepoliaLabel,
AvalancheLabel,
AvalancheFujiLabel,
BlastLabel,
BlastSepoliaLabel,
ProofOfPlayApexLabel,
StarknetLabel,
StarknetSepoliaLabel,
MantleLabel,
MantleSepoliaLabel,
]
]
if blockchain_type == AvailableBlockchainType.ETHEREUM:
label_model = EthereumLabel
elif blockchain_type == AvailableBlockchainType.POLYGON:
label_model = PolygonLabel
elif blockchain_type == AvailableBlockchainType.MUMBAI:
label_model = MumbaiLabel
elif blockchain_type == AvailableBlockchainType.AMOY:
label_model = AmoyLabel
elif blockchain_type == AvailableBlockchainType.XDAI:
label_model = XDaiLabel
elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA:
label_model = ZkSyncEraLabel
elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_SEPOLIA:
label_model = ZkSyncEraSepoliaLabel
elif blockchain_type == AvailableBlockchainType.ARBITRUM_ONE:
label_model = ArbitrumOneLabel
elif blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA:
label_model = ArbitrumNovaLabel
elif blockchain_type == AvailableBlockchainType.ARBITRUM_SEPOLIA:
label_model = ArbitrumSepoliaLabel
elif blockchain_type == AvailableBlockchainType.XAI:
label_model = XaiLabel
elif blockchain_type == AvailableBlockchainType.XAI_SEPOLIA:
label_model = XaiSepoliaLabel
elif blockchain_type == AvailableBlockchainType.AVALANCHE:
label_model = AvalancheLabel
elif blockchain_type == AvailableBlockchainType.AVALANCHE_FUJI:
label_model = AvalancheFujiLabel
elif blockchain_type == AvailableBlockchainType.BLAST:
label_model = BlastLabel
elif blockchain_type == AvailableBlockchainType.BLAST_SEPOLIA:
label_model = BlastSepoliaLabel
elif blockchain_type == AvailableBlockchainType.PROOFOFPLAY_APEX:
label_model = ProofOfPlayApexLabel
elif blockchain_type == AvailableBlockchainType.STARKNET:
label_model = StarknetLabel
elif blockchain_type == AvailableBlockchainType.STARKNET_SEPOLIA:
label_model = StarknetSepoliaLabel
elif blockchain_type == AvailableBlockchainType.MANTLE:
label_model = MantleLabel
elif blockchain_type == AvailableBlockchainType.MANTLE_SEPOLIA:
label_model = MantleSepoliaLabel
elif blockchain_type == AvailableBlockchainType.GAME7_ORBIT_ARBITRUM_SEPOLIA:
label_model = Game7OrbitArbitrumSepoliaLabel
else:
raise ValueError(f"Unknown blockchain type: {blockchain_type}")
return label_model

Wyświetl plik

@ -122,6 +122,9 @@ class EthereumLogIndex(EvmBasedLogs):
__tablename__ = "ethereum_logs"
__table_args__ = (
Index(
"idx_ethereum_logs_address_selector", "address", "selector", unique=False
),
Index(
"idx_ethereum_logs_block_hash_log_index",
"block_hash",
@ -172,6 +175,7 @@ class PolygonLogIndex(EvmBasedLogs):
__tablename__ = "polygon_logs"
__table_args__ = (
Index("idx_polygon_logs_address_selector", "address", "selector", unique=False),
UniqueConstraint(
"transaction_hash",
"log_index",
@ -218,6 +222,7 @@ class XaiLogIndex(EvmBasedLogs):
__tablename__ = "xai_logs"
__table_args__ = (
Index("idx_xai_logs_address_selector", "address", "selector", unique=False),
UniqueConstraint(
"transaction_hash",
"log_index",
@ -262,6 +267,9 @@ class XaiSepoliaLogIndex(EvmBasedLogs):
__tablename__ = "xai_sepolia_logs"
__table_args__ = (
Index(
"idx_xai_sepolia_logs_address_selector", "address", "selector", unique=False
),
UniqueConstraint(
"transaction_hash",
"log_index",
@ -308,6 +316,12 @@ class ArbitrumOneLogIndex(EvmBasedLogs):
__tablename__ = "arbitrum_one_logs"
__table_args__ = (
Index(
"idx_arbitrum_one_logs_address_selector",
"address",
"selector",
unique=False,
),
UniqueConstraint(
"transaction_hash",
"log_index",
@ -354,6 +368,12 @@ class ArbitrumSepoliaLogIndex(EvmBasedLogs):
__tablename__ = "arbitrum_sepolia_logs"
__table_args__ = (
Index(
"idx_arbitrum_sepolia_logs_address_selector",
"address",
"selector",
unique=False,
),
UniqueConstraint(
"transaction_hash",
"log_index",
@ -402,6 +422,12 @@ class Game7OrbitArbitrumSepoliaLogIndex(EvmBasedLogs):
__tablename__ = "game7_orbit_arbitrum_sepolia_logs"
__table_args__ = (
Index(
"idx_game7_orbit_arbitrum_sepolia_logs_address_selector",
"address",
"selector",
unique=False,
),
UniqueConstraint(
"transaction_hash",
"log_index",
@ -449,6 +475,7 @@ class MantleLogIndex(EvmBasedLogs):
__tablename__ = "mantle_logs"
__table_args__ = (
Index("idx_mantle_logs_address_selector", "address", "selector", unique=False),
Index(
"idx_mantle_logs_block_hash_log_index",
"block_hash",
@ -498,6 +525,12 @@ class MantleSepoliaLogIndex(EvmBasedLogs):
__tablename__ = "mantle_sepolia_logs"
__table_args__ = (
Index(
"idx_mantle_sepolia_logs_address_selector",
"address",
"selector",
unique=False,
),
Index(
"idx_mantle_sepolia_logs_block_hash_log_index",
"block_hash",

Wyświetl plik

@ -1 +1 @@
0.0.8
0.0.9