diff --git a/db/alembic/env.py b/db/alembic/env.py index 3ab13670..2e7def32 100644 --- a/db/alembic/env.py +++ b/db/alembic/env.py @@ -32,6 +32,9 @@ from moonstreamdb.models import ( PolygonBlock, PolygonTransaction, PolygonLabel, + MumbaiBlock, + MumbaiTransaction, + MumbaiLabel, ESDFunctionSignature, ESDEventSignature, OpenSeaCrawlingState, @@ -46,6 +49,9 @@ def include_symbol(tablename, schema): PolygonBlock.__tablename__, PolygonTransaction.__tablename__, PolygonLabel.__tablename__, + MumbaiBlock.__tablename__, + MumbaiTransaction.__tablename__, + MumbaiLabel.__tablename__, ESDFunctionSignature.__tablename__, ESDEventSignature.__tablename__, OpenSeaCrawlingState.__tablename__, diff --git a/db/alembic/versions/11233cf42d62_mumbai_model.py b/db/alembic/versions/11233cf42d62_mumbai_model.py new file mode 100644 index 00000000..fc6d200b --- /dev/null +++ b/db/alembic/versions/11233cf42d62_mumbai_model.py @@ -0,0 +1,116 @@ +"""mumbai model + +Revision ID: 11233cf42d62 +Revises: 42ca451a9c33 +Create Date: 2022-08-09 12:50:15.154638 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = '11233cf42d62' +down_revision = '42ca451a9c33' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('mumbai_blocks', + sa.Column('block_number', sa.BigInteger(), nullable=False), + sa.Column('difficulty', sa.BigInteger(), nullable=True), + sa.Column('extra_data', sa.VARCHAR(length=128), nullable=True), + sa.Column('gas_limit', sa.BigInteger(), nullable=True), + sa.Column('gas_used', sa.BigInteger(), nullable=True), + sa.Column('base_fee_per_gas', sa.Numeric(precision=78, scale=0), nullable=True), + sa.Column('hash', sa.VARCHAR(length=256), nullable=True), + sa.Column('logs_bloom', sa.VARCHAR(length=1024), nullable=True), + sa.Column('miner', sa.VARCHAR(length=256), nullable=True), + sa.Column('nonce', sa.VARCHAR(length=256), nullable=True), + sa.Column('parent_hash', sa.VARCHAR(length=256), nullable=True), + sa.Column('receipt_root', sa.VARCHAR(length=256), nullable=True), + sa.Column('uncles', sa.VARCHAR(length=256), nullable=True), + sa.Column('size', sa.Integer(), nullable=True), + sa.Column('state_root', sa.VARCHAR(length=256), nullable=True), + sa.Column('timestamp', sa.BigInteger(), nullable=True), + sa.Column('total_difficulty', sa.VARCHAR(length=256), nullable=True), + sa.Column('transactions_root', sa.VARCHAR(length=256), nullable=True), + sa.Column('indexed_at', sa.DateTime(timezone=True), server_default=sa.text("TIMEZONE('utc', statement_timestamp())"), nullable=False), + sa.PrimaryKeyConstraint('block_number', name=op.f('pk_mumbai_blocks')) + ) + op.create_index(op.f('ix_mumbai_blocks_block_number'), 'mumbai_blocks', ['block_number'], unique=True) + op.create_index(op.f('ix_mumbai_blocks_hash'), 'mumbai_blocks', ['hash'], unique=False) + op.create_index(op.f('ix_mumbai_blocks_timestamp'), 'mumbai_blocks', ['timestamp'], unique=False) + op.create_table('mumbai_labels', + sa.Column('id', postgresql.UUID(as_uuid=True), nullable=False), + sa.Column('label', sa.VARCHAR(length=256), nullable=False), + sa.Column('block_number', sa.BigInteger(), nullable=True), + sa.Column('address', sa.VARCHAR(length=256), nullable=True), + sa.Column('transaction_hash', sa.VARCHAR(length=256), nullable=True), + sa.Column('label_data', postgresql.JSONB(astext_type=sa.Text()), nullable=True), + sa.Column('block_timestamp', sa.BigInteger(), nullable=True), + sa.Column('log_index', sa.Integer(), nullable=True), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text("TIMEZONE('utc', statement_timestamp())"), nullable=False), + sa.PrimaryKeyConstraint('id', name=op.f('pk_mumbai_labels')), + sa.UniqueConstraint('id', name=op.f('uq_mumbai_labels_id')) + ) + op.create_index(op.f('ix_mumbai_labels_address'), 'mumbai_labels', ['address'], unique=False) + op.create_index('ix_mumbai_labels_address_block_number', 'mumbai_labels', ['address', 'block_number'], unique=False) + op.create_index('ix_mumbai_labels_address_block_timestamp', 'mumbai_labels', ['address', 'block_timestamp'], unique=False) + op.create_index(op.f('ix_mumbai_labels_block_number'), 'mumbai_labels', ['block_number'], unique=False) + op.create_index(op.f('ix_mumbai_labels_block_timestamp'), 'mumbai_labels', ['block_timestamp'], unique=False) + op.create_index(op.f('ix_mumbai_labels_label'), 'mumbai_labels', ['label'], unique=False) + op.create_index(op.f('ix_mumbai_labels_transaction_hash'), 'mumbai_labels', ['transaction_hash'], unique=False) + op.create_table('mumbai_transactions', + sa.Column('hash', sa.VARCHAR(length=256), nullable=False), + sa.Column('block_number', sa.BigInteger(), nullable=False), + sa.Column('from_address', sa.VARCHAR(length=256), nullable=True), + sa.Column('to_address', sa.VARCHAR(length=256), nullable=True), + sa.Column('gas', sa.Numeric(precision=78, scale=0), nullable=True), + sa.Column('gas_price', sa.Numeric(precision=78, scale=0), nullable=True), + sa.Column('max_fee_per_gas', sa.Numeric(precision=78, scale=0), nullable=True), + sa.Column('max_priority_fee_per_gas', sa.Numeric(precision=78, scale=0), nullable=True), + sa.Column('input', sa.Text(), nullable=True), + sa.Column('nonce', sa.VARCHAR(length=256), nullable=True), + sa.Column('transaction_index', sa.BigInteger(), nullable=True), + sa.Column('transaction_type', sa.Integer(), nullable=True), + sa.Column('value', sa.Numeric(precision=78, scale=0), nullable=True), + sa.Column('indexed_at', sa.DateTime(timezone=True), server_default=sa.text("TIMEZONE('utc', statement_timestamp())"), nullable=False), + sa.ForeignKeyConstraint(['block_number'], ['mumbai_blocks.block_number'], name=op.f('fk_mumbai_transactions_block_number_mumbai_blocks'), ondelete='CASCADE'), + sa.PrimaryKeyConstraint('hash', name=op.f('pk_mumbai_transactions')) + ) + op.create_index(op.f('ix_mumbai_transactions_block_number'), 'mumbai_transactions', ['block_number'], unique=False) + op.create_index(op.f('ix_mumbai_transactions_from_address'), 'mumbai_transactions', ['from_address'], unique=False) + op.create_index(op.f('ix_mumbai_transactions_gas'), 'mumbai_transactions', ['gas'], unique=False) + op.create_index(op.f('ix_mumbai_transactions_gas_price'), 'mumbai_transactions', ['gas_price'], unique=False) + op.create_index(op.f('ix_mumbai_transactions_hash'), 'mumbai_transactions', ['hash'], unique=True) + op.create_index(op.f('ix_mumbai_transactions_to_address'), 'mumbai_transactions', ['to_address'], unique=False) + op.create_index(op.f('ix_mumbai_transactions_value'), 'mumbai_transactions', ['value'], unique=False) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index(op.f('ix_mumbai_transactions_value'), table_name='mumbai_transactions') + op.drop_index(op.f('ix_mumbai_transactions_to_address'), table_name='mumbai_transactions') + op.drop_index(op.f('ix_mumbai_transactions_hash'), table_name='mumbai_transactions') + op.drop_index(op.f('ix_mumbai_transactions_gas_price'), table_name='mumbai_transactions') + op.drop_index(op.f('ix_mumbai_transactions_gas'), table_name='mumbai_transactions') + op.drop_index(op.f('ix_mumbai_transactions_from_address'), table_name='mumbai_transactions') + op.drop_index(op.f('ix_mumbai_transactions_block_number'), table_name='mumbai_transactions') + op.drop_table('mumbai_transactions') + op.drop_index(op.f('ix_mumbai_labels_transaction_hash'), table_name='mumbai_labels') + op.drop_index(op.f('ix_mumbai_labels_label'), table_name='mumbai_labels') + op.drop_index(op.f('ix_mumbai_labels_block_timestamp'), table_name='mumbai_labels') + op.drop_index(op.f('ix_mumbai_labels_block_number'), table_name='mumbai_labels') + op.drop_index('ix_mumbai_labels_address_block_timestamp', table_name='mumbai_labels') + op.drop_index('ix_mumbai_labels_address_block_number', table_name='mumbai_labels') + op.drop_index(op.f('ix_mumbai_labels_address'), table_name='mumbai_labels') + op.drop_table('mumbai_labels') + op.drop_index(op.f('ix_mumbai_blocks_timestamp'), table_name='mumbai_blocks') + op.drop_index(op.f('ix_mumbai_blocks_hash'), table_name='mumbai_blocks') + op.drop_index(op.f('ix_mumbai_blocks_block_number'), table_name='mumbai_blocks') + op.drop_table('mumbai_blocks') + # ### end Alembic commands ### diff --git a/db/moonstreamdb/blockchain.py b/db/moonstreamdb/blockchain.py index 0a78f9c9..4b3d1ae8 100644 --- a/db/moonstreamdb/blockchain.py +++ b/db/moonstreamdb/blockchain.py @@ -6,6 +6,9 @@ from .models import ( PolygonBlock, PolygonLabel, PolygonTransaction, + MumbaiBlock, + MumbaiLabel, + MumbaiTransaction, XDaiBlock, XDaiLabel, XDaiTransaction, @@ -19,21 +22,24 @@ from typing import Type, Union class AvailableBlockchainType(Enum): ETHEREUM = "ethereum" POLYGON = "polygon" + MUMBAI = "mumbai" XDAI = "xdai" def get_block_model( blockchain_type: AvailableBlockchainType, -) -> Type[Union[EthereumBlock, PolygonBlock, XDaiBlock]]: +) -> Type[Union[EthereumBlock, PolygonBlock, MumbaiBlock, XDaiBlock]]: """ - Depends on provided blockchain type: Ethereum, Polygon or XDai, - set proper blocks model: EthereumBlock, PolygonBlock or XdaiBlock. + Depends on provided blockchain type: Ethereum, Polygon, Mumbai or XDai, + set proper blocks model. """ - block_model: Type[Union[EthereumBlock, PolygonBlock, XDaiBlock]] + block_model: Type[Union[EthereumBlock, PolygonBlock, MumbaiBlock, XDaiBlock]] if blockchain_type == AvailableBlockchainType.ETHEREUM: block_model = EthereumBlock elif blockchain_type == AvailableBlockchainType.POLYGON: block_model = PolygonBlock + elif blockchain_type == AvailableBlockchainType.MUMBAI: + block_model = MumbaiBlock elif blockchain_type == AvailableBlockchainType.XDAI: block_model = XDaiBlock else: @@ -44,16 +50,18 @@ def get_block_model( def get_label_model( blockchain_type: AvailableBlockchainType, -) -> Type[Union[EthereumLabel, PolygonLabel, XDaiLabel]]: +) -> Type[Union[EthereumLabel, PolygonLabel, MumbaiLabel, XDaiLabel]]: """ - Depends on provided blockchain type: Ethereum, Polygon or XDai, - set proper block label model: EthereumLabel, PolygonLabel or XdaiLabel. + Depends on provided blockchain type: Ethereum, Polygon, Mumbai or XDai, + set proper block label model. """ - label_model: Type[Union[EthereumLabel, PolygonLabel, XDaiLabel]] + label_model: Type[Union[EthereumLabel, PolygonLabel, MumbaiLabel, XDaiLabel]] if blockchain_type == AvailableBlockchainType.ETHEREUM: label_model = EthereumLabel elif blockchain_type == AvailableBlockchainType.POLYGON: label_model = PolygonLabel + elif blockchain_type == AvailableBlockchainType.MUMBAI: + label_model = MumbaiLabel elif blockchain_type == AvailableBlockchainType.XDAI: label_model = XDaiLabel else: @@ -64,18 +72,24 @@ def get_label_model( def get_transaction_model( blockchain_type: AvailableBlockchainType, -) -> Type[Union[EthereumTransaction, PolygonTransaction, XDaiTransaction]]: +) -> Type[ + Union[EthereumTransaction, PolygonTransaction, MumbaiTransaction, XDaiTransaction] +]: """ - Depends on provided blockchain type: Ethereum or Polygon, - set proper block transactions model: EthereumTransaction or PolygonTransaction. + Depends on provided blockchain type: Ethereum, Polygon, Mumbai or XDai, + set proper block transactions model. """ transaction_model: Type[ - Union[EthereumTransaction, PolygonTransaction, XDaiTransaction] + Union[ + EthereumTransaction, PolygonTransaction, MumbaiTransaction, XDaiTransaction + ] ] if blockchain_type == AvailableBlockchainType.ETHEREUM: transaction_model = EthereumTransaction elif blockchain_type == AvailableBlockchainType.POLYGON: transaction_model = PolygonTransaction + elif blockchain_type == AvailableBlockchainType.MUMBAI: + transaction_model = MumbaiTransaction elif blockchain_type == AvailableBlockchainType.XDAI: transaction_model = XDaiTransaction else: diff --git a/db/moonstreamdb/db.py b/db/moonstreamdb/db.py index 4e6c7383..1fe07d4f 100644 --- a/db/moonstreamdb/db.py +++ b/db/moonstreamdb/db.py @@ -35,17 +35,24 @@ try: ) except: raise ValueError( - f"MOONSTREAM_DB_STATEMENT_TIMEOUT_MILLIOS must be an integer: {MOONSTREAM_DB_STATEMENT_TIMEOUT_MILLIS_RAW}" + f"MOONSTREAM_DB_STATEMENT_TIMEOUT_MILLIS must be an integer: {MOONSTREAM_DB_STATEMENT_TIMEOUT_MILLIS_RAW}" ) -# Pooling: https://docs.sqlalchemy.org/en/14/core/pooling.html#sqlalchemy.pool.QueuePool -# Statement timeout: https://stackoverflow.com/a/44936982 -engine = create_engine( - MOONSTREAM_DB_URI, + +def create_moonstream_engine(url: str, pool_size: int, statement_timeout: int): + # Pooling: https://docs.sqlalchemy.org/en/14/core/pooling.html#sqlalchemy.pool.QueuePool + # Statement timeout: https://stackoverflow.com/a/44936982 + return create_engine( + url=url, + pool_size=pool_size, + connect_args={"options": f"-c statement_timeout={statement_timeout}"}, + ) + + +engine = create_moonstream_engine( + url=MOONSTREAM_DB_URI, pool_size=MOONSTREAM_POOL_SIZE, - connect_args={ - "options": f"-c statement_timeout={MOONSTREAM_DB_STATEMENT_TIMEOUT_MILLIS}" - }, + statement_timeout=MOONSTREAM_DB_STATEMENT_TIMEOUT_MILLIS, ) SessionLocal = sessionmaker(bind=engine) @@ -66,13 +73,10 @@ def yield_db_session() -> Session: yield_db_session_ctx = contextmanager(yield_db_session) # Read only - -RO_engine = create_engine( - MOONSTREAM_DB_URI_READ_ONLY, +RO_engine = create_moonstream_engine( + url=MOONSTREAM_DB_URI_READ_ONLY, pool_size=MOONSTREAM_POOL_SIZE, - connect_args={ - "options": f"-c statement_timeout={MOONSTREAM_DB_STATEMENT_TIMEOUT_MILLIS}" - }, + statement_timeout=MOONSTREAM_DB_STATEMENT_TIMEOUT_MILLIS, ) RO_SessionLocal = sessionmaker(bind=RO_engine) diff --git a/db/moonstreamdb/models.py b/db/moonstreamdb/models.py index ff95e82c..df6a4a38 100644 --- a/db/moonstreamdb/models.py +++ b/db/moonstreamdb/models.py @@ -277,6 +277,112 @@ class PolygonLabel(Base): # type: ignore ) +class MumbaiBlock(Base): # type: ignore + __tablename__ = "mumbai_blocks" + + block_number = Column( + BigInteger, primary_key=True, unique=True, nullable=False, index=True + ) + difficulty = Column(BigInteger) + extra_data = Column(VARCHAR(128)) + gas_limit = Column(BigInteger) + gas_used = Column(BigInteger) + base_fee_per_gas = Column(Numeric(precision=78, scale=0), nullable=True) + hash = Column(VARCHAR(256), index=True) + logs_bloom = Column(VARCHAR(1024)) + miner = Column(VARCHAR(256)) + nonce = Column(VARCHAR(256)) + parent_hash = Column(VARCHAR(256)) + receipt_root = Column(VARCHAR(256)) + uncles = Column(VARCHAR(256)) + size = Column(Integer) + state_root = Column(VARCHAR(256)) + timestamp = Column(BigInteger, index=True) + total_difficulty = Column(VARCHAR(256)) + transactions_root = Column(VARCHAR(256)) + indexed_at = Column( + DateTime(timezone=True), server_default=utcnow(), nullable=False + ) + + +class MumbaiTransaction(Base): # type: ignore + __tablename__ = "mumbai_transactions" + + hash = Column( + VARCHAR(256), primary_key=True, unique=True, nullable=False, index=True + ) + block_number = Column( + BigInteger, + ForeignKey("mumbai_blocks.block_number", ondelete="CASCADE"), + nullable=False, + index=True, + ) + from_address = Column(VARCHAR(256), index=True) + to_address = Column(VARCHAR(256), index=True) + gas = Column(Numeric(precision=78, scale=0), index=True) + gas_price = Column(Numeric(precision=78, scale=0), index=True) + max_fee_per_gas = Column(Numeric(precision=78, scale=0), nullable=True) + max_priority_fee_per_gas = Column(Numeric(precision=78, scale=0), nullable=True) + input = Column(Text) + nonce = Column(VARCHAR(256)) + transaction_index = Column(BigInteger) + transaction_type = Column(Integer, nullable=True) + value = Column(Numeric(precision=78, scale=0), index=True) + + indexed_at = Column( + DateTime(timezone=True), server_default=utcnow(), nullable=False + ) + + +class MumbaiLabel(Base): # type: ignore + __tablename__ = "mumbai_labels" + + __table_args__ = ( + Index( + "ix_mumbai_labels_address_block_number", + "address", + "block_number", + unique=False, + ), + Index( + "ix_mumbai_labels_address_block_timestamp", + "address", + "block_timestamp", + unique=False, + ), + ) + + id = Column( + UUID(as_uuid=True), + primary_key=True, + default=uuid.uuid4, + unique=True, + nullable=False, + ) + label = Column(VARCHAR(256), nullable=False, index=True) + block_number = Column( + BigInteger, + nullable=True, + index=True, + ) + address = Column( + VARCHAR(256), + nullable=True, + index=True, + ) + transaction_hash = Column( + VARCHAR(256), + nullable=True, + index=True, + ) + label_data = Column(JSONB, nullable=True) + block_timestamp = Column(BigInteger, index=True) + log_index = Column(Integer, nullable=True) + created_at = Column( + DateTime(timezone=True), server_default=utcnow(), nullable=False + ) + + class XDaiBlock(Base): # type: ignore __tablename__ = "xdai_blocks" diff --git a/db/moonstreamdb/version.py b/db/moonstreamdb/version.py index 72fa3d89..897118e3 100644 --- a/db/moonstreamdb/version.py +++ b/db/moonstreamdb/version.py @@ -2,4 +2,4 @@ Moonstream database version. """ -MOONSTREAMDB_VERSION = "0.2.5" +MOONSTREAMDB_VERSION = "0.2.6"