diff --git a/backend/requirements.txt b/backend/requirements.txt index 8a39a592..9d5a15f3 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -11,7 +11,7 @@ fastapi==0.66.0 h11==0.12.0 idna==3.2 jmespath==0.10.0 --e git+https://git@github.com/bugout-dev/moonstream.git@b9c828fc7f811af88a9f3a45dd7f5c4053433366#egg=moonstreamdb&subdirectory=db +-e git+https://git@github.com/bugout-dev/moonstream.git@3326379b174a626cf876c7b60dd7db8afe2daba9#egg=moonstreamdb&subdirectory=db mypy==0.910 mypy-extensions==0.4.3 pathspec==0.9.0 diff --git a/crawlers/moonstreamcrawlers/ethereum.py b/crawlers/moonstreamcrawlers/ethereum.py index 20b6c921..ade7c26a 100644 --- a/crawlers/moonstreamcrawlers/ethereum.py +++ b/crawlers/moonstreamcrawlers/ethereum.py @@ -66,13 +66,13 @@ def get_latest_blocks(with_transactions: bool = False) -> None: "latest", full_transactions=with_transactions ) with yield_db_session_ctx() as db_session: - block_latest_exist = ( - db_session.query(EthereumBlock) + block_number_latest_exist = ( + db_session.query(EthereumBlock.block_number) .order_by(EthereumBlock.block_number.desc()) .first() ) - return block_latest_exist.block_number, block_latest.number + return block_number_latest_exist, block_latest.number def crawl_blocks( @@ -107,7 +107,7 @@ def check_missing_blocks(blocks_numbers: List[int]) -> List[int]: for block_number in blocks_numbers: with yield_db_session_ctx() as db_session: block_exist = ( - db_session.query(EthereumBlock) + db_session.query(EthereumBlock.block_number) .filter(EthereumBlock.block_number == block_number) .one_or_none() ) diff --git a/crawlers/requirements.txt b/crawlers/requirements.txt index 748be495..63c16872 100644 Binary files a/crawlers/requirements.txt and b/crawlers/requirements.txt differ diff --git a/db/alembic/versions/1e33c3d07306_added_tables_to_store_data_from_.py b/db/alembic/versions/1e33c3d07306_added_tables_to_store_data_from_.py deleted file mode 100644 index 94ba8a9b..00000000 --- a/db/alembic/versions/1e33c3d07306_added_tables_to_store_data_from_.py +++ /dev/null @@ -1,42 +0,0 @@ -"""Added tables to store data from Ethereum Signature Database - -Revision ID: 1e33c3d07306 -Revises: aa903a90b8bf -Create Date: 2021-07-27 00:04:31.042487 - -""" -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision = '1e33c3d07306' -down_revision = 'aa903a90b8bf' -branch_labels = None -depends_on = None - - -def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.create_table('esd_event_signatures', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('text_signature', sa.Text(), nullable=False), - sa.Column('hex_signature', sa.VARCHAR(length=66), nullable=False), - sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text("TIMEZONE('utc', statement_timestamp())"), nullable=False), - sa.PrimaryKeyConstraint('id', name=op.f('pk_esd_event_signatures')) - ) - op.create_table('esd_function_signatures', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('text_signature', sa.Text(), nullable=False), - sa.Column('hex_signature', sa.VARCHAR(length=10), nullable=False), - sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text("TIMEZONE('utc', statement_timestamp())"), nullable=False), - sa.PrimaryKeyConstraint('id', name=op.f('pk_esd_function_signatures')) - ) - # ### end Alembic commands ### - - -def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.drop_table('esd_function_signatures') - op.drop_table('esd_event_signatures') - # ### end Alembic commands ### diff --git a/db/alembic/versions/aa903a90b8bf_inital_for_blocks_and_transactions.py b/db/alembic/versions/21cced47077c_inital_migration.py similarity index 50% rename from db/alembic/versions/aa903a90b8bf_inital_for_blocks_and_transactions.py rename to db/alembic/versions/21cced47077c_inital_migration.py index 1d1ef8c1..6b04810f 100644 --- a/db/alembic/versions/aa903a90b8bf_inital_for_blocks_and_transactions.py +++ b/db/alembic/versions/21cced47077c_inital_migration.py @@ -1,8 +1,8 @@ -"""Inital for blocks and transactions +"""Inital migration -Revision ID: aa903a90b8bf +Revision ID: 21cced47077c Revises: -Create Date: 2021-07-26 13:55:38.057312 +Create Date: 2021-07-29 20:30:25.863724 """ from alembic import op @@ -10,7 +10,7 @@ import sqlalchemy as sa # revision identifiers, used by Alembic. -revision = 'aa903a90b8bf' +revision = '21cced47077c' down_revision = None branch_labels = None depends_on = None @@ -18,6 +18,22 @@ depends_on = None def upgrade(): # ### commands auto generated by Alembic - please adjust! ### + op.create_table('esd_event_signatures', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('text_signature', sa.Text(), nullable=False), + sa.Column('hex_signature', sa.VARCHAR(length=66), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text("TIMEZONE('utc', statement_timestamp())"), nullable=False), + sa.PrimaryKeyConstraint('id', name=op.f('pk_esd_event_signatures')) + ) + op.create_index(op.f('ix_esd_event_signatures_id'), 'esd_event_signatures', ['id'], unique=True) + op.create_table('esd_function_signatures', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('text_signature', sa.Text(), nullable=False), + sa.Column('hex_signature', sa.VARCHAR(length=10), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text("TIMEZONE('utc', statement_timestamp())"), nullable=False), + sa.PrimaryKeyConstraint('id', name=op.f('pk_esd_function_signatures')) + ) + op.create_index(op.f('ix_esd_function_signatures_id'), 'esd_function_signatures', ['id'], unique=True) op.create_table('ethereum_blocks', sa.Column('block_number', sa.BigInteger(), nullable=False), sa.Column('difficulty', sa.BigInteger(), nullable=True), @@ -37,57 +53,82 @@ def upgrade(): sa.Column('total_difficulty', sa.VARCHAR(length=256), nullable=True), sa.Column('transactions_root', sa.VARCHAR(length=256), nullable=True), sa.Column('indexed_at', sa.DateTime(timezone=True), server_default=sa.text("TIMEZONE('utc', statement_timestamp())"), nullable=False), - sa.PrimaryKeyConstraint('block_number', name=op.f('pk_ethereum_blocks')), - sa.UniqueConstraint('block_number', name=op.f('uq_ethereum_blocks_block_number')) + sa.PrimaryKeyConstraint('block_number', name=op.f('pk_ethereum_blocks')) ) + op.create_index(op.f('ix_ethereum_blocks_block_number'), 'ethereum_blocks', ['block_number'], unique=True) + op.create_index(op.f('ix_ethereum_blocks_hash'), 'ethereum_blocks', ['hash'], unique=False) op.create_index(op.f('ix_ethereum_blocks_timestamp'), 'ethereum_blocks', ['timestamp'], unique=False) op.create_table('ethereum_pending_transactions', sa.Column('hash', sa.VARCHAR(length=256), nullable=False), sa.Column('block_number', sa.BigInteger(), nullable=False), sa.Column('from_address', sa.VARCHAR(length=256), nullable=True), sa.Column('to_address', sa.VARCHAR(length=256), nullable=True), - sa.Column('gas', sa.Text(), nullable=True), - sa.Column('gas_price', sa.Text(), nullable=True), + sa.Column('gas', sa.Numeric(precision=78, scale=0), nullable=True), + sa.Column('gas_price', sa.Numeric(precision=78, scale=0), nullable=True), sa.Column('input', sa.Text(), nullable=True), sa.Column('nonce', sa.VARCHAR(length=256), nullable=True), sa.Column('transaction_index', sa.BigInteger(), nullable=True), - sa.Column('value', sa.Text(), nullable=True), + sa.Column('value', sa.Numeric(precision=78, scale=0), nullable=True), sa.Column('indexed_at', sa.DateTime(timezone=True), server_default=sa.text("TIMEZONE('utc', statement_timestamp())"), nullable=False), sa.ForeignKeyConstraint(['block_number'], ['ethereum_blocks.block_number'], name=op.f('fk_ethereum_pending_transactions_block_number_ethereum_blocks'), ondelete='CASCADE'), - sa.PrimaryKeyConstraint('hash', name=op.f('pk_ethereum_pending_transactions')), - sa.UniqueConstraint('hash', name=op.f('uq_ethereum_pending_transactions_hash')) + sa.PrimaryKeyConstraint('hash', name=op.f('pk_ethereum_pending_transactions')) ) + op.create_index(op.f('ix_ethereum_pending_transactions_block_number'), 'ethereum_pending_transactions', ['block_number'], unique=False) op.create_index(op.f('ix_ethereum_pending_transactions_from_address'), 'ethereum_pending_transactions', ['from_address'], unique=False) + op.create_index(op.f('ix_ethereum_pending_transactions_gas'), 'ethereum_pending_transactions', ['gas'], unique=False) + op.create_index(op.f('ix_ethereum_pending_transactions_gas_price'), 'ethereum_pending_transactions', ['gas_price'], unique=False) + op.create_index(op.f('ix_ethereum_pending_transactions_hash'), 'ethereum_pending_transactions', ['hash'], unique=True) op.create_index(op.f('ix_ethereum_pending_transactions_to_address'), 'ethereum_pending_transactions', ['to_address'], unique=False) + op.create_index(op.f('ix_ethereum_pending_transactions_value'), 'ethereum_pending_transactions', ['value'], unique=False) op.create_table('ethereum_transactions', sa.Column('hash', sa.VARCHAR(length=256), nullable=False), sa.Column('block_number', sa.BigInteger(), nullable=False), sa.Column('from_address', sa.VARCHAR(length=256), nullable=True), sa.Column('to_address', sa.VARCHAR(length=256), nullable=True), - sa.Column('gas', sa.Text(), nullable=True), - sa.Column('gas_price', sa.Text(), nullable=True), + sa.Column('gas', sa.Numeric(precision=78, scale=0), nullable=True), + sa.Column('gas_price', sa.Numeric(precision=78, scale=0), nullable=True), sa.Column('input', sa.Text(), nullable=True), sa.Column('nonce', sa.VARCHAR(length=256), nullable=True), sa.Column('transaction_index', sa.BigInteger(), nullable=True), - sa.Column('value', sa.Text(), nullable=True), + sa.Column('value', sa.Numeric(precision=78, scale=0), nullable=True), sa.Column('indexed_at', sa.DateTime(timezone=True), server_default=sa.text("TIMEZONE('utc', statement_timestamp())"), nullable=False), sa.ForeignKeyConstraint(['block_number'], ['ethereum_blocks.block_number'], name=op.f('fk_ethereum_transactions_block_number_ethereum_blocks'), ondelete='CASCADE'), - sa.PrimaryKeyConstraint('hash', name=op.f('pk_ethereum_transactions')), - sa.UniqueConstraint('hash', name=op.f('uq_ethereum_transactions_hash')) + sa.PrimaryKeyConstraint('hash', name=op.f('pk_ethereum_transactions')) ) + op.create_index(op.f('ix_ethereum_transactions_block_number'), 'ethereum_transactions', ['block_number'], unique=False) op.create_index(op.f('ix_ethereum_transactions_from_address'), 'ethereum_transactions', ['from_address'], unique=False) + op.create_index(op.f('ix_ethereum_transactions_gas'), 'ethereum_transactions', ['gas'], unique=False) + op.create_index(op.f('ix_ethereum_transactions_gas_price'), 'ethereum_transactions', ['gas_price'], unique=False) + op.create_index(op.f('ix_ethereum_transactions_hash'), 'ethereum_transactions', ['hash'], unique=True) op.create_index(op.f('ix_ethereum_transactions_to_address'), 'ethereum_transactions', ['to_address'], unique=False) + op.create_index(op.f('ix_ethereum_transactions_value'), 'ethereum_transactions', ['value'], unique=False) # ### end Alembic commands ### def downgrade(): # ### commands auto generated by Alembic - please adjust! ### + op.drop_index(op.f('ix_ethereum_transactions_value'), table_name='ethereum_transactions') op.drop_index(op.f('ix_ethereum_transactions_to_address'), table_name='ethereum_transactions') + op.drop_index(op.f('ix_ethereum_transactions_hash'), table_name='ethereum_transactions') + op.drop_index(op.f('ix_ethereum_transactions_gas_price'), table_name='ethereum_transactions') + op.drop_index(op.f('ix_ethereum_transactions_gas'), table_name='ethereum_transactions') op.drop_index(op.f('ix_ethereum_transactions_from_address'), table_name='ethereum_transactions') + op.drop_index(op.f('ix_ethereum_transactions_block_number'), table_name='ethereum_transactions') op.drop_table('ethereum_transactions') + op.drop_index(op.f('ix_ethereum_pending_transactions_value'), table_name='ethereum_pending_transactions') op.drop_index(op.f('ix_ethereum_pending_transactions_to_address'), table_name='ethereum_pending_transactions') + op.drop_index(op.f('ix_ethereum_pending_transactions_hash'), table_name='ethereum_pending_transactions') + op.drop_index(op.f('ix_ethereum_pending_transactions_gas_price'), table_name='ethereum_pending_transactions') + op.drop_index(op.f('ix_ethereum_pending_transactions_gas'), table_name='ethereum_pending_transactions') op.drop_index(op.f('ix_ethereum_pending_transactions_from_address'), table_name='ethereum_pending_transactions') + op.drop_index(op.f('ix_ethereum_pending_transactions_block_number'), table_name='ethereum_pending_transactions') op.drop_table('ethereum_pending_transactions') op.drop_index(op.f('ix_ethereum_blocks_timestamp'), table_name='ethereum_blocks') + op.drop_index(op.f('ix_ethereum_blocks_hash'), table_name='ethereum_blocks') + op.drop_index(op.f('ix_ethereum_blocks_block_number'), table_name='ethereum_blocks') op.drop_table('ethereum_blocks') + op.drop_index(op.f('ix_esd_function_signatures_id'), table_name='esd_function_signatures') + op.drop_table('esd_function_signatures') + op.drop_index(op.f('ix_esd_event_signatures_id'), table_name='esd_event_signatures') + op.drop_table('esd_event_signatures') # ### end Alembic commands ### diff --git a/db/moonstreamdb/models.py b/db/moonstreamdb/models.py index bd2c8873..968f366b 100644 --- a/db/moonstreamdb/models.py +++ b/db/moonstreamdb/models.py @@ -6,6 +6,7 @@ from sqlalchemy import ( Integer, ForeignKey, MetaData, + Numeric, Text, VARCHAR, ) @@ -49,16 +50,13 @@ class EthereumBlock(Base): # type: ignore __tablename__ = "ethereum_blocks" block_number = Column( - BigInteger, - primary_key=True, - unique=True, - nullable=False, + BigInteger, primary_key=True, unique=True, nullable=False, index=True ) difficulty = Column(BigInteger) extra_data = Column(VARCHAR(128)) gas_limit = Column(BigInteger) gas_used = Column(BigInteger) - hash = Column(VARCHAR(256)) + hash = Column(VARCHAR(256), index=True) logs_bloom = Column(VARCHAR(1024)) miner = Column(VARCHAR(256)) nonce = Column(VARCHAR(256)) @@ -79,24 +77,22 @@ class EthereumTransaction(Base): # type: ignore __tablename__ = "ethereum_transactions" hash = Column( - VARCHAR(256), - primary_key=True, - unique=True, - nullable=False, + VARCHAR(256), primary_key=True, unique=True, nullable=False, index=True ) block_number = Column( BigInteger, ForeignKey("ethereum_blocks.block_number", ondelete="CASCADE"), nullable=False, + index=True, ) from_address = Column(VARCHAR(256), index=True) to_address = Column(VARCHAR(256), index=True) - gas = Column(Text) - gas_price = Column(Text) + gas = Column(Numeric(precision=78, scale=0), index=True) + gas_price = Column(Numeric(precision=78, scale=0), index=True) input = Column(Text) nonce = Column(VARCHAR(256)) transaction_index = Column(BigInteger) - value = Column(Text) + value = Column(Numeric(precision=78, scale=0), index=True) indexed_at = Column( DateTime(timezone=True), server_default=utcnow(), nullable=False @@ -107,24 +103,22 @@ class EthereumPendingTransaction(Base): # type: ignore __tablename__ = "ethereum_pending_transactions" hash = Column( - VARCHAR(256), - primary_key=True, - unique=True, - nullable=False, + VARCHAR(256), primary_key=True, unique=True, nullable=False, index=True ) block_number = Column( BigInteger, ForeignKey("ethereum_blocks.block_number", ondelete="CASCADE"), nullable=False, + index=True, ) from_address = Column(VARCHAR(256), index=True) to_address = Column(VARCHAR(256), index=True) - gas = Column(Text) - gas_price = Column(Text) + gas = Column(Numeric(precision=78, scale=0), index=True) + gas_price = Column(Numeric(precision=78, scale=0), index=True) input = Column(Text) nonce = Column(VARCHAR(256)) transaction_index = Column(BigInteger) - value = Column(Text) + value = Column(Numeric(precision=78, scale=0), index=True) indexed_at = Column( DateTime(timezone=True), server_default=utcnow(), nullable=False @@ -138,7 +132,7 @@ class ESDFunctionSignature(Base): __tablename__ = "esd_function_signatures" - id = Column(Integer, primary_key=True) + id = Column(Integer, primary_key=True, unique=True, nullable=False, index=True) text_signature = Column(Text, nullable=False) hex_signature = Column(VARCHAR(10), nullable=False) created_at = Column( @@ -153,7 +147,7 @@ class ESDEventSignature(Base): __tablename__ = "esd_event_signatures" - id = Column(Integer, primary_key=True) + id = Column(Integer, primary_key=True, unique=True, nullable=False, index=True) text_signature = Column(Text, nullable=False) hex_signature = Column(VARCHAR(66), nullable=False) created_at = Column(