Merge pull request #34 from bugout-dev/db-one-migration

New updated migration

backend/requirements.txt

@@ -11,7 +11,7 @@ fastapi==0.66.0
 h11==0.12.0
 idna==3.2
 jmespath==0.10.0
--e git+https://git@github.com/bugout-dev/moonstream.git@b9c828fc7f811af88a9f3a45dd7f5c4053433366#egg=moonstreamdb&subdirectory=db
+-e git+https://git@github.com/bugout-dev/moonstream.git@3326379b174a626cf876c7b60dd7db8afe2daba9#egg=moonstreamdb&subdirectory=db
 mypy==0.910
 mypy-extensions==0.4.3
 pathspec==0.9.0

crawlers/moonstreamcrawlers/ethereum.py

@@ -66,13 +66,13 @@ def get_latest_blocks(with_transactions: bool = False) -> None:
         "latest", full_transactions=with_transactions
     )
     with yield_db_session_ctx() as db_session:
-        block_latest_exist = (
+        block_number_latest_exist = (
-            db_session.query(EthereumBlock)
+            db_session.query(EthereumBlock.block_number)
             .order_by(EthereumBlock.block_number.desc())
             .first()
         )
 
-    return block_latest_exist.block_number, block_latest.number
+    return block_number_latest_exist, block_latest.number
 
 
 def crawl_blocks(
@@ -107,7 +107,7 @@ def check_missing_blocks(blocks_numbers: List[int]) -> List[int]:
     for block_number in blocks_numbers:
         with yield_db_session_ctx() as db_session:
             block_exist = (
-                db_session.query(EthereumBlock)
+                db_session.query(EthereumBlock.block_number)
                 .filter(EthereumBlock.block_number == block_number)
                 .one_or_none()
             )

db/alembic/versions/1e33c3d07306_added_tables_to_store_data_from_.py

@@ -1,42 +0,0 @@
-"""Added tables to store data from Ethereum Signature Database
-
-Revision ID: 1e33c3d07306
-Revises: aa903a90b8bf
-Create Date: 2021-07-27 00:04:31.042487
-
-"""
-from alembic import op
-import sqlalchemy as sa
-
-
-# revision identifiers, used by Alembic.
-revision = '1e33c3d07306'
-down_revision = 'aa903a90b8bf'
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
-    op.create_table('esd_event_signatures',
-    sa.Column('id', sa.Integer(), nullable=False),
-    sa.Column('text_signature', sa.Text(), nullable=False),
-    sa.Column('hex_signature', sa.VARCHAR(length=66), nullable=False),
-    sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text("TIMEZONE('utc', statement_timestamp())"), nullable=False),
-    sa.PrimaryKeyConstraint('id', name=op.f('pk_esd_event_signatures'))
-    )
-    op.create_table('esd_function_signatures',
-    sa.Column('id', sa.Integer(), nullable=False),
-    sa.Column('text_signature', sa.Text(), nullable=False),
-    sa.Column('hex_signature', sa.VARCHAR(length=10), nullable=False),
-    sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text("TIMEZONE('utc', statement_timestamp())"), nullable=False),
-    sa.PrimaryKeyConstraint('id', name=op.f('pk_esd_function_signatures'))
-    )
-    # ### end Alembic commands ###
-
-
-def downgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
-    op.drop_table('esd_function_signatures')
-    op.drop_table('esd_event_signatures')
-    # ### end Alembic commands ###

db/alembic/versions/21cced47077c_inital_migration.py

@@ -1,8 +1,8 @@
-"""Inital for blocks and transactions
+"""Inital migration
 
-Revision ID: aa903a90b8bf
+Revision ID: 21cced47077c
 Revises: 
-Create Date: 2021-07-26 13:55:38.057312
+Create Date: 2021-07-29 20:30:25.863724
 
 """
 from alembic import op
@@ -10,7 +10,7 @@ import sqlalchemy as sa
 
 
 # revision identifiers, used by Alembic.
-revision = 'aa903a90b8bf'
+revision = '21cced47077c'
 down_revision = None
 branch_labels = None
 depends_on = None
@@ -18,6 +18,22 @@ depends_on = None
 
 def upgrade():
     # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('esd_event_signatures',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('text_signature', sa.Text(), nullable=False),
+    sa.Column('hex_signature', sa.VARCHAR(length=66), nullable=False),
+    sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text("TIMEZONE('utc', statement_timestamp())"), nullable=False),
+    sa.PrimaryKeyConstraint('id', name=op.f('pk_esd_event_signatures'))
+    )
+    op.create_index(op.f('ix_esd_event_signatures_id'), 'esd_event_signatures', ['id'], unique=True)
+    op.create_table('esd_function_signatures',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('text_signature', sa.Text(), nullable=False),
+    sa.Column('hex_signature', sa.VARCHAR(length=10), nullable=False),
+    sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text("TIMEZONE('utc', statement_timestamp())"), nullable=False),
+    sa.PrimaryKeyConstraint('id', name=op.f('pk_esd_function_signatures'))
+    )
+    op.create_index(op.f('ix_esd_function_signatures_id'), 'esd_function_signatures', ['id'], unique=True)
     op.create_table('ethereum_blocks',
     sa.Column('block_number', sa.BigInteger(), nullable=False),
     sa.Column('difficulty', sa.BigInteger(), nullable=True),
@@ -37,57 +53,82 @@ def upgrade():
     sa.Column('total_difficulty', sa.VARCHAR(length=256), nullable=True),
     sa.Column('transactions_root', sa.VARCHAR(length=256), nullable=True),
     sa.Column('indexed_at', sa.DateTime(timezone=True), server_default=sa.text("TIMEZONE('utc', statement_timestamp())"), nullable=False),
-    sa.PrimaryKeyConstraint('block_number', name=op.f('pk_ethereum_blocks')),
+    sa.PrimaryKeyConstraint('block_number', name=op.f('pk_ethereum_blocks'))
-    sa.UniqueConstraint('block_number', name=op.f('uq_ethereum_blocks_block_number'))
     )
+    op.create_index(op.f('ix_ethereum_blocks_block_number'), 'ethereum_blocks', ['block_number'], unique=True)
+    op.create_index(op.f('ix_ethereum_blocks_hash'), 'ethereum_blocks', ['hash'], unique=False)
     op.create_index(op.f('ix_ethereum_blocks_timestamp'), 'ethereum_blocks', ['timestamp'], unique=False)
     op.create_table('ethereum_pending_transactions',
     sa.Column('hash', sa.VARCHAR(length=256), nullable=False),
     sa.Column('block_number', sa.BigInteger(), nullable=False),
     sa.Column('from_address', sa.VARCHAR(length=256), nullable=True),
     sa.Column('to_address', sa.VARCHAR(length=256), nullable=True),
-    sa.Column('gas', sa.Text(), nullable=True),
+    sa.Column('gas', sa.Numeric(precision=78, scale=0), nullable=True),
-    sa.Column('gas_price', sa.Text(), nullable=True),
+    sa.Column('gas_price', sa.Numeric(precision=78, scale=0), nullable=True),
     sa.Column('input', sa.Text(), nullable=True),
     sa.Column('nonce', sa.VARCHAR(length=256), nullable=True),
     sa.Column('transaction_index', sa.BigInteger(), nullable=True),
-    sa.Column('value', sa.Text(), nullable=True),
+    sa.Column('value', sa.Numeric(precision=78, scale=0), nullable=True),
     sa.Column('indexed_at', sa.DateTime(timezone=True), server_default=sa.text("TIMEZONE('utc', statement_timestamp())"), nullable=False),
     sa.ForeignKeyConstraint(['block_number'], ['ethereum_blocks.block_number'], name=op.f('fk_ethereum_pending_transactions_block_number_ethereum_blocks'), ondelete='CASCADE'),
-    sa.PrimaryKeyConstraint('hash', name=op.f('pk_ethereum_pending_transactions')),
+    sa.PrimaryKeyConstraint('hash', name=op.f('pk_ethereum_pending_transactions'))
-    sa.UniqueConstraint('hash', name=op.f('uq_ethereum_pending_transactions_hash'))
     )
+    op.create_index(op.f('ix_ethereum_pending_transactions_block_number'), 'ethereum_pending_transactions', ['block_number'], unique=False)
     op.create_index(op.f('ix_ethereum_pending_transactions_from_address'), 'ethereum_pending_transactions', ['from_address'], unique=False)
+    op.create_index(op.f('ix_ethereum_pending_transactions_gas'), 'ethereum_pending_transactions', ['gas'], unique=False)
+    op.create_index(op.f('ix_ethereum_pending_transactions_gas_price'), 'ethereum_pending_transactions', ['gas_price'], unique=False)
+    op.create_index(op.f('ix_ethereum_pending_transactions_hash'), 'ethereum_pending_transactions', ['hash'], unique=True)
     op.create_index(op.f('ix_ethereum_pending_transactions_to_address'), 'ethereum_pending_transactions', ['to_address'], unique=False)
+    op.create_index(op.f('ix_ethereum_pending_transactions_value'), 'ethereum_pending_transactions', ['value'], unique=False)
     op.create_table('ethereum_transactions',
     sa.Column('hash', sa.VARCHAR(length=256), nullable=False),
     sa.Column('block_number', sa.BigInteger(), nullable=False),
     sa.Column('from_address', sa.VARCHAR(length=256), nullable=True),
     sa.Column('to_address', sa.VARCHAR(length=256), nullable=True),
-    sa.Column('gas', sa.Text(), nullable=True),
+    sa.Column('gas', sa.Numeric(precision=78, scale=0), nullable=True),
-    sa.Column('gas_price', sa.Text(), nullable=True),
+    sa.Column('gas_price', sa.Numeric(precision=78, scale=0), nullable=True),
     sa.Column('input', sa.Text(), nullable=True),
     sa.Column('nonce', sa.VARCHAR(length=256), nullable=True),
     sa.Column('transaction_index', sa.BigInteger(), nullable=True),
-    sa.Column('value', sa.Text(), nullable=True),
+    sa.Column('value', sa.Numeric(precision=78, scale=0), nullable=True),
     sa.Column('indexed_at', sa.DateTime(timezone=True), server_default=sa.text("TIMEZONE('utc', statement_timestamp())"), nullable=False),
     sa.ForeignKeyConstraint(['block_number'], ['ethereum_blocks.block_number'], name=op.f('fk_ethereum_transactions_block_number_ethereum_blocks'), ondelete='CASCADE'),
-    sa.PrimaryKeyConstraint('hash', name=op.f('pk_ethereum_transactions')),
+    sa.PrimaryKeyConstraint('hash', name=op.f('pk_ethereum_transactions'))
-    sa.UniqueConstraint('hash', name=op.f('uq_ethereum_transactions_hash'))
     )
+    op.create_index(op.f('ix_ethereum_transactions_block_number'), 'ethereum_transactions', ['block_number'], unique=False)
     op.create_index(op.f('ix_ethereum_transactions_from_address'), 'ethereum_transactions', ['from_address'], unique=False)
+    op.create_index(op.f('ix_ethereum_transactions_gas'), 'ethereum_transactions', ['gas'], unique=False)
+    op.create_index(op.f('ix_ethereum_transactions_gas_price'), 'ethereum_transactions', ['gas_price'], unique=False)
+    op.create_index(op.f('ix_ethereum_transactions_hash'), 'ethereum_transactions', ['hash'], unique=True)
     op.create_index(op.f('ix_ethereum_transactions_to_address'), 'ethereum_transactions', ['to_address'], unique=False)
+    op.create_index(op.f('ix_ethereum_transactions_value'), 'ethereum_transactions', ['value'], unique=False)
     # ### end Alembic commands ###
 
 
 def downgrade():
     # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(op.f('ix_ethereum_transactions_value'), table_name='ethereum_transactions')
     op.drop_index(op.f('ix_ethereum_transactions_to_address'), table_name='ethereum_transactions')
+    op.drop_index(op.f('ix_ethereum_transactions_hash'), table_name='ethereum_transactions')
+    op.drop_index(op.f('ix_ethereum_transactions_gas_price'), table_name='ethereum_transactions')
+    op.drop_index(op.f('ix_ethereum_transactions_gas'), table_name='ethereum_transactions')
     op.drop_index(op.f('ix_ethereum_transactions_from_address'), table_name='ethereum_transactions')
+    op.drop_index(op.f('ix_ethereum_transactions_block_number'), table_name='ethereum_transactions')
     op.drop_table('ethereum_transactions')
+    op.drop_index(op.f('ix_ethereum_pending_transactions_value'), table_name='ethereum_pending_transactions')
     op.drop_index(op.f('ix_ethereum_pending_transactions_to_address'), table_name='ethereum_pending_transactions')
+    op.drop_index(op.f('ix_ethereum_pending_transactions_hash'), table_name='ethereum_pending_transactions')
+    op.drop_index(op.f('ix_ethereum_pending_transactions_gas_price'), table_name='ethereum_pending_transactions')
+    op.drop_index(op.f('ix_ethereum_pending_transactions_gas'), table_name='ethereum_pending_transactions')
     op.drop_index(op.f('ix_ethereum_pending_transactions_from_address'), table_name='ethereum_pending_transactions')
+    op.drop_index(op.f('ix_ethereum_pending_transactions_block_number'), table_name='ethereum_pending_transactions')
     op.drop_table('ethereum_pending_transactions')
     op.drop_index(op.f('ix_ethereum_blocks_timestamp'), table_name='ethereum_blocks')
+    op.drop_index(op.f('ix_ethereum_blocks_hash'), table_name='ethereum_blocks')
+    op.drop_index(op.f('ix_ethereum_blocks_block_number'), table_name='ethereum_blocks')
     op.drop_table('ethereum_blocks')
+    op.drop_index(op.f('ix_esd_function_signatures_id'), table_name='esd_function_signatures')
+    op.drop_table('esd_function_signatures')
+    op.drop_index(op.f('ix_esd_event_signatures_id'), table_name='esd_event_signatures')
+    op.drop_table('esd_event_signatures')
     # ### end Alembic commands ###

db/moonstreamdb/models.py

@@ -6,6 +6,7 @@ from sqlalchemy import (
     Integer,
     ForeignKey,
     MetaData,
+    Numeric,
     Text,
     VARCHAR,
 )
@@ -49,16 +50,13 @@ class EthereumBlock(Base):  # type: ignore
     __tablename__ = "ethereum_blocks"
 
     block_number = Column(
-        BigInteger,
+        BigInteger, primary_key=True, unique=True, nullable=False, index=True
-        primary_key=True,
-        unique=True,
-        nullable=False,
     )
     difficulty = Column(BigInteger)
     extra_data = Column(VARCHAR(128))
     gas_limit = Column(BigInteger)
     gas_used = Column(BigInteger)
-    hash = Column(VARCHAR(256))
+    hash = Column(VARCHAR(256), index=True)
     logs_bloom = Column(VARCHAR(1024))
     miner = Column(VARCHAR(256))
     nonce = Column(VARCHAR(256))
@@ -79,24 +77,22 @@ class EthereumTransaction(Base):  # type: ignore
     __tablename__ = "ethereum_transactions"
 
     hash = Column(
-        VARCHAR(256),
+        VARCHAR(256), primary_key=True, unique=True, nullable=False, index=True
-        primary_key=True,
-        unique=True,
-        nullable=False,
     )
     block_number = Column(
         BigInteger,
         ForeignKey("ethereum_blocks.block_number", ondelete="CASCADE"),
         nullable=False,
+        index=True,
     )
     from_address = Column(VARCHAR(256), index=True)
     to_address = Column(VARCHAR(256), index=True)
-    gas = Column(Text)
+    gas = Column(Numeric(precision=78, scale=0), index=True)
-    gas_price = Column(Text)
+    gas_price = Column(Numeric(precision=78, scale=0), index=True)
     input = Column(Text)
     nonce = Column(VARCHAR(256))
     transaction_index = Column(BigInteger)
-    value = Column(Text)
+    value = Column(Numeric(precision=78, scale=0), index=True)
 
     indexed_at = Column(
         DateTime(timezone=True), server_default=utcnow(), nullable=False
@@ -107,24 +103,22 @@ class EthereumPendingTransaction(Base):  # type: ignore
     __tablename__ = "ethereum_pending_transactions"
 
     hash = Column(
-        VARCHAR(256),
+        VARCHAR(256), primary_key=True, unique=True, nullable=False, index=True
-        primary_key=True,
-        unique=True,
-        nullable=False,
     )
     block_number = Column(
         BigInteger,
         ForeignKey("ethereum_blocks.block_number", ondelete="CASCADE"),
         nullable=False,
+        index=True,
     )
     from_address = Column(VARCHAR(256), index=True)
     to_address = Column(VARCHAR(256), index=True)
-    gas = Column(Text)
+    gas = Column(Numeric(precision=78, scale=0), index=True)
-    gas_price = Column(Text)
+    gas_price = Column(Numeric(precision=78, scale=0), index=True)
     input = Column(Text)
     nonce = Column(VARCHAR(256))
     transaction_index = Column(BigInteger)
-    value = Column(Text)
+    value = Column(Numeric(precision=78, scale=0), index=True)
 
     indexed_at = Column(
         DateTime(timezone=True), server_default=utcnow(), nullable=False
@@ -138,7 +132,7 @@ class ESDFunctionSignature(Base):
 
     __tablename__ = "esd_function_signatures"
 
-    id = Column(Integer, primary_key=True)
+    id = Column(Integer, primary_key=True, unique=True, nullable=False, index=True)
     text_signature = Column(Text, nullable=False)
     hex_signature = Column(VARCHAR(10), nullable=False)
     created_at = Column(
@@ -153,7 +147,7 @@ class ESDEventSignature(Base):
 
     __tablename__ = "esd_event_signatures"
 
-    id = Column(Integer, primary_key=True)
+    id = Column(Integer, primary_key=True, unique=True, nullable=False, index=True)
     text_signature = Column(Text, nullable=False)
     hex_signature = Column(VARCHAR(66), nullable=False)
     created_at = Column(