mirror
/
moonstream
kopia lustrzana https://github.com/bugout-dev/moonstream
1
0
Forkuj 0
Kod Zgłoszenia Projekty Wydania Wiki Aktywność

Merge pull request #34 from bugout-dev/db-one-migration 

New updated migration
pull/27/head
Neeraj Kashyap 2021-07-29 11:13:02 -07:00 zatwierdzone przez GitHub
rodzic 5b7cdf6832 4f8f589df2
commit 7aff3f4159
Nie znaleziono w bazie danych klucza dla tego podpisu
ID klucza GPG: 4AEE18F83AFDEB23
6 zmienionych plików z 77 dodań i 84 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

2
backend/requirements.txt
Wyświetl plik

@ -11,7 +11,7 @@ fastapi==0.66.0
h11==0.12.0
idna==3.2
jmespath==0.10.0
-e git+https://git@github.com/bugout-dev/moonstream.git@b9c828fc7f811af88a9f3a45dd7f5c4053433366#egg=moonstreamdb&subdirectory=db
-e git+https://git@github.com/bugout-dev/moonstream.git@3326379b174a626cf876c7b60dd7db8afe2daba9#egg=moonstreamdb&subdirectory=db
mypy==0.910
mypy-extensions==0.4.3
pathspec==0.9.0

8
crawlers/moonstreamcrawlers/ethereum.py
Wyświetl plik

@ -66,13 +66,13 @@ def get_latest_blocks(with_transactions: bool = False) -> None:
"latest", full_transactions=with_transactions
)
with yield_db_session_ctx() as db_session:
block_latest_exist = (
db_session.query(EthereumBlock)
block_number_latest_exist = (
db_session.query(EthereumBlock.block_number)
.order_by(EthereumBlock.block_number.desc())
.first()
)
return block_latest_exist.block_number, block_latest.number
return block_number_latest_exist, block_latest.number
def crawl_blocks(
@ -107,7 +107,7 @@ def check_missing_blocks(blocks_numbers: List[int]) -> List[int]:
for block_number in blocks_numbers:
with yield_db_session_ctx() as db_session:
block_exist = (
db_session.query(EthereumBlock)
db_session.query(EthereumBlock.block_number)
.filter(EthereumBlock.block_number == block_number)
.one_or_none()
)

BIN
crawlers/requirements.txt
Wyświetl plik

Plik binarny nie jest wyświetlany.

42
db/alembic/versions/1e33c3d07306_added_tables_to_store_data_from_.py
Wyświetl plik

@ -1,42 +0,0 @@
"""Added tables to store data from Ethereum Signature Database
Revision ID: 1e33c3d07306
Revises: aa903a90b8bf
Create Date: 2021-07-27 00:04:31.042487
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '1e33c3d07306'
down_revision = 'aa903a90b8bf'
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.create_table('esd_event_signatures',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('text_signature', sa.Text(), nullable=False),
sa.Column('hex_signature', sa.VARCHAR(length=66), nullable=False),
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text("TIMEZONE('utc', statement_timestamp())"), nullable=False),
sa.PrimaryKeyConstraint('id', name=op.f('pk_esd_event_signatures'))
)
op.create_table('esd_function_signatures',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('text_signature', sa.Text(), nullable=False),
sa.Column('hex_signature', sa.VARCHAR(length=10), nullable=False),
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text("TIMEZONE('utc', statement_timestamp())"), nullable=False),
sa.PrimaryKeyConstraint('id', name=op.f('pk_esd_function_signatures'))
)
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.drop_table('esd_function_signatures')
op.drop_table('esd_event_signatures')
# ### end Alembic commands ###

73
db/alembic/versions/aa903a90b8bf_inital_for_blocks_and_transactions.py → db/alembic/versions/21cced47077c_inital_migration.py
Wyświetl plik

@ -1,8 +1,8 @@
"""Inital for blocks and transactions
"""Inital migration
Revision ID: aa903a90b8bf
Revision ID: 21cced47077c
Revises:
Create Date: 2021-07-26 13:55:38.057312
Create Date: 2021-07-29 20:30:25.863724
"""
from alembic import op
@ -10,7 +10,7 @@ import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = 'aa903a90b8bf'
revision = '21cced47077c'
down_revision = None
branch_labels = None
depends_on = None
@ -18,6 +18,22 @@ depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.create_table('esd_event_signatures',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('text_signature', sa.Text(), nullable=False),
sa.Column('hex_signature', sa.VARCHAR(length=66), nullable=False),
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text("TIMEZONE('utc', statement_timestamp())"), nullable=False),
sa.PrimaryKeyConstraint('id', name=op.f('pk_esd_event_signatures'))
)
op.create_index(op.f('ix_esd_event_signatures_id'), 'esd_event_signatures', ['id'], unique=True)
op.create_table('esd_function_signatures',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('text_signature', sa.Text(), nullable=False),
sa.Column('hex_signature', sa.VARCHAR(length=10), nullable=False),
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text("TIMEZONE('utc', statement_timestamp())"), nullable=False),
sa.PrimaryKeyConstraint('id', name=op.f('pk_esd_function_signatures'))
)
op.create_index(op.f('ix_esd_function_signatures_id'), 'esd_function_signatures', ['id'], unique=True)
op.create_table('ethereum_blocks',
sa.Column('block_number', sa.BigInteger(), nullable=False),
sa.Column('difficulty', sa.BigInteger(), nullable=True),
@ -37,57 +53,82 @@ def upgrade():
sa.Column('total_difficulty', sa.VARCHAR(length=256), nullable=True),
sa.Column('transactions_root', sa.VARCHAR(length=256), nullable=True),
sa.Column('indexed_at', sa.DateTime(timezone=True), server_default=sa.text("TIMEZONE('utc', statement_timestamp())"), nullable=False),
sa.PrimaryKeyConstraint('block_number', name=op.f('pk_ethereum_blocks')),
sa.UniqueConstraint('block_number', name=op.f('uq_ethereum_blocks_block_number'))
sa.PrimaryKeyConstraint('block_number', name=op.f('pk_ethereum_blocks'))
)
op.create_index(op.f('ix_ethereum_blocks_block_number'), 'ethereum_blocks', ['block_number'], unique=True)
op.create_index(op.f('ix_ethereum_blocks_hash'), 'ethereum_blocks', ['hash'], unique=False)
op.create_index(op.f('ix_ethereum_blocks_timestamp'), 'ethereum_blocks', ['timestamp'], unique=False)
op.create_table('ethereum_pending_transactions',
sa.Column('hash', sa.VARCHAR(length=256), nullable=False),
sa.Column('block_number', sa.BigInteger(), nullable=False),
sa.Column('from_address', sa.VARCHAR(length=256), nullable=True),
sa.Column('to_address', sa.VARCHAR(length=256), nullable=True),
sa.Column('gas', sa.Text(), nullable=True),
sa.Column('gas_price', sa.Text(), nullable=True),
sa.Column('gas', sa.Numeric(precision=78, scale=0), nullable=True),
sa.Column('gas_price', sa.Numeric(precision=78, scale=0), nullable=True),
sa.Column('input', sa.Text(), nullable=True),
sa.Column('nonce', sa.VARCHAR(length=256), nullable=True),
sa.Column('transaction_index', sa.BigInteger(), nullable=True),
sa.Column('value', sa.Text(), nullable=True),
sa.Column('value', sa.Numeric(precision=78, scale=0), nullable=True),
sa.Column('indexed_at', sa.DateTime(timezone=True), server_default=sa.text("TIMEZONE('utc', statement_timestamp())"), nullable=False),
sa.ForeignKeyConstraint(['block_number'], ['ethereum_blocks.block_number'], name=op.f('fk_ethereum_pending_transactions_block_number_ethereum_blocks'), ondelete='CASCADE'),
sa.PrimaryKeyConstraint('hash', name=op.f('pk_ethereum_pending_transactions')),
sa.UniqueConstraint('hash', name=op.f('uq_ethereum_pending_transactions_hash'))
sa.PrimaryKeyConstraint('hash', name=op.f('pk_ethereum_pending_transactions'))
)
op.create_index(op.f('ix_ethereum_pending_transactions_block_number'), 'ethereum_pending_transactions', ['block_number'], unique=False)
op.create_index(op.f('ix_ethereum_pending_transactions_from_address'), 'ethereum_pending_transactions', ['from_address'], unique=False)
op.create_index(op.f('ix_ethereum_pending_transactions_gas'), 'ethereum_pending_transactions', ['gas'], unique=False)
op.create_index(op.f('ix_ethereum_pending_transactions_gas_price'), 'ethereum_pending_transactions', ['gas_price'], unique=False)
op.create_index(op.f('ix_ethereum_pending_transactions_hash'), 'ethereum_pending_transactions', ['hash'], unique=True)
op.create_index(op.f('ix_ethereum_pending_transactions_to_address'), 'ethereum_pending_transactions', ['to_address'], unique=False)
op.create_index(op.f('ix_ethereum_pending_transactions_value'), 'ethereum_pending_transactions', ['value'], unique=False)
op.create_table('ethereum_transactions',
sa.Column('hash', sa.VARCHAR(length=256), nullable=False),
sa.Column('block_number', sa.BigInteger(), nullable=False),
sa.Column('from_address', sa.VARCHAR(length=256), nullable=True),
sa.Column('to_address', sa.VARCHAR(length=256), nullable=True),
sa.Column('gas', sa.Text(), nullable=True),
sa.Column('gas_price', sa.Text(), nullable=True),
sa.Column('gas', sa.Numeric(precision=78, scale=0), nullable=True),
sa.Column('gas_price', sa.Numeric(precision=78, scale=0), nullable=True),
sa.Column('input', sa.Text(), nullable=True),
sa.Column('nonce', sa.VARCHAR(length=256), nullable=True),
sa.Column('transaction_index', sa.BigInteger(), nullable=True),
sa.Column('value', sa.Text(), nullable=True),
sa.Column('value', sa.Numeric(precision=78, scale=0), nullable=True),
sa.Column('indexed_at', sa.DateTime(timezone=True), server_default=sa.text("TIMEZONE('utc', statement_timestamp())"), nullable=False),
sa.ForeignKeyConstraint(['block_number'], ['ethereum_blocks.block_number'], name=op.f('fk_ethereum_transactions_block_number_ethereum_blocks'), ondelete='CASCADE'),
sa.PrimaryKeyConstraint('hash', name=op.f('pk_ethereum_transactions')),
sa.UniqueConstraint('hash', name=op.f('uq_ethereum_transactions_hash'))
sa.PrimaryKeyConstraint('hash', name=op.f('pk_ethereum_transactions'))
)
op.create_index(op.f('ix_ethereum_transactions_block_number'), 'ethereum_transactions', ['block_number'], unique=False)
op.create_index(op.f('ix_ethereum_transactions_from_address'), 'ethereum_transactions', ['from_address'], unique=False)
op.create_index(op.f('ix_ethereum_transactions_gas'), 'ethereum_transactions', ['gas'], unique=False)
op.create_index(op.f('ix_ethereum_transactions_gas_price'), 'ethereum_transactions', ['gas_price'], unique=False)
op.create_index(op.f('ix_ethereum_transactions_hash'), 'ethereum_transactions', ['hash'], unique=True)
op.create_index(op.f('ix_ethereum_transactions_to_address'), 'ethereum_transactions', ['to_address'], unique=False)
op.create_index(op.f('ix_ethereum_transactions_value'), 'ethereum_transactions', ['value'], unique=False)
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.drop_index(op.f('ix_ethereum_transactions_value'), table_name='ethereum_transactions')
op.drop_index(op.f('ix_ethereum_transactions_to_address'), table_name='ethereum_transactions')
op.drop_index(op.f('ix_ethereum_transactions_hash'), table_name='ethereum_transactions')
op.drop_index(op.f('ix_ethereum_transactions_gas_price'), table_name='ethereum_transactions')
op.drop_index(op.f('ix_ethereum_transactions_gas'), table_name='ethereum_transactions')
op.drop_index(op.f('ix_ethereum_transactions_from_address'), table_name='ethereum_transactions')
op.drop_index(op.f('ix_ethereum_transactions_block_number'), table_name='ethereum_transactions')
op.drop_table('ethereum_transactions')
op.drop_index(op.f('ix_ethereum_pending_transactions_value'), table_name='ethereum_pending_transactions')
op.drop_index(op.f('ix_ethereum_pending_transactions_to_address'), table_name='ethereum_pending_transactions')
op.drop_index(op.f('ix_ethereum_pending_transactions_hash'), table_name='ethereum_pending_transactions')
op.drop_index(op.f('ix_ethereum_pending_transactions_gas_price'), table_name='ethereum_pending_transactions')
op.drop_index(op.f('ix_ethereum_pending_transactions_gas'), table_name='ethereum_pending_transactions')
op.drop_index(op.f('ix_ethereum_pending_transactions_from_address'), table_name='ethereum_pending_transactions')
op.drop_index(op.f('ix_ethereum_pending_transactions_block_number'), table_name='ethereum_pending_transactions')
op.drop_table('ethereum_pending_transactions')
op.drop_index(op.f('ix_ethereum_blocks_timestamp'), table_name='ethereum_blocks')
op.drop_index(op.f('ix_ethereum_blocks_hash'), table_name='ethereum_blocks')
op.drop_index(op.f('ix_ethereum_blocks_block_number'), table_name='ethereum_blocks')
op.drop_table('ethereum_blocks')
op.drop_index(op.f('ix_esd_function_signatures_id'), table_name='esd_function_signatures')
op.drop_table('esd_function_signatures')
op.drop_index(op.f('ix_esd_event_signatures_id'), table_name='esd_event_signatures')
op.drop_table('esd_event_signatures')
# ### end Alembic commands ###

36
db/moonstreamdb/models.py
Wyświetl plik

@ -6,6 +6,7 @@ from sqlalchemy import (
Integer,
ForeignKey,
MetaData,
Numeric,
Text,
VARCHAR,
)
@ -49,16 +50,13 @@ class EthereumBlock(Base): # type: ignore
__tablename__ = "ethereum_blocks"
block_number = Column(
BigInteger,
primary_key=True,
unique=True,
nullable=False,
BigInteger, primary_key=True, unique=True, nullable=False, index=True
)
difficulty = Column(BigInteger)
extra_data = Column(VARCHAR(128))
gas_limit = Column(BigInteger)
gas_used = Column(BigInteger)
hash = Column(VARCHAR(256))
hash = Column(VARCHAR(256), index=True)
logs_bloom = Column(VARCHAR(1024))
miner = Column(VARCHAR(256))
nonce = Column(VARCHAR(256))
@ -79,24 +77,22 @@ class EthereumTransaction(Base): # type: ignore
__tablename__ = "ethereum_transactions"
hash = Column(
VARCHAR(256),
primary_key=True,
unique=True,
nullable=False,
VARCHAR(256), primary_key=True, unique=True, nullable=False, index=True
)
block_number = Column(
BigInteger,
ForeignKey("ethereum_blocks.block_number", ondelete="CASCADE"),
nullable=False,
index=True,
)
from_address = Column(VARCHAR(256), index=True)
to_address = Column(VARCHAR(256), index=True)
gas = Column(Text)
gas_price = Column(Text)
gas = Column(Numeric(precision=78, scale=0), index=True)
gas_price = Column(Numeric(precision=78, scale=0), index=True)
input = Column(Text)
nonce = Column(VARCHAR(256))
transaction_index = Column(BigInteger)
value = Column(Text)
value = Column(Numeric(precision=78, scale=0), index=True)
indexed_at = Column(
DateTime(timezone=True), server_default=utcnow(), nullable=False
@ -107,24 +103,22 @@ class EthereumPendingTransaction(Base): # type: ignore
__tablename__ = "ethereum_pending_transactions"
hash = Column(
VARCHAR(256),
primary_key=True,
unique=True,
nullable=False,
VARCHAR(256), primary_key=True, unique=True, nullable=False, index=True
)
block_number = Column(
BigInteger,
ForeignKey("ethereum_blocks.block_number", ondelete="CASCADE"),
nullable=False,
index=True,
)
from_address = Column(VARCHAR(256), index=True)
to_address = Column(VARCHAR(256), index=True)
gas = Column(Text)
gas_price = Column(Text)
gas = Column(Numeric(precision=78, scale=0), index=True)
gas_price = Column(Numeric(precision=78, scale=0), index=True)
input = Column(Text)
nonce = Column(VARCHAR(256))
transaction_index = Column(BigInteger)
value = Column(Text)
value = Column(Numeric(precision=78, scale=0), index=True)
indexed_at = Column(
DateTime(timezone=True), server_default=utcnow(), nullable=False
@ -138,7 +132,7 @@ class ESDFunctionSignature(Base):
__tablename__ = "esd_function_signatures"
id = Column(Integer, primary_key=True)
id = Column(Integer, primary_key=True, unique=True, nullable=False, index=True)
text_signature = Column(Text, nullable=False)
hex_signature = Column(VARCHAR(10), nullable=False)
created_at = Column(
@ -153,7 +147,7 @@ class ESDEventSignature(Base):
__tablename__ = "esd_event_signatures"
id = Column(Integer, primary_key=True)
id = Column(Integer, primary_key=True, unique=True, nullable=False, index=True)
text_signature = Column(Text, nullable=False)
hex_signature = Column(VARCHAR(66), nullable=False)
created_at = Column(