From 205b2234477f217bf4dca1913b4ca1d055c85f9a Mon Sep 17 00:00:00 2001 From: kompotkot Date: Tue, 26 Oct 2021 09:59:26 +0000 Subject: [PATCH 1/4] Migration with field for EIP 1559 --- .../0b46c8e17bf2_fields_for_eip_1559.py | 32 +++++++++++++++++++ db/moonstreamdb/models.py | 3 ++ 2 files changed, 35 insertions(+) create mode 100644 db/alembic/versions/0b46c8e17bf2_fields_for_eip_1559.py diff --git a/db/alembic/versions/0b46c8e17bf2_fields_for_eip_1559.py b/db/alembic/versions/0b46c8e17bf2_fields_for_eip_1559.py new file mode 100644 index 00000000..488170bc --- /dev/null +++ b/db/alembic/versions/0b46c8e17bf2_fields_for_eip_1559.py @@ -0,0 +1,32 @@ +"""fields-for-eip-1559 + +Revision ID: 0b46c8e17bf2 +Revises: 240476c67b9f +Create Date: 2021-10-26 09:52:17.367528 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = '0b46c8e17bf2' +down_revision = '240476c67b9f' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('ethereum_blocks', sa.Column('base_fee_per_gas', sa.Numeric(precision=78, scale=0), nullable=True)) + op.add_column('ethereum_transactions', sa.Column('max_fee_per_gas', sa.Numeric(precision=78, scale=0), nullable=True)) + op.add_column('ethereum_transactions', sa.Column('max_priority_fee_per_gas', sa.Numeric(precision=78, scale=0), nullable=True)) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('ethereum_transactions', 'max_priority_fee_per_gas') + op.drop_column('ethereum_transactions', 'max_fee_per_gas') + op.drop_column('ethereum_blocks', 'base_fee_per_gas') + # ### end Alembic commands ### diff --git a/db/moonstreamdb/models.py b/db/moonstreamdb/models.py index 40c0ef93..0dfa3326 100644 --- a/db/moonstreamdb/models.py +++ b/db/moonstreamdb/models.py @@ -59,6 +59,7 @@ class EthereumBlock(Base): # type: ignore extra_data = Column(VARCHAR(128)) gas_limit = Column(BigInteger) gas_used = Column(BigInteger) + base_fee_per_gas = Column(Numeric(precision=78, scale=0), nullable=True) hash = Column(VARCHAR(256), index=True) logs_bloom = Column(VARCHAR(1024)) miner = Column(VARCHAR(256)) @@ -92,6 +93,8 @@ class EthereumTransaction(Base): # type: ignore to_address = Column(VARCHAR(256), index=True) gas = Column(Numeric(precision=78, scale=0), index=True) gas_price = Column(Numeric(precision=78, scale=0), index=True) + max_fee_per_gas = Column(Numeric(precision=78, scale=0), nullable=True) + max_priority_fee_per_gas = Column(Numeric(precision=78, scale=0), nullable=True) input = Column(Text) nonce = Column(VARCHAR(256)) transaction_index = Column(BigInteger) From 7cf1ef411faafaee10bcf10d35c6ffc51fad9681 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Tue, 26 Oct 2021 12:02:55 +0000 Subject: [PATCH 2/4] Block and tx crawlers works with new fields --- crawlers/mooncrawl/mooncrawl/ethereum.py | 3 +++ crawlers/mooncrawl/requirements.txt | 2 +- crawlers/mooncrawl/setup.py | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/crawlers/mooncrawl/mooncrawl/ethereum.py b/crawlers/mooncrawl/mooncrawl/ethereum.py index e33350ad..5b4f6109 100644 --- a/crawlers/mooncrawl/mooncrawl/ethereum.py +++ b/crawlers/mooncrawl/mooncrawl/ethereum.py @@ -61,6 +61,7 @@ def add_block(db_session, block: Any) -> None: extra_data=block.extraData.hex(), gas_limit=block.gasLimit, gas_used=block.gasUsed, + base_fee_per_gas=block.get("baseFeePerGas", None), hash=block.hash.hex(), logs_bloom=block.logsBloom.hex(), miner=block.miner, @@ -91,6 +92,8 @@ def add_block_transactions(db_session, block: Any) -> None: to_address=tx.to, gas=tx.gas, gas_price=tx.gasPrice, + max_fee_per_gas=tx.get("maxFeePerGas", None), + max_priority_fee_per_gas=tx.get("maxPriorityFeePerGas", None), input=tx.input, nonce=tx.nonce, transaction_index=tx.transactionIndex, diff --git a/crawlers/mooncrawl/requirements.txt b/crawlers/mooncrawl/requirements.txt index 1d81a7a3..f0fcb74d 100644 --- a/crawlers/mooncrawl/requirements.txt +++ b/crawlers/mooncrawl/requirements.txt @@ -12,7 +12,7 @@ chardet==4.0.0 charset-normalizer==2.0.4 click==8.0.1 cytoolz==0.11.0 --e git+https://git@github.com/bugout-dev/moonstream.git@6b5b6049b58b1edf0e5de261614c616e8e034b6e#egg=moonstreamdb&subdirectory=db +-e git+https://git@github.com/bugout-dev/moonstream.git@205b2234477f217bf4dca1913b4ca1d055c85f9a#egg=moonstreamdb&subdirectory=db eth-abi==2.1.1 eth-account==0.5.5 eth-hash==0.3.2 diff --git a/crawlers/mooncrawl/setup.py b/crawlers/mooncrawl/setup.py index 5ab37727..50074231 100644 --- a/crawlers/mooncrawl/setup.py +++ b/crawlers/mooncrawl/setup.py @@ -34,7 +34,7 @@ setup( install_requires=[ "boto3", "bugout >= 0.1.17", - "moonstreamdb @ git+https://git@github.com/bugout-dev/moonstream.git@6b5b6049b58b1edf0e5de261614c616e8e034b6e#egg=moonstreamdb&subdirectory=db", + "moonstreamdb @ git+https://git@github.com/bugout-dev/moonstream.git@205b2234477f217bf4dca1913b4ca1d055c85f9a#egg=moonstreamdb&subdirectory=db", "humbug", "python-dateutil", "requests", From 0a771ddfbca1254be331149ccf2d162aa09b7bc0 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Thu, 28 Oct 2021 12:43:13 +0000 Subject: [PATCH 3/4] Transaction type column --- db/alembic/versions/0b46c8e17bf2_fields_for_eip_1559.py | 2 ++ db/moonstreamdb/models.py | 1 + 2 files changed, 3 insertions(+) diff --git a/db/alembic/versions/0b46c8e17bf2_fields_for_eip_1559.py b/db/alembic/versions/0b46c8e17bf2_fields_for_eip_1559.py index 488170bc..93863217 100644 --- a/db/alembic/versions/0b46c8e17bf2_fields_for_eip_1559.py +++ b/db/alembic/versions/0b46c8e17bf2_fields_for_eip_1559.py @@ -21,11 +21,13 @@ def upgrade(): op.add_column('ethereum_blocks', sa.Column('base_fee_per_gas', sa.Numeric(precision=78, scale=0), nullable=True)) op.add_column('ethereum_transactions', sa.Column('max_fee_per_gas', sa.Numeric(precision=78, scale=0), nullable=True)) op.add_column('ethereum_transactions', sa.Column('max_priority_fee_per_gas', sa.Numeric(precision=78, scale=0), nullable=True)) + op.add_column('ethereum_transactions', sa.Column('transaction_type', sa.Integer(), nullable=True)) # ### end Alembic commands ### def downgrade(): # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('ethereum_transactions', 'transaction_type') op.drop_column('ethereum_transactions', 'max_priority_fee_per_gas') op.drop_column('ethereum_transactions', 'max_fee_per_gas') op.drop_column('ethereum_blocks', 'base_fee_per_gas') diff --git a/db/moonstreamdb/models.py b/db/moonstreamdb/models.py index 0dfa3326..f88c8bea 100644 --- a/db/moonstreamdb/models.py +++ b/db/moonstreamdb/models.py @@ -98,6 +98,7 @@ class EthereumTransaction(Base): # type: ignore input = Column(Text) nonce = Column(VARCHAR(256)) transaction_index = Column(BigInteger) + transaction_type = Column(Integer, nullable=True) value = Column(Numeric(precision=78, scale=0), index=True) indexed_at = Column( From e8e0f480ac0d93e6b17ceff612110c9bf15e9c69 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Thu, 28 Oct 2021 13:14:48 +0000 Subject: [PATCH 4/4] Crawlers work with transaction type column --- crawlers/mooncrawl/mooncrawl/ethereum.py | 1 + crawlers/mooncrawl/requirements.txt | 2 +- crawlers/mooncrawl/setup.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/crawlers/mooncrawl/mooncrawl/ethereum.py b/crawlers/mooncrawl/mooncrawl/ethereum.py index 5b4f6109..c02a9eb5 100644 --- a/crawlers/mooncrawl/mooncrawl/ethereum.py +++ b/crawlers/mooncrawl/mooncrawl/ethereum.py @@ -97,6 +97,7 @@ def add_block_transactions(db_session, block: Any) -> None: input=tx.input, nonce=tx.nonce, transaction_index=tx.transactionIndex, + transaction_type=int(tx["type"], 0) if tx["type"] is not None else None, value=tx.value, ) db_session.add(tx_obj) diff --git a/crawlers/mooncrawl/requirements.txt b/crawlers/mooncrawl/requirements.txt index f0fcb74d..529b1e94 100644 --- a/crawlers/mooncrawl/requirements.txt +++ b/crawlers/mooncrawl/requirements.txt @@ -12,7 +12,7 @@ chardet==4.0.0 charset-normalizer==2.0.4 click==8.0.1 cytoolz==0.11.0 --e git+https://git@github.com/bugout-dev/moonstream.git@205b2234477f217bf4dca1913b4ca1d055c85f9a#egg=moonstreamdb&subdirectory=db +-e git+https://git@github.com/bugout-dev/moonstream.git@0a771ddfbca1254be331149ccf2d162aa09b7bc0#egg=moonstreamdb&subdirectory=db eth-abi==2.1.1 eth-account==0.5.5 eth-hash==0.3.2 diff --git a/crawlers/mooncrawl/setup.py b/crawlers/mooncrawl/setup.py index 50074231..bc387784 100644 --- a/crawlers/mooncrawl/setup.py +++ b/crawlers/mooncrawl/setup.py @@ -34,7 +34,7 @@ setup( install_requires=[ "boto3", "bugout >= 0.1.17", - "moonstreamdb @ git+https://git@github.com/bugout-dev/moonstream.git@205b2234477f217bf4dca1913b4ca1d055c85f9a#egg=moonstreamdb&subdirectory=db", + "moonstreamdb @ git+https://git@github.com/bugout-dev/moonstream.git@0a771ddfbca1254be331149ccf2d162aa09b7bc0#egg=moonstreamdb&subdirectory=db", "humbug", "python-dateutil", "requests",