mirror
/
moonstream
kopia lustrzana https://github.com/bugout-dev/moonstream
1
0
Forkuj 0
Kod Zgłoszenia Projekty Wydania Wiki Aktywność

Add tested version.

pull/761/head
Andrey 2023-03-02 14:26:36 +02:00
rodzic 059d759890
commit 1e60876519
10 zmienionych plików z 284 dodań i 12 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

13
backend/moonstreamapi/actions.py
Wyświetl plik

@ -48,10 +48,12 @@ blockchain_by_subscription_id = {
"polygon_blockchain": "polygon",
"mumbai_blockchain": "mumbai",
"xdai_blockchain": "xdai",
"caldera_blockchain": "caldera",
"ethereum_smartcontract": "ethereum",
"polygon_smartcontract": "polygon",
"mumbai_smartcontract": "mumbai",
"xdai_smartcontract": "xdai",
"caldera_smartcontract": "caldera",
}
@ -129,7 +131,6 @@ def get_ens_name(web3: Web3, address: str) -> Optional[str]:
def get_ens_address(web3: Web3, name: str) -> Optional[str]:
if not is_valid_ens_name(name):
raise ValueError(f"{name} is not valid ens name")
@ -148,7 +149,6 @@ def get_ens_address(web3: Web3, name: str) -> Optional[str]:
def get_ethereum_address_info(
db_session: Session, web3: Web3, address: str
) -> Optional[data.EthereumAddressInfo]:
if not is_address(address):
raise ValueError(f"Invalid ethereum address : {address}")
@ -265,7 +265,6 @@ def create_onboarding_resource(
"is_complete": False,
},
) -> BugoutResource:
resource = bc.create_resource(
token=token,
application_id=MOONSTREAM_APPLICATION_ID,
@ -317,7 +316,6 @@ def dashboards_abi_validation(
abi: Any,
s3_path: str,
):
"""
Validate current dashboard subscription : https://github.com/bugout-dev/moonstream/issues/345#issuecomment-953052444
with contract abi on S3
@ -332,7 +330,6 @@ def dashboards_abi_validation(
}
if not dashboard_subscription.all_methods:
for method in dashboard_subscription.methods:
if method["name"] not in abi_functions:
# Method not exists
logger.error(
@ -342,9 +339,7 @@ def dashboards_abi_validation(
)
raise MoonstreamHTTPException(status_code=400)
if method.get("filters") and isinstance(method["filters"], dict):
for input_argument_name, value in method["filters"].items():
if input_argument_name not in abi_functions[method["name"]]:
# Argument not exists
logger.error(
@ -373,7 +368,6 @@ def dashboards_abi_validation(
if not dashboard_subscription.all_events:
for event in dashboard_subscription.events:
if event["name"] not in abi_events:
logger.error(
f"Error on dashboard resource validation event:{event['name']}"
@ -383,9 +377,7 @@ def dashboards_abi_validation(
raise MoonstreamHTTPException(status_code=400)
if event.get("filters") and isinstance(event["filters"], dict):
for input_argument_name, value in event["filters"].items():
if input_argument_name not in abi_events[event["name"]]:
# Argument not exists
logger.error(
@ -482,7 +474,6 @@ def get_all_entries_from_search(
reporter.error_report(e)
if len(results) != existing_metods.total_results:
for offset in range(limit, existing_metods.total_results, limit):
existing_metods = bc.search(
token=token,

20
backend/moonstreamapi/admin/subscription_types.py
Wyświetl plik

@ -57,6 +57,16 @@ CANONICAL_SUBSCRIPTION_TYPES = {
stripe_price_id=None,
active=True,
),
"caldera_smartcontract": SubscriptionTypeResourceData(
id="caldera_smartcontract",
name="Caldera smartcontract",
choices=["input:address", "tag:erc721"],
description="Contracts events and tx_calls of contract of Caldera blockchain.",
icon_url="https://s3.amazonaws.com/static.simiotics.com/moonstream/assets/xdai-token-logo.png",
stripe_product_id=None,
stripe_price_id=None,
active=True,
),
"ethereum_blockchain": SubscriptionTypeResourceData(
id="ethereum_blockchain",
name="Ethereum transactions",
@ -97,6 +107,16 @@ CANONICAL_SUBSCRIPTION_TYPES = {
stripe_price_id=None,
active=True,
),
"caldera_blockchain": SubscriptionTypeResourceData(
id="caldera_blockchain",
name="Caldera transactions",
choices=["input:address", "tag:erc721"],
description="Caldera chain transactions subscription.",
icon_url="https://s3.amazonaws.com/static.simiotics.com/moonstream/assets/xdai-token-logo.png",
stripe_product_id=None,
stripe_price_id=None,
active=True,
),
"ethereum_whalewatch": SubscriptionTypeResourceData(
id="ethereum_whalewatch",
name="Ethereum whale watch",

3
crawlers/mooncrawl/mooncrawl/blockchain.py
Wyświetl plik

@ -29,6 +29,7 @@ from .settings import (
MOONSTREAM_POLYGON_WEB3_PROVIDER_URI,
MOONSTREAM_MUMBAI_WEB3_PROVIDER_URI,
MOONSTREAM_XDAI_WEB3_PROVIDER_URI,
MOONSTREAM_CALDERA_WEB3_PROVIDER_URI,
NB_ACCESS_ID_HEADER,
NB_DATA_SOURCE_HEADER,
WEB3_CLIENT_REQUEST_TIMEOUT_SECONDS,
@ -70,6 +71,8 @@ def connect(
web3_uri = MOONSTREAM_MUMBAI_WEB3_PROVIDER_URI
elif blockchain_type == AvailableBlockchainType.XDAI:
web3_uri = MOONSTREAM_XDAI_WEB3_PROVIDER_URI
elif blockchain_type == AvailableBlockchainType.CALDERA:
web3_uri = MOONSTREAM_CALDERA_WEB3_PROVIDER_URI
else:
raise Exception("Wrong blockchain type provided for web3 URI")

2
crawlers/mooncrawl/mooncrawl/moonworm_crawler/continuous_crawler.py
Wyświetl plik

@ -124,6 +124,8 @@ def continuous_crawler(
network = Network.mumbai
elif blockchain_type == AvailableBlockchainType.XDAI:
network = Network.xdai
elif blockchain_type == AvailableBlockchainType.CALDERA:
network = Network.caldera
else:
raise ValueError(f"Unknown blockchain type: {blockchain_type}")

3
crawlers/mooncrawl/mooncrawl/moonworm_crawler/crawler.py
Wyświetl plik

@ -31,6 +31,7 @@ class SubscriptionTypes(Enum):
ETHEREUM_BLOCKCHAIN = "ethereum_smartcontract"
MUMBAI_BLOCKCHAIN = "mumbai_smartcontract"
XDAI_BLOCKCHAIN = "xdai_smartcontract"
CALDERA_BLOCKCHAIN = "caldera_smartcontract"
def abi_input_signature(input_abi: Dict[str, Any]) -> str:
@ -133,6 +134,8 @@ def blockchain_type_to_subscription_type(
return SubscriptionTypes.MUMBAI_BLOCKCHAIN
elif blockchain_type == AvailableBlockchainType.XDAI:
return SubscriptionTypes.XDAI_BLOCKCHAIN
elif blockchain_type == AvailableBlockchainType.CALDERA:
return SubscriptionTypes.CALDERA_BLOCKCHAIN
else:
raise ValueError(f"Unknown blockchain type: {blockchain_type}")

6
crawlers/mooncrawl/mooncrawl/settings.py
Wyświetl plik

@ -99,6 +99,12 @@ MOONSTREAM_XDAI_WEB3_PROVIDER_URI = os.environ.get(
if MOONSTREAM_XDAI_WEB3_PROVIDER_URI == "":
raise Exception("MOONSTREAM_XDAI_WEB3_PROVIDER_URI env variable is not set")
MOONSTREAM_CALDERA_WEB3_PROVIDER_URI = os.environ.get(
"MOONSTREAM_CALDERA_WEB3_PROVIDER_URI", ""
)
if MOONSTREAM_CALDERA_WEB3_PROVIDER_URI == "":
raise Exception("MOONSTREAM_CALDERA_WEB3_PROVIDER_URI env variable is not set")
MOONSTREAM_CRAWL_WORKERS = 4
MOONSTREAM_CRAWL_WORKERS_RAW = os.environ.get("MOONSTREAM_CRAWL_WORKERS")
try:

116
db/alembic/versions/5093601738e5_create_caldera_blockchain_tables.py 100644
Wyświetl plik

@ -0,0 +1,116 @@
"""Create caldera blockchain tables
Revision ID: 5093601738e5
Revises: 11233cf42d62
Create Date: 2023-03-01 17:16:32.033661
"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql
# revision identifiers, used by Alembic.
revision = '5093601738e5'
down_revision = '11233cf42d62'
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.create_table('caldera_blocks',
sa.Column('block_number', sa.BigInteger(), nullable=False),
sa.Column('difficulty', sa.BigInteger(), nullable=True),
sa.Column('extra_data', sa.VARCHAR(length=128), nullable=True),
sa.Column('gas_limit', sa.BigInteger(), nullable=True),
sa.Column('gas_used', sa.BigInteger(), nullable=True),
sa.Column('base_fee_per_gas', sa.Numeric(precision=78, scale=0), nullable=True),
sa.Column('hash', sa.VARCHAR(length=256), nullable=True),
sa.Column('logs_bloom', sa.VARCHAR(length=1024), nullable=True),
sa.Column('miner', sa.VARCHAR(length=256), nullable=True),
sa.Column('nonce', sa.VARCHAR(length=256), nullable=True),
sa.Column('parent_hash', sa.VARCHAR(length=256), nullable=True),
sa.Column('receipt_root', sa.VARCHAR(length=256), nullable=True),
sa.Column('uncles', sa.VARCHAR(length=256), nullable=True),
sa.Column('size', sa.Integer(), nullable=True),
sa.Column('state_root', sa.VARCHAR(length=256), nullable=True),
sa.Column('timestamp', sa.BigInteger(), nullable=True),
sa.Column('total_difficulty', sa.VARCHAR(length=256), nullable=True),
sa.Column('transactions_root', sa.VARCHAR(length=256), nullable=True),
sa.Column('indexed_at', sa.DateTime(timezone=True), server_default=sa.text("TIMEZONE('utc', statement_timestamp())"), nullable=False),
sa.PrimaryKeyConstraint('block_number', name=op.f('pk_caldera_blocks'))
)
op.create_index(op.f('ix_caldera_blocks_block_number'), 'caldera_blocks', ['block_number'], unique=True)
op.create_index(op.f('ix_caldera_blocks_hash'), 'caldera_blocks', ['hash'], unique=False)
op.create_index(op.f('ix_caldera_blocks_timestamp'), 'caldera_blocks', ['timestamp'], unique=False)
op.create_table('caldera_labels',
sa.Column('id', sa.UUID(), nullable=False),
sa.Column('label', sa.VARCHAR(length=256), nullable=False),
sa.Column('block_number', sa.BigInteger(), nullable=True),
sa.Column('address', sa.VARCHAR(length=256), nullable=True),
sa.Column('transaction_hash', sa.VARCHAR(length=256), nullable=True),
sa.Column('label_data', postgresql.JSONB(astext_type=sa.Text()), nullable=True),
sa.Column('block_timestamp', sa.BigInteger(), nullable=True),
sa.Column('log_index', sa.Integer(), nullable=True),
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text("TIMEZONE('utc', statement_timestamp())"), nullable=False),
sa.PrimaryKeyConstraint('id', name=op.f('pk_caldera_labels')),
sa.UniqueConstraint('id', name=op.f('uq_caldera_labels_id'))
)
op.create_index(op.f('ix_caldera_labels_address'), 'caldera_labels', ['address'], unique=False)
op.create_index('ix_caldera_labels_address_block_number', 'caldera_labels', ['address', 'block_number'], unique=False)
op.create_index('ix_caldera_labels_address_block_timestamp', 'caldera_labels', ['address', 'block_timestamp'], unique=False)
op.create_index(op.f('ix_caldera_labels_block_number'), 'caldera_labels', ['block_number'], unique=False)
op.create_index(op.f('ix_caldera_labels_block_timestamp'), 'caldera_labels', ['block_timestamp'], unique=False)
op.create_index(op.f('ix_caldera_labels_label'), 'caldera_labels', ['label'], unique=False)
op.create_index(op.f('ix_caldera_labels_transaction_hash'), 'caldera_labels', ['transaction_hash'], unique=False)
op.create_table('caldera_transactions',
sa.Column('hash', sa.VARCHAR(length=256), nullable=False),
sa.Column('block_number', sa.BigInteger(), nullable=False),
sa.Column('from_address', sa.VARCHAR(length=256), nullable=True),
sa.Column('to_address', sa.VARCHAR(length=256), nullable=True),
sa.Column('gas', sa.Numeric(precision=78, scale=0), nullable=True),
sa.Column('gas_price', sa.Numeric(precision=78, scale=0), nullable=True),
sa.Column('max_fee_per_gas', sa.Numeric(precision=78, scale=0), nullable=True),
sa.Column('max_priority_fee_per_gas', sa.Numeric(precision=78, scale=0), nullable=True),
sa.Column('input', sa.Text(), nullable=True),
sa.Column('nonce', sa.VARCHAR(length=256), nullable=True),
sa.Column('transaction_index', sa.BigInteger(), nullable=True),
sa.Column('transaction_type', sa.Integer(), nullable=True),
sa.Column('value', sa.Numeric(precision=78, scale=0), nullable=True),
sa.Column('indexed_at', sa.DateTime(timezone=True), server_default=sa.text("TIMEZONE('utc', statement_timestamp())"), nullable=False),
sa.ForeignKeyConstraint(['block_number'], ['caldera_blocks.block_number'], name=op.f('fk_caldera_transactions_block_number_caldera_blocks'), ondelete='CASCADE'),
sa.PrimaryKeyConstraint('hash', name=op.f('pk_caldera_transactions'))
)
op.create_index(op.f('ix_caldera_transactions_block_number'), 'caldera_transactions', ['block_number'], unique=False)
op.create_index(op.f('ix_caldera_transactions_from_address'), 'caldera_transactions', ['from_address'], unique=False)
op.create_index(op.f('ix_caldera_transactions_gas'), 'caldera_transactions', ['gas'], unique=False)
op.create_index(op.f('ix_caldera_transactions_gas_price'), 'caldera_transactions', ['gas_price'], unique=False)
op.create_index(op.f('ix_caldera_transactions_hash'), 'caldera_transactions', ['hash'], unique=True)
op.create_index(op.f('ix_caldera_transactions_to_address'), 'caldera_transactions', ['to_address'], unique=False)
op.create_index(op.f('ix_caldera_transactions_value'), 'caldera_transactions', ['value'], unique=False)
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.drop_index(op.f('ix_caldera_transactions_value'), table_name='caldera_transactions')
op.drop_index(op.f('ix_caldera_transactions_to_address'), table_name='caldera_transactions')
op.drop_index(op.f('ix_caldera_transactions_hash'), table_name='caldera_transactions')
op.drop_index(op.f('ix_caldera_transactions_gas_price'), table_name='caldera_transactions')
op.drop_index(op.f('ix_caldera_transactions_gas'), table_name='caldera_transactions')
op.drop_index(op.f('ix_caldera_transactions_from_address'), table_name='caldera_transactions')
op.drop_index(op.f('ix_caldera_transactions_block_number'), table_name='caldera_transactions')
op.drop_table('caldera_transactions')
op.drop_index(op.f('ix_caldera_labels_transaction_hash'), table_name='caldera_labels')
op.drop_index(op.f('ix_caldera_labels_label'), table_name='caldera_labels')
op.drop_index(op.f('ix_caldera_labels_block_timestamp'), table_name='caldera_labels')
op.drop_index(op.f('ix_caldera_labels_block_number'), table_name='caldera_labels')
op.drop_index('ix_caldera_labels_address_block_timestamp', table_name='caldera_labels')
op.drop_index('ix_caldera_labels_address_block_number', table_name='caldera_labels')
op.drop_index(op.f('ix_caldera_labels_address'), table_name='caldera_labels')
op.drop_table('caldera_labels')
op.drop_index(op.f('ix_caldera_blocks_timestamp'), table_name='caldera_blocks')
op.drop_index(op.f('ix_caldera_blocks_hash'), table_name='caldera_blocks')
op.drop_index(op.f('ix_caldera_blocks_block_number'), table_name='caldera_blocks')
op.drop_table('caldera_blocks')
# ### end Alembic commands ###

10
db/moonstreamdb/blockchain.py
Wyświetl plik

@ -12,6 +12,9 @@ from .models import (
XDaiBlock,
XDaiLabel,
XDaiTransaction,
CalderaBlock,
CalderaLabel,
CalderaTransaction
)
from enum import Enum
@ -24,6 +27,7 @@ class AvailableBlockchainType(Enum):
POLYGON = "polygon"
MUMBAI = "mumbai"
XDAI = "xdai"
CALDERA = "caldera"
def get_block_model(
@ -42,6 +46,8 @@ def get_block_model(
block_model = MumbaiBlock
elif blockchain_type == AvailableBlockchainType.XDAI:
block_model = XDaiBlock
elif blockchain_type == AvailableBlockchainType.CALDERA:
block_model = CalderaBlock
else:
raise Exception("Unsupported blockchain type provided")
@ -64,6 +70,8 @@ def get_label_model(
label_model = MumbaiLabel
elif blockchain_type == AvailableBlockchainType.XDAI:
label_model = XDaiLabel
elif blockchain_type == AvailableBlockchainType.CALDERA:
label_model = CalderaLabel
else:
raise Exception("Unsupported blockchain type provided")
@ -92,6 +100,8 @@ def get_transaction_model(
transaction_model = MumbaiTransaction
elif blockchain_type == AvailableBlockchainType.XDAI:
transaction_model = XDaiTransaction
elif blockchain_type == AvailableBlockchainType.CALDERA:
transaction_model = CalderaTransaction
else:
raise Exception("Unsupported blockchain type provided")

121
db/moonstreamdb/models.py
Wyświetl plik

@ -493,6 +493,127 @@ class XDaiLabel(Base): # type: ignore
)
class CalderaBlock(Base): # type: ignore
__tablename__ = "caldera_blocks"
block_number = Column(
BigInteger, primary_key=True, unique=True, nullable=False, index=True
)
difficulty = Column(BigInteger)
extra_data = Column(VARCHAR(128))
gas_limit = Column(BigInteger)
gas_used = Column(BigInteger)
base_fee_per_gas = Column(Numeric(precision=78, scale=0), nullable=True)
hash = Column(VARCHAR(256), index=True)
logs_bloom = Column(VARCHAR(1024))
miner = Column(VARCHAR(256))
nonce = Column(VARCHAR(256))
parent_hash = Column(VARCHAR(256))
receipt_root = Column(VARCHAR(256))
uncles = Column(VARCHAR(256))
size = Column(Integer)
state_root = Column(VARCHAR(256))
timestamp = Column(BigInteger, index=True)
total_difficulty = Column(VARCHAR(256))
transactions_root = Column(VARCHAR(256))
indexed_at = Column(
DateTime(timezone=True), server_default=utcnow(), nullable=False
)
class CalderaTransaction(Base): # type: ignore
__tablename__ = "caldera_transactions"
hash = Column(
VARCHAR(256), primary_key=True, unique=True, nullable=False, index=True
)
block_number = Column(
BigInteger,
ForeignKey("caldera_blocks.block_number", ondelete="CASCADE"),
nullable=False,
index=True,
)
from_address = Column(VARCHAR(256), index=True)
to_address = Column(VARCHAR(256), index=True)
gas = Column(Numeric(precision=78, scale=0), index=True)
gas_price = Column(Numeric(precision=78, scale=0), index=True)
max_fee_per_gas = Column(Numeric(precision=78, scale=0), nullable=True)
max_priority_fee_per_gas = Column(Numeric(precision=78, scale=0), nullable=True)
input = Column(Text)
nonce = Column(VARCHAR(256))
transaction_index = Column(BigInteger)
transaction_type = Column(Integer, nullable=True)
value = Column(Numeric(precision=78, scale=0), index=True)
indexed_at = Column(
DateTime(timezone=True), server_default=utcnow(), nullable=False
)
class CalderaLabel(Base): # type: ignore
"""
Example of label_data:
{
"label": "ERC20",
"label_data": {
"name": "Uniswap",
"symbol": "UNI"
}
},
{
"label": "Exchange"
"label_data": {...}
}
"""
__tablename__ = "caldera_labels"
__table_args__ = (
Index(
"ix_caldera_labels_address_block_number",
"address",
"block_number",
unique=False,
),
Index(
"ix_caldera_labels_address_block_timestamp",
"address",
"block_timestamp",
unique=False,
),
)
id = Column(
UUID(as_uuid=True),
primary_key=True,
default=uuid.uuid4,
unique=True,
nullable=False,
)
label = Column(VARCHAR(256), nullable=False, index=True)
block_number = Column(
BigInteger,
nullable=True,
index=True,
)
address = Column(
VARCHAR(256),
nullable=True,
index=True,
)
transaction_hash = Column(
VARCHAR(256),
nullable=True,
index=True,
)
label_data = Column(JSONB, nullable=True)
block_timestamp = Column(BigInteger, index=True)
log_index = Column(Integer, nullable=True)
created_at = Column(
DateTime(timezone=True), server_default=utcnow(), nullable=False
)
class ESDFunctionSignature(Base): # type: ignore
"""
Function signature from blockchain (Ethereum/Polygon) Signature Database.

2
db/moonstreamdb/version.py
Wyświetl plik

@ -2,4 +2,4 @@
Moonstream database version.
"""
MOONSTREAMDB_VERSION = "0.3.2"
MOONSTREAMDB_VERSION = "0.3.3"