kopia lustrzana https://github.com/bugout-dev/moonstream
Merge pull request #18 from bugout-dev/smart-contract-crawlers
Smart contract information - crawlers and API endpointnew-indices
Neeraj Kashyap
GitHub
commit
9e6fa856bb
|
|
@ -165,3 +165,4 @@ dev.env
|
|||
prod.env
|
||||
.moonstream
|
||||
.venv
|
||||
.secrets
|
||||
|
|
|
|||
|
|
@ -0,0 +1,108 @@
|
|||
import argparse
|
||||
import binascii
|
||||
import sys
|
||||
from typing import List, Optional, Union, Type, cast
|
||||
|
||||
import pyevmasm
|
||||
|
||||
from moonstreamdb.db import yield_db_session
|
||||
from moonstreamdb.models import ESDEventSignature, ESDFunctionSignature
|
||||
from sqlalchemy.orm.session import Session
|
||||
from sqlalchemy.sql.expression import text
|
||||
from .data import EVMEventSignature, EVMFunctionSignature, ContractABI
|
||||
|
||||
|
||||
def query_for_text_signatures(
|
||||
session: Session,
|
||||
hex_signature: str,
|
||||
db_model: Union[ESDFunctionSignature, ESDEventSignature],
|
||||
) -> List[str]:
|
||||
query = session.query(db_model)
|
||||
query = query.filter(db_model.hex_signature == hex_signature)
|
||||
results = query.all()
|
||||
text_signatures = []
|
||||
for el in results:
|
||||
text_signatures.append(el.text_signature)
|
||||
return text_signatures
|
||||
|
||||
|
||||
def decode_signatures(
|
||||
session: Session,
|
||||
hex_signatures: List[str],
|
||||
data_model: Union[Type[EVMEventSignature], Type[EVMFunctionSignature]],
|
||||
db_model: Union[ESDEventSignature, ESDFunctionSignature],
|
||||
) -> List[Union[EVMEventSignature, EVMFunctionSignature]]:
|
||||
decoded_signatures = []
|
||||
for hex_signature in hex_signatures:
|
||||
signature = data_model(hex_signature=hex_signature)
|
||||
signature.text_signature_candidates = query_for_text_signatures(
|
||||
session, hex_signature, db_model
|
||||
)
|
||||
decoded_signatures.append(signature)
|
||||
return decoded_signatures
|
||||
|
||||
|
||||
def decode_abi(source: str, session: Optional[Session] = None) -> ContractABI:
|
||||
normalized_source = source
|
||||
if normalized_source[:2] == "0x":
|
||||
normalized_source = normalized_source[2:]
|
||||
disassembled = pyevmasm.disassemble_all(binascii.unhexlify(normalized_source))
|
||||
function_hex_signatures = []
|
||||
event_hex_signatures = []
|
||||
|
||||
should_close_session = False
|
||||
if session is None:
|
||||
should_close_session = True
|
||||
session = next(yield_db_session())
|
||||
|
||||
for instruction in disassembled:
|
||||
if instruction.name == "PUSH4":
|
||||
hex_signature = "0x{:x}".format(instruction.operand)
|
||||
if hex_signature not in function_hex_signatures:
|
||||
function_hex_signatures.append(hex_signature)
|
||||
elif instruction.name == "PUSH32":
|
||||
hex_signature = "0x{:x}".format(instruction.operand)
|
||||
if hex_signature not in event_hex_signatures:
|
||||
event_hex_signatures.append(hex_signature)
|
||||
|
||||
try:
|
||||
function_signatures = decode_signatures(
|
||||
session, function_hex_signatures, EVMFunctionSignature, ESDFunctionSignature
|
||||
)
|
||||
event_signatures = decode_signatures(
|
||||
session, event_hex_signatures, EVMEventSignature, ESDEventSignature
|
||||
)
|
||||
finally:
|
||||
if should_close_session:
|
||||
session.close()
|
||||
|
||||
abi = ContractABI(
|
||||
functions=cast(EVMFunctionSignature, function_signatures),
|
||||
events=cast(EVMEventSignature, event_signatures),
|
||||
)
|
||||
return abi
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Decode Ethereum smart contract ABIs")
|
||||
parser.add_argument(
|
||||
"-i",
|
||||
"--infile",
|
||||
type=argparse.FileType("r"),
|
||||
default=sys.stdin,
|
||||
help="File containing the ABI to decode",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
source: Optional[str] = None
|
||||
with args.infile as ifp:
|
||||
source = ifp.read().strip()
|
||||
if source is None:
|
||||
raise ValueError("Could not read ABI.")
|
||||
|
||||
abi = decode_abi(source)
|
||||
print(abi.json())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
@ -9,6 +9,7 @@ from fastapi.middleware.cors import CORSMiddleware
|
|||
from . import data
|
||||
from .routes.subscriptions import app as subscriptions_api
|
||||
from .routes.users import app as users_api
|
||||
from .routes.txinfo import app as txinfo_api
|
||||
from .settings import ORIGINS
|
||||
from .version import MOONSTREAM_VERSION
|
||||
|
||||
|
|
@ -38,3 +39,4 @@ async def version_handler() -> data.VersionResponse:
|
|||
|
||||
app.mount("/subscriptions", subscriptions_api)
|
||||
app.mount("/users", users_api)
|
||||
app.mount("/txinfo", txinfo_api)
|
||||
|
|
|
|||
|
|
@ -70,4 +70,42 @@ class SubscriptionResponse(BaseModel):
|
|||
|
||||
|
||||
class SubscriptionsListResponse(BaseModel):
|
||||
subscriptions: List[SubscriptionResourceData] = Field(default_factory=list)
|
||||
subscriptions: List[SubscriptionResponse] = Field(default_factory=list)
|
||||
|
||||
|
||||
class EVMFunctionSignature(BaseModel):
|
||||
type = "function"
|
||||
hex_signature: str
|
||||
text_signature_candidates: List[str] = Field(default_factory=list)
|
||||
|
||||
|
||||
class EVMEventSignature(BaseModel):
|
||||
type = "event"
|
||||
hex_signature: str
|
||||
text_signature_candidates: List[str] = Field(default_factory=list)
|
||||
|
||||
|
||||
class ContractABI(BaseModel):
|
||||
functions: List[EVMFunctionSignature]
|
||||
events: List[EVMEventSignature]
|
||||
|
||||
|
||||
class EthereumTransaction(BaseModel):
|
||||
gas: int
|
||||
gasPrice: int
|
||||
value: int
|
||||
from_address: str = Field(alias="from")
|
||||
to_address: Optional[str] = Field(default=None, alias="to")
|
||||
hash: Optional[str] = None
|
||||
input: Optional[str] = None
|
||||
|
||||
|
||||
class TxinfoEthereumBlockchainRequest(BaseModel):
|
||||
tx: EthereumTransaction
|
||||
|
||||
|
||||
class TxinfoEthereumBlockchainResponse(BaseModel):
|
||||
tx: EthereumTransaction
|
||||
abi: Optional[ContractABI] = None
|
||||
errors: List[str] = Field(default_factory=list)
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,80 @@
|
|||
"""
|
||||
Moonstream's /txinfo endpoints.
|
||||
|
||||
These endpoints enrich raw blockchain transactions (as well as pending transactions, hypothetical
|
||||
transactions, etc.) with side information and return objects that are better suited for displaying to
|
||||
end users.
|
||||
"""
|
||||
import logging
|
||||
from typing import Any, Dict
|
||||
|
||||
from fastapi import (
|
||||
FastAPI,
|
||||
Depends,
|
||||
HTTPException,
|
||||
Request,
|
||||
)
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from moonstreamdb.db import yield_db_session
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ..abi_decoder import decode_abi
|
||||
from ..data import TxinfoEthereumBlockchainRequest, TxinfoEthereumBlockchainResponse
|
||||
from ..middleware import BroodAuthMiddleware
|
||||
from ..settings import (
|
||||
MOONSTREAM_APPLICATION_ID,
|
||||
DOCS_TARGET_PATH,
|
||||
ORIGINS,
|
||||
DOCS_PATHS,
|
||||
bugout_client as bc,
|
||||
)
|
||||
from ..version import MOONSTREAM_VERSION
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
tags_metadata = [
|
||||
{"name": "users", "description": "Operations with users."},
|
||||
{"name": "tokens", "description": "Operations with user tokens."},
|
||||
]
|
||||
|
||||
app = FastAPI(
|
||||
title=f"Moonstream users API.",
|
||||
description="User, token and password handlers.",
|
||||
version=MOONSTREAM_VERSION,
|
||||
openapi_tags=tags_metadata,
|
||||
openapi_url="/openapi.json",
|
||||
docs_url=None,
|
||||
redoc_url=f"/{DOCS_TARGET_PATH}",
|
||||
)
|
||||
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=ORIGINS,
|
||||
allow_credentials=True,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
whitelist_paths: Dict[str, str] = {}
|
||||
whitelist_paths.update(DOCS_PATHS)
|
||||
app.add_middleware(BroodAuthMiddleware, whitelist=whitelist_paths)
|
||||
|
||||
|
||||
@app.post(
|
||||
"/ethereum_blockchain",
|
||||
tags=["txinfo"],
|
||||
response_model=TxinfoEthereumBlockchainResponse,
|
||||
)
|
||||
async def txinfo_ethereum_blockchain_handler(
|
||||
txinfo_request: TxinfoEthereumBlockchainRequest,
|
||||
db_session: Session = Depends(yield_db_session),
|
||||
) -> TxinfoEthereumBlockchainResponse:
|
||||
response = TxinfoEthereumBlockchainResponse(tx=txinfo_request.tx)
|
||||
if txinfo_request.tx.input is not None:
|
||||
try:
|
||||
response.abi = decode_abi(txinfo_request.tx.input, db_session)
|
||||
except Exception as err:
|
||||
logger.error(r"Could not decode ABI:")
|
||||
logger.error(err)
|
||||
response.errors.append("Could not decode ABI from the given input")
|
||||
return response
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
[mypy]
|
||||
|
||||
[mypy-sqlalchemy.*]
|
||||
ignore_missing_imports = True
|
||||
|
||||
[mypy-moonstreamdb.*]
|
||||
ignore_missing_imports = True
|
||||
|
||||
[mypy-pyevmasm.*]
|
||||
ignore_missing_imports = True
|
||||
|
|
@ -8,14 +8,15 @@ certifi==2021.5.30
|
|||
charset-normalizer==2.0.3
|
||||
click==8.0.1
|
||||
fastapi==0.66.0
|
||||
-e git+https://git@github.com/bugout-dev/moonstream.git@876c23aac10f07da700798f47c44797a4ae157bb#egg=moonstreamdb&subdirectory=db
|
||||
h11==0.12.0
|
||||
idna==3.2
|
||||
jmespath==0.10.0
|
||||
-e git+ssh://git@github.com/bugout-dev/moonstream.git@b9c828fc7f811af88a9f3a45dd7f5c4053433366#egg=moonstreamdb&subdirectory=db
|
||||
mypy==0.910
|
||||
mypy-extensions==0.4.3
|
||||
pathspec==0.9.0
|
||||
pydantic==1.8.2
|
||||
pyevmasm==0.2.3
|
||||
python-dateutil==2.8.2
|
||||
python-multipart==0.0.5
|
||||
regex==2021.7.6
|
||||
|
|
|
|||
|
|
@ -1,4 +1,6 @@
|
|||
export MOONSTREAM_CORS_ALLOWED_ORIGINS="http://localhost:3000,https://moonstream.to,https://www.moonstream.to"
|
||||
export MOONSTREAM_OPENAPI_LIST="users,subscriptions"
|
||||
export MOONSTREAM_OPENAPI_LIST="users,subscriptions,txinfo"
|
||||
export MOONSTREAM_APPLICATION_ID="<issued_bugout_application_id>"
|
||||
export MOONSTREAM_DATA_JOURNAL_ID="<bugout_journal_id_to_store_blockchain_data>"
|
||||
export MOONSTREAM_DB_URI="postgresql://<username>:<password>@<db_host>:<db_port>/<db_name>"
|
||||
export MOONSTREAM_POOL_SIZE=0
|
||||
|
|
|
|||
|
|
@ -0,0 +1,52 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
TIMESTAMP="$(date +%s)"
|
||||
SCRIPT_DIR=$(realpath $(dirname $0))
|
||||
|
||||
API_URL="${MOONSTREAM_DEV_API_URL:-http://localhost:7481}"
|
||||
|
||||
MOONSTREAM_USERNAME="devuser_$TIMESTAMP"
|
||||
MOONSTREAM_PASSWORD="peppercat"
|
||||
MOONSTREAM_EMAIL="devuser_$TIMESTAMP@example.com"
|
||||
|
||||
OUTPUT_DIR=$(mktemp -d)
|
||||
echo "Writing responses to directory: $OUTPUT_DIR"
|
||||
|
||||
# Create a new user
|
||||
curl -X POST \
|
||||
-H "Content-Type: multipart/form-data" \
|
||||
"$API_URL/users/" \
|
||||
-F "username=$MOONSTREAM_USERNAME" \
|
||||
-F "password=$MOONSTREAM_PASSWORD" \
|
||||
-F "email=$MOONSTREAM_EMAIL" \
|
||||
-o $OUTPUT_DIR/user.json
|
||||
|
||||
# Create a token for this user
|
||||
curl -X POST \
|
||||
-H "Content-Type: multipart/form-data" \
|
||||
"$API_URL/users/token" \
|
||||
-F "username=$MOONSTREAM_USERNAME" \
|
||||
-F "password=$MOONSTREAM_PASSWORD" \
|
||||
-o $OUTPUT_DIR/token.json
|
||||
|
||||
API_TOKEN=$(jq -r '.id' $OUTPUT_DIR/token.json)
|
||||
|
||||
set -e
|
||||
|
||||
ETHEREUM_TXINFO_REQUEST_BODY_JSON=$(jq -r . $SCRIPT_DIR/txinfo_ethereum_blockchain_request.json)
|
||||
curl -f -X POST \
|
||||
-H "Content-Type: application/json" \
|
||||
-H "Authorization: Bearer $API_TOKEN" \
|
||||
"$API_URL/txinfo/ethereum_blockchain" \
|
||||
-d "$ETHEREUM_TXINFO_REQUEST_BODY_JSON" \
|
||||
-o $OUTPUT_DIR/txinfo_response.json
|
||||
|
||||
echo "Response:"
|
||||
jq . $OUTPUT_DIR/txinfo_response.json
|
||||
|
||||
if [ "$DEBUG" != true ]
|
||||
then
|
||||
echo "Deleting output directory: $OUTPUT_DIR"
|
||||
echo "Please set DEBUG=true if you would prefer to retain this directory in the future"
|
||||
rm -r $OUTPUT_DIR
|
||||
fi
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
{
|
||||
"tx": {
|
||||
"to": null,
|
||||
"from": "0x2E337E0Fb68F5e51ce9295E80BCd02273d7420c4",
|
||||
"gas": 2265656,
|
||||
"gasPrice": 1000000000,
|
||||
"hash": "0x5f0b6e212e55c7120f36fe6f88d46eb001c848064fd099116b42805bb3564ae6",
|
||||
"value": 0,
|
||||
"input": "0x606061026b61014039602061026b60c03960c05160a01c1561002057600080fd5b61014051600055610160516001556001546101805181818301101561004457600080fd5b80820190509050600255600254421061005c57600080fd5b61025356600436101561000d576101ec565b600035601c52600051631998aeef8114156100855760015442101561003157600080fd5b600254421061003f57600080fd5b600454341161004d57600080fd5b600660035460e05260c052604060c020805460045481818301101561007157600080fd5b808201905090508155503360035534600455005b341561009057600080fd5b633ccfd60b8114156100db5760063360e05260c052604060c0205461014052600060063360e05260c052604060c02055600060006000600061014051336000f16100d957600080fd5b005b63fe67a54b811415610124576002544210156100f657600080fd5b6005541561010357600080fd5b600160055560006000600060006004546000546000f161012257600080fd5b005b6338af3eed81141561013c5760005460005260206000f35b634f245ef78114156101545760015460005260206000f35b632a24f46c81141561016c5760025460005260206000f35b6391f901578114156101845760035460005260206000f35b63d57bde7981141561019c5760045460005260206000f35b6312fa6feb8114156101b45760055460005260206000f35b6326b387bb8114156101ea5760043560a01c156101d057600080fd5b600660043560e05260c052604060c0205460005260206000f35b505b60006000fd5b61006161025303610061600039610061610253036000f30000000000000000000000002e337e0fb68f5e51ce9295e80bcd02273d7420c40000000000000000000000000000000000000000000000000000000060d2b04a00000000000000000000000000000000000000000000000000000000616b46ca"
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,171 @@
|
|||
|
||||
# Created by https://www.toptal.com/developers/gitignore/api/python,visualstudiocode
|
||||
# Edit at https://www.toptal.com/developers/gitignore?templates=python,visualstudiocode
|
||||
|
||||
### Python ###
|
||||
# Byte-compiled / optimized / DLL files
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
|
||||
# C extensions
|
||||
*.so
|
||||
|
||||
# Distribution / packaging
|
||||
.Python
|
||||
build/
|
||||
develop-eggs/
|
||||
dist/
|
||||
downloads/
|
||||
eggs/
|
||||
.eggs/
|
||||
lib/
|
||||
lib64/
|
||||
parts/
|
||||
sdist/
|
||||
var/
|
||||
wheels/
|
||||
share/python-wheels/
|
||||
*.egg-info/
|
||||
.installed.cfg
|
||||
*.egg
|
||||
MANIFEST
|
||||
|
||||
# PyInstaller
|
||||
# Usually these files are written by a python script from a template
|
||||
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
||||
*.manifest
|
||||
*.spec
|
||||
|
||||
# Installer logs
|
||||
pip-log.txt
|
||||
pip-delete-this-directory.txt
|
||||
|
||||
# Unit test / coverage reports
|
||||
htmlcov/
|
||||
.tox/
|
||||
.nox/
|
||||
.coverage
|
||||
.coverage.*
|
||||
.cache
|
||||
nosetests.xml
|
||||
coverage.xml
|
||||
*.cover
|
||||
*.py,cover
|
||||
.hypothesis/
|
||||
.pytest_cache/
|
||||
cover/
|
||||
|
||||
# Translations
|
||||
*.mo
|
||||
*.pot
|
||||
|
||||
# Django stuff:
|
||||
*.log
|
||||
local_settings.py
|
||||
db.sqlite3
|
||||
db.sqlite3-journal
|
||||
|
||||
# Flask stuff:
|
||||
instance/
|
||||
.webassets-cache
|
||||
|
||||
# Scrapy stuff:
|
||||
.scrapy
|
||||
|
||||
# Sphinx documentation
|
||||
docs/_build/
|
||||
|
||||
# PyBuilder
|
||||
.pybuilder/
|
||||
target/
|
||||
|
||||
# Jupyter Notebook
|
||||
.ipynb_checkpoints
|
||||
|
||||
# IPython
|
||||
profile_default/
|
||||
ipython_config.py
|
||||
|
||||
# pyenv
|
||||
# For a library or package, you might want to ignore these files since the code is
|
||||
# intended to run in multiple environments; otherwise, check them in:
|
||||
# .python-version
|
||||
|
||||
# pipenv
|
||||
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
||||
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
||||
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
||||
# install all needed dependencies.
|
||||
#Pipfile.lock
|
||||
|
||||
# PEP 582; used by e.g. github.com/David-OConnor/pyflow
|
||||
__pypackages__/
|
||||
|
||||
# Celery stuff
|
||||
celerybeat-schedule
|
||||
celerybeat.pid
|
||||
|
||||
# SageMath parsed files
|
||||
*.sage.py
|
||||
|
||||
# Environments
|
||||
.env
|
||||
.venv
|
||||
env/
|
||||
venv/
|
||||
ENV/
|
||||
env.bak/
|
||||
venv.bak/
|
||||
|
||||
# Spyder project settings
|
||||
.spyderproject
|
||||
.spyproject
|
||||
|
||||
# Rope project settings
|
||||
.ropeproject
|
||||
|
||||
# mkdocs documentation
|
||||
/site
|
||||
|
||||
# mypy
|
||||
.mypy_cache/
|
||||
.dmypy.json
|
||||
dmypy.json
|
||||
|
||||
# Pyre type checker
|
||||
.pyre/
|
||||
|
||||
# pytype static type analyzer
|
||||
.pytype/
|
||||
|
||||
# Cython debug symbols
|
||||
cython_debug/
|
||||
|
||||
### VisualStudioCode ###
|
||||
.vscode/*
|
||||
!.vscode/settings.json
|
||||
!.vscode/tasks.json
|
||||
!.vscode/launch.json
|
||||
!.vscode/extensions.json
|
||||
*.code-workspace
|
||||
|
||||
# Local History for Visual Studio Code
|
||||
.history/
|
||||
|
||||
### VisualStudioCode Patch ###
|
||||
# Ignore all local history of files
|
||||
.history
|
||||
.ionide
|
||||
|
||||
# End of https://www.toptal.com/developers/gitignore/api/python,visualstudiocode
|
||||
|
||||
# Custom
|
||||
dev.env
|
||||
prod.env
|
||||
alembic.dev.ini
|
||||
alembic.prod.ini
|
||||
.db/
|
||||
.venv/
|
||||
.esd/
|
||||
.secrets/
|
||||
|
|
@ -0,0 +1,30 @@
|
|||
# Crawler: Ethereum Signature Database
|
||||
|
||||
This crawler retrieves Ethereum function signatures from the Ethereum Signature Database at
|
||||
[https://4byte.directory](https://4byte.directory).
|
||||
|
||||
### Installation
|
||||
|
||||
(Use Python 3)
|
||||
|
||||
```bash
|
||||
pip install -r requirements.txt
|
||||
```
|
||||
|
||||
### Database access
|
||||
|
||||
Make sure that the `EXPLORATION_DB_URI` environment variable is set as a Postgres connection string.
|
||||
|
||||
For a sample, view [`sample.env`](./sample.env).
|
||||
|
||||
### Crawling ESD function signatures
|
||||
|
||||
```bash
|
||||
python esd.py --interval 0.3 functions
|
||||
```
|
||||
|
||||
### Crawling ESD event signatures
|
||||
|
||||
```bash
|
||||
python esd.py --interval 0.3 events
|
||||
```
|
||||
|
|
@ -0,0 +1,68 @@
|
|||
import argparse
|
||||
import sys
|
||||
import time
|
||||
from typing import Optional, Union
|
||||
|
||||
from moonstreamdb.db import yield_db_session_ctx
|
||||
from moonstreamdb.models import ESDEventSignature, ESDFunctionSignature
|
||||
from sqlalchemy.orm import Session
|
||||
import requests
|
||||
|
||||
CRAWL_URLS = {
|
||||
"functions": "https://www.4byte.directory/api/v1/signatures/",
|
||||
"events": "https://www.4byte.directory/api/v1/event-signatures/",
|
||||
}
|
||||
|
||||
DB_MODELS = {
|
||||
"functions": ESDFunctionSignature,
|
||||
"events": ESDEventSignature,
|
||||
}
|
||||
|
||||
def crawl_step(db_session: Session, crawl_url: str, db_model: Union[ESDEventSignature, ESDFunctionSignature]) -> Optional[str]:
|
||||
attempt = 0
|
||||
current_interval = 2
|
||||
success = False
|
||||
|
||||
response: Optional[requests.Response] = None
|
||||
while (not success) and attempt < 3:
|
||||
attempt += 1
|
||||
try:
|
||||
response = requests.get(crawl_url)
|
||||
response.raise_for_status()
|
||||
success = True
|
||||
except:
|
||||
current_interval *= 2
|
||||
time.sleep(current_interval)
|
||||
|
||||
if response is None:
|
||||
print(f"Could not process URL: {crawl_url}", file=sys.stderr)
|
||||
return None
|
||||
|
||||
page = response.json()
|
||||
results = page.get("results", [])
|
||||
|
||||
rows = [db_model(id=row.get("id"), text_signature=row.get("text_signature"), hex_signature=row.get("hex_signature"), created_at=row.get("created_at")) for row in results]
|
||||
db_session.bulk_save_objects(rows)
|
||||
db_session.commit()
|
||||
|
||||
return page.get("next")
|
||||
|
||||
def crawl(crawl_type: str, interval: float) -> None:
|
||||
crawl_url: Optional[str] = CRAWL_URLS[crawl_type]
|
||||
db_model = DB_MODELS[crawl_type]
|
||||
with yield_db_session_ctx() as db_session:
|
||||
while crawl_url is not None:
|
||||
print(f"Crawling: {crawl_url}")
|
||||
crawl_url = crawl_step(db_session, crawl_url, db_model)
|
||||
time.sleep(interval)
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Crawls function and event signatures from the Ethereum Signature Database (https://www.4byte.directory/)")
|
||||
parser.add_argument("crawl_type", choices=CRAWL_URLS, help="Specifies whether to crawl function signatures or event signatures")
|
||||
parser.add_argument("--interval", type=float, default=0.1, help="Number of seconds to wait between requests to the Ethereum Signature Database API")
|
||||
args = parser.parse_args()
|
||||
|
||||
crawl(args.crawl_type, args.interval)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
alembic==1.6.5
|
||||
certifi==2021.5.30
|
||||
charset-normalizer==2.0.3
|
||||
greenlet==1.1.0
|
||||
idna==3.2
|
||||
Mako==1.1.4
|
||||
MarkupSafe==2.0.1
|
||||
-e git+ssh://git@github.com/bugout-dev/moonstock.git@8acebb7c8a1872cd0a9c2b663f86be3877a20636#egg=moonstreamdb&subdirectory=db
|
||||
psycopg2-binary==2.9.1
|
||||
python-dateutil==2.8.2
|
||||
python-editor==1.0.4
|
||||
requests==2.26.0
|
||||
six==1.16.0
|
||||
SQLAlchemy==1.4.22
|
||||
urllib3==1.26.6
|
||||
|
|
@ -0,0 +1 @@
|
|||
export EXPLORATION_DB_URI="postgresql://<username>:<password>@<db_host>:<db_port>/<db_name>"
|
||||
|
|
@ -167,3 +167,5 @@ alembic.dev.ini
|
|||
alembic.prod.ini
|
||||
.db/
|
||||
.venv/
|
||||
.secrets/
|
||||
.moonstreamdb
|
||||
|
|
|
|||
39
db/README.md
39
db/README.md
|
|
@ -1 +1,38 @@
|
|||
# moonstream db
|
||||
# moonstream db
|
||||
|
||||
### Setting up moonstreamdb
|
||||
|
||||
Copy `sample.env` to a new file and set the environment variables to appropriate values. This new file
|
||||
should be sourced every time you want to access the database with the `moonstreamdb` application or any
|
||||
dependents.
|
||||
|
||||
To be able to run migrations, copy [`alembic.sample.ini`](./alembic.sample.ini) to a separate file
|
||||
(e.g. `./secrets/alembic.dev.ini`) and modify the `sqlalchemy.url` setting in the new file to point
|
||||
at your database.
|
||||
|
||||
Make sure your database is at the latest alembic migration:
|
||||
|
||||
```bash
|
||||
alembic -c ./secrets/alembic.dev.ini upgrade head
|
||||
```
|
||||
|
||||
### Adding a new table to database
|
||||
|
||||
Add SQLAlchemy model in [`moonstreamdb/models.py`](./moonstreamdb/models.py)
|
||||
|
||||
Import new model and add tablename to whitelist in [`alembic/env.py`](.alembic/env.py)
|
||||
|
||||
Create a migration:
|
||||
|
||||
```bash
|
||||
alembic -c <alembic config file> revision -m "<revision message>" --autogenerate
|
||||
```
|
||||
|
||||
Always check the autogenerated file to make sure that it isn't performing any actions that you don't want it to.
|
||||
A good policy is to delete any operations that don't touch the tables that you created.
|
||||
|
||||
Then run the migration:
|
||||
|
||||
```bash
|
||||
alembic -c <alembic config file> upgrade head
|
||||
```
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ target_metadata = ExplorationBase.metadata
|
|||
# can be acquired:
|
||||
# my_important_option = config.get_main_option("my_important_option")
|
||||
# ... etc.
|
||||
from moonstreamdb.models import EthereumBlock, EthereumTransaction, EthereumPendingTransaction
|
||||
from moonstreamdb.models import EthereumBlock, EthereumTransaction, EthereumPendingTransaction, ESDEventSignature, ESDFunctionSignature
|
||||
|
||||
|
||||
def include_symbol(tablename, schema):
|
||||
|
|
@ -33,6 +33,8 @@ def include_symbol(tablename, schema):
|
|||
EthereumBlock.__tablename__,
|
||||
EthereumTransaction.__tablename__,
|
||||
EthereumPendingTransaction.__tablename__,
|
||||
ESDEventSignature.__tablename__,
|
||||
ESDFunctionSignature.__tablename__,
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,42 @@
|
|||
"""Added tables to store data from Ethereum Signature Database
|
||||
|
||||
Revision ID: 1e33c3d07306
|
||||
Revises: aa903a90b8bf
|
||||
Create Date: 2021-07-27 00:04:31.042487
|
||||
|
||||
"""
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = '1e33c3d07306'
|
||||
down_revision = 'aa903a90b8bf'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.create_table('esd_event_signatures',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('text_signature', sa.Text(), nullable=False),
|
||||
sa.Column('hex_signature', sa.VARCHAR(length=66), nullable=False),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text("TIMEZONE('utc', statement_timestamp())"), nullable=False),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_esd_event_signatures'))
|
||||
)
|
||||
op.create_table('esd_function_signatures',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('text_signature', sa.Text(), nullable=False),
|
||||
sa.Column('hex_signature', sa.VARCHAR(length=10), nullable=False),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text("TIMEZONE('utc', statement_timestamp())"), nullable=False),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_esd_function_signatures'))
|
||||
)
|
||||
# ### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade():
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.drop_table('esd_function_signatures')
|
||||
op.drop_table('esd_event_signatures')
|
||||
# ### end Alembic commands ###
|
||||
|
|
@ -129,3 +129,33 @@ class EthereumPendingTransaction(Base): # type: ignore
|
|||
indexed_at = Column(
|
||||
DateTime(timezone=True), server_default=utcnow(), nullable=False
|
||||
)
|
||||
|
||||
|
||||
class ESDFunctionSignature(Base):
|
||||
"""
|
||||
Function signature from Ethereum Signature Database.
|
||||
"""
|
||||
|
||||
__tablename__ = "esd_function_signatures"
|
||||
|
||||
id = Column(Integer, primary_key=True)
|
||||
text_signature = Column(Text, nullable=False)
|
||||
hex_signature = Column(VARCHAR(10), nullable=False)
|
||||
created_at = Column(
|
||||
DateTime(timezone=True), server_default=utcnow(), nullable=False
|
||||
)
|
||||
|
||||
|
||||
class ESDEventSignature(Base):
|
||||
"""
|
||||
Function signature from Ethereum Signature Database.
|
||||
"""
|
||||
|
||||
__tablename__ = "esd_event_signatures"
|
||||
|
||||
id = Column(Integer, primary_key=True)
|
||||
text_signature = Column(Text, nullable=False)
|
||||
hex_signature = Column(VARCHAR(66), nullable=False)
|
||||
created_at = Column(
|
||||
DateTime(timezone=True), server_default=utcnow(), nullable=False
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,2 +1,2 @@
|
|||
export MOONSTREAM_DB_URI="<database_uri>"
|
||||
export MOONSTREAM_DB_URI="postgresql://<username>:<password>@<db_host>:<db_port>/<db_name>"
|
||||
export MOONSTREAM_POOL_SIZE=0
|
||||
|
|
|
|||
|
|
@ -31,6 +31,11 @@ prod.env
|
|||
.env.production
|
||||
dev.env.local
|
||||
.env.local
|
||||
.env.dev
|
||||
.env.development
|
||||
.env.development.local
|
||||
.env.test.local
|
||||
.env.production.local
|
||||
.secrets/
|
||||
|
||||
# vercel
|
||||
.vercel
|
||||
|
||||
|
|
|
|||
Ładowanie…
Reference in New Issue