mirror
/
moonstream
kopia lustrzana https://github.com/bugout-dev/moonstream
1
0
Forkuj 0
Kod Zgłoszenia Projekty Wydania Wiki Aktywność

Humbug reporter for moonstream backend

pull/220/head
kompotkot 2021-09-01 12:03:44 +00:00
rodzic e542ab44a8
commit 8d16c68a10
14 zmienionych plików z 166 dodań i 65 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

7
backend/moonstream/__init__.py
Wyświetl plik

@ -0,0 +1,7 @@
from .reporter import reporter
from .version import MOONSTREAM_VERSION
# Reporting
reporter.tags.append(f"version:{MOONSTREAM_VERSION}")
reporter.system_report(publish=True)
reporter.setup_excepthook(publish=True)

9
backend/moonstream/actions.py
Wyświetl plik

@ -1,16 +1,18 @@
import json
import logging
from typing import Dict, Any, List, Optional
from typing import Optional
from enum import Enum
import boto3 # type: ignore
from moonstreamdb.models import (
EthereumAddress,
EthereumLabel,
)
from sqlalchemy import text
from sqlalchemy.orm import Session, query, query_expression
from sqlalchemy.orm import Session
from . import data
from .reporter import reporter
from .settings import ETHERSCAN_SMARTCONTRACTS_BUCKET
logger = logging.getLogger(__name__)
@ -46,8 +48,9 @@ def get_contract_source_info(
abi=obj_data["ABI"],
)
return contract_source_info
except:
except Exception as e:
logger.error(f"Failed to load smart contract {object_uri}")
reporter.error_report(e)
return None

2
backend/moonstream/middleware.py
Wyświetl plik

@ -6,6 +6,7 @@ from bugout.exceptions import BugoutResponseException
from starlette.middleware.base import BaseHTTPMiddleware
from fastapi import Request, Response
from .reporter import reporter
from .settings import MOONSTREAM_APPLICATION_ID, bugout_client as bc
logger = logging.getLogger(__name__)
@ -61,6 +62,7 @@ class BroodAuthMiddleware(BaseHTTPMiddleware):
return Response(status_code=e.status_code, content=e.detail)
except Exception as e:
logger.error(f"Error processing Brood response: {str(e)}")
reporter.error_report(e)
return Response(status_code=500, content="Internal server error")
request.state.user = user

15
backend/moonstream/providers/__init__.py
Wyświetl plik

@ -39,6 +39,13 @@ from ..stream_queries import StreamQuery
logger = logging.getLogger(__name__)
logger.setLevel(logging.WARN)
class ReceivingEventsException(Exception):
"""
Raised when error occurs during receiving events from provider.
"""
event_providers: Dict[str, Any] = {
ethereum_blockchain.event_type: ethereum_blockchain,
bugout.whalewatch_provider.event_type: bugout.whalewatch_provider,
@ -91,7 +98,7 @@ def get_events(
f"Error receiving events from provider: {provider_name}:\n{repr(e)}"
)
else:
raise e
raise ReceivingEventsException(e)
events = [event for _, event_list in results.values() for event in event_list]
if sort_events:
@ -149,7 +156,7 @@ def latest_events(
f"Error receiving events from provider: {provider_name}:\n{repr(e)}"
)
else:
raise e
raise ReceivingEventsException(e)
events = [event for event_list in results.values() for event in event_list]
if sort_events:
@ -202,7 +209,7 @@ def next_event(
f"Error receiving events from provider: {provider_name}:\n{repr(e)}"
)
else:
raise e
raise ReceivingEventsException(e)
event: Optional[data.Event] = None
for candidate in results.values():
@ -258,7 +265,7 @@ def previous_event(
f"Error receiving events from provider: {provider_name}:\n{repr(e)}"
)
else:
raise e
raise ReceivingEventsException(e)
event: Optional[data.Event] = None
for candidate in results.values():

18
backend/moonstream/reporter.py 100644
Wyświetl plik

@ -0,0 +1,18 @@
import uuid
from humbug.consent import HumbugConsent
from humbug.report import HumbugReporter
from .settings import BUGOUT_HUMBUG_TOKEN
session_id = str(uuid.uuid4())
client_id = "moonstream-backend"
reporter = HumbugReporter(
name="moonstream",
consent=HumbugConsent(True),
client_id=client_id,
session_id=session_id,
bugout_token=BUGOUT_HUMBUG_TOKEN,
tags=[],
)

2
backend/moonstream/routes/address_info.py
Wyświetl plik

@ -11,6 +11,7 @@ from sqlalchemy.orm import Session
from .. import actions
from .. import data
from ..middleware import BroodAuthMiddleware
from ..reporter import reporter
from ..settings import DOCS_TARGET_PATH, ORIGINS, DOCS_PATHS
from ..version import MOONSTREAM_VERSION
@ -82,6 +83,7 @@ async def addresses_labels_bulk_handler(
)
except Exception as err:
logger.error(f"Unable to get info about Ethereum addresses {err}")
reporter.error_report(err)
raise HTTPException(status_code=500)
return addresses_response

130
backend/moonstream/routes/streams.py
Wyświetl plik

@ -14,12 +14,14 @@ from sqlalchemy.orm import Session
from .. import data
from ..middleware import BroodAuthMiddleware
from ..providers import (
ReceivingEventsException,
event_providers,
get_events,
latest_events,
next_event,
previous_event,
)
from ..reporter import reporter
from ..settings import (
DOCS_TARGET_PATH,
MOONSTREAM_ADMIN_ACCESS_TOKEN,
@ -121,17 +123,27 @@ async def stream_handler(
if q.strip() != "":
query = stream_queries.parse_query_string(q)
_, events = get_events(
db_session,
bc,
MOONSTREAM_DATA_JOURNAL_ID,
MOONSTREAM_ADMIN_ACCESS_TOKEN,
stream_boundary,
query,
user_subscriptions,
result_timeout=10.0,
raise_on_error=True,
)
try:
_, events = get_events(
db_session,
bc,
MOONSTREAM_DATA_JOURNAL_ID,
MOONSTREAM_ADMIN_ACCESS_TOKEN,
stream_boundary,
query,
user_subscriptions,
result_timeout=10.0,
raise_on_error=True,
)
except ReceivingEventsException as e:
logger.error("Error receiving events from provider")
reporter.error_report(e)
raise HTTPException(status_code=500)
except Exception as e:
logger.error("Unable to get events")
reporter.error_report(e)
raise HTTPException(status_code=500)
response = data.GetEventsResponse(stream_boundary=stream_boundary, events=events)
return response
@ -155,18 +167,28 @@ async def latest_events_handler(
if q.strip() != "":
query = stream_queries.parse_query_string(q)
events = latest_events(
db_session,
bc,
MOONSTREAM_DATA_JOURNAL_ID,
MOONSTREAM_ADMIN_ACCESS_TOKEN,
query,
1,
user_subscriptions,
result_timeout=6.0,
raise_on_error=True,
sort_events=True,
)
try:
events = latest_events(
db_session,
bc,
MOONSTREAM_DATA_JOURNAL_ID,
MOONSTREAM_ADMIN_ACCESS_TOKEN,
query,
1,
user_subscriptions,
result_timeout=6.0,
raise_on_error=True,
sort_events=True,
)
except ReceivingEventsException as e:
logger.error("Error receiving events from provider")
reporter.error_report(e)
raise HTTPException(status_code=500)
except Exception as e:
logger.error("Unable to get latest events")
reporter.error_report(e)
raise HTTPException(status_code=500)
return events
@ -203,17 +225,26 @@ async def next_event_handler(
if q.strip() != "":
query = stream_queries.parse_query_string(q)
event = next_event(
db_session,
bc,
MOONSTREAM_DATA_JOURNAL_ID,
MOONSTREAM_ADMIN_ACCESS_TOKEN,
stream_boundary,
query,
user_subscriptions,
result_timeout=6.0,
raise_on_error=True,
)
try:
event = next_event(
db_session,
bc,
MOONSTREAM_DATA_JOURNAL_ID,
MOONSTREAM_ADMIN_ACCESS_TOKEN,
stream_boundary,
query,
user_subscriptions,
result_timeout=6.0,
raise_on_error=True,
)
except ReceivingEventsException as e:
logger.error("Error receiving events from provider")
reporter.error_report(e)
raise HTTPException(status_code=500)
except Exception as e:
logger.error("Unable to get next events")
reporter.error_report(e)
raise HTTPException(status_code=500)
return event
@ -251,16 +282,25 @@ async def previous_event_handler(
if q.strip() != "":
query = stream_queries.parse_query_string(q)
event = previous_event(
db_session,
bc,
MOONSTREAM_DATA_JOURNAL_ID,
MOONSTREAM_ADMIN_ACCESS_TOKEN,
stream_boundary,
query,
user_subscriptions,
result_timeout=6.0,
raise_on_error=True,
)
try:
event = previous_event(
db_session,
bc,
MOONSTREAM_DATA_JOURNAL_ID,
MOONSTREAM_ADMIN_ACCESS_TOKEN,
stream_boundary,
query,
user_subscriptions,
result_timeout=6.0,
raise_on_error=True,
)
except ReceivingEventsException as e:
logger.error("Error receiving events from provider")
reporter.error_report(e)
raise HTTPException(status_code=500)
except Exception as e:
logger.error("Unable to get previous events")
reporter.error_report(e)
raise HTTPException(status_code=500)
return event

19
backend/moonstream/routes/subscriptions.py
Wyświetl plik

@ -12,6 +12,7 @@ from fastapi.middleware.cors import CORSMiddleware
from ..admin import subscription_types
from .. import data
from ..middleware import BroodAuthMiddleware
from ..reporter import reporter
from ..settings import (
DOCS_TARGET_PATH,
DOCS_PATHS,
@ -100,8 +101,8 @@ async def add_subscription_handler(
resource_data=resource_data,
)
except Exception as e:
logger.error("Error creating subscription resource:")
logger.error(e)
logger.error(f"Error creating subscription resource: {str(e)}")
reporter.error_report(e)
raise HTTPException(status_code=500)
return data.SubscriptionResourceData(
@ -123,13 +124,14 @@ async def delete_subscription_handler(request: Request, subscription_id: str):
"""
Delete subscriptions.
"""
token = request.state.token
try:
deleted_resource = bc.delete_resource(token=token, resource_id=subscription_id)
except BugoutResponseException as e:
raise HTTPException(status_code=e.status_code, detail=e.detail)
except Exception as e:
logger.error(f"Error deleting subscription: {str(e)}")
reporter.error_report(e)
raise HTTPException(status_code=500)
return data.SubscriptionResourceData(
@ -156,9 +158,9 @@ async def get_subscriptions_handler(request: Request) -> data.SubscriptionsListR
resources: BugoutResources = bc.list_resources(token=token, params=params)
except Exception as e:
logger.error(
f"Error listing subscriptions for user ({request.user.id}) with token ({request.state.token})"
f"Error listing subscriptions for user ({request.user.id}) with token ({request.state.token}), error: {str(e)}"
)
logger.error(e)
reporter.error_report(e)
raise HTTPException(status_code=500)
return data.SubscriptionsListResponse(
@ -190,7 +192,6 @@ async def update_subscriptions_handler(
"""
Get user's subscriptions.
"""
token = request.state.token
update = {}
@ -212,6 +213,8 @@ async def update_subscriptions_handler(
except BugoutResponseException as e:
raise HTTPException(status_code=e.status_code, detail=e.detail)
except Exception as e:
logger.error(f"Error getting user subscriptions: {str(e)}")
reporter.error_report(e)
raise HTTPException(status_code=500)
return data.SubscriptionResourceData(
@ -239,8 +242,8 @@ async def list_subscription_types() -> data.SubscriptionTypesListResponse:
for resource in response.resources
]
except Exception as e:
logger.error("Error reading subscription types from Brood API:")
logger.error(e)
logger.error(f"Error reading subscription types from Brood API: {str(e)}")
reporter.error_report(e)
raise HTTPException(status_code=500)
return data.SubscriptionTypesListResponse(subscription_types=results)

5
backend/moonstream/routes/txinfo.py
Wyświetl plik

@ -6,9 +6,7 @@ transactions, etc.) with side information and return objects that are better sui
end users.
"""
import logging
from typing import Dict, Optional
from sqlalchemy.sql.expression import true
from typing import Dict
from fastapi import FastAPI, Depends
from fastapi.middleware.cors import CORSMiddleware
@ -54,6 +52,7 @@ app.add_middleware(BroodAuthMiddleware, whitelist=whitelist_paths)
# TODO(zomglings): Factor out the enrichment logic into a separate action, because it may be useful
# independently from serving API calls (e.g. data processing).
# TODO(kompotkot): Re-organize function to be able handle each steps with exceptions.
@app.post(
"/ethereum_blockchain",
tags=["txinfo"],

8
backend/moonstream/routes/users.py
Wyświetl plik

@ -16,6 +16,7 @@ from fastapi import (
from fastapi.middleware.cors import CORSMiddleware
from ..middleware import BroodAuthMiddleware
from ..reporter import reporter
from ..settings import (
MOONSTREAM_APPLICATION_ID,
DOCS_TARGET_PATH,
@ -77,6 +78,7 @@ async def create_user_handler(
except BugoutResponseException as e:
raise HTTPException(status_code=e.status_code, detail=e.detail)
except Exception as e:
reporter.error_report(e)
raise HTTPException(status_code=500)
return user
@ -94,6 +96,7 @@ async def restore_password_handler(email: str = Form(...)) -> Dict[str, Any]:
except BugoutResponseException as e:
raise HTTPException(status_code=e.status_code, detail=e.detail)
except Exception as e:
reporter.error_report(e)
raise HTTPException(status_code=500)
return response
@ -107,6 +110,7 @@ async def reset_password_handler(
except BugoutResponseException as e:
raise HTTPException(status_code=e.status_code, detail=e.detail)
except Exception as e:
reporter.error_report(e)
raise HTTPException(status_code=500)
return response
@ -123,6 +127,7 @@ async def change_password_handler(
except BugoutResponseException as e:
raise HTTPException(status_code=e.status_code, detail=e.detail)
except Exception as e:
reporter.error_report(e)
raise HTTPException(status_code=500)
return user
@ -138,6 +143,7 @@ async def delete_user_handler(
except BugoutResponseException as e:
raise HTTPException(status_code=e.status_code, detail=e.detail)
except Exception as e:
reporter.error_report(e)
raise HTTPException(status_code=500)
return user
@ -157,6 +163,7 @@ async def login_handler(
status_code=e.status_code, detail=f"Error from Brood API: {e.detail}"
)
except Exception as e:
reporter.error_report(e)
raise HTTPException(status_code=500)
return token
@ -169,5 +176,6 @@ async def logout_handler(request: Request) -> uuid.UUID:
except BugoutResponseException as e:
raise HTTPException(status_code=e.status_code, detail=e.detail)
except Exception as e:
reporter.error_report(e)
raise HTTPException(status_code=500)
return token_id

2
backend/moonstream/settings.py
Wyświetl plik

@ -9,6 +9,8 @@ bugout_client = Bugout(brood_api_url=BUGOUT_BROOD_URL, spire_api_url=BUGOUT_SPIR
BUGOUT_REQUEST_TIMEOUT_SECONDS = 5
BUGOUT_HUMBUG_TOKEN = os.environ.get("BUGOUT_HUMBUG_TOKEN")
# Default value is "" instead of None so that mypy understands that MOONSTREAM_APPLICATION_ID is a string
MOONSTREAM_APPLICATION_ID = os.environ.get("MOONSTREAM_APPLICATION_ID", "")
if MOONSTREAM_APPLICATION_ID == "":

1
backend/requirements.txt
Wyświetl plik

@ -11,6 +11,7 @@ fastapi==0.66.0
h11==0.12.0
idna==3.2
jmespath==0.10.0
humbug==0.2.7
-e git+https://git@github.com/bugout-dev/moonstream.git@94135b054cabb9dc11b0a2406431619279979469#egg=moonstreamdb&subdirectory=db
mypy==0.910
mypy-extensions==0.4.3

3
backend/sample.env
Wyświetl plik

@ -5,6 +5,7 @@ export MOONSTREAM_DATA_JOURNAL_ID="<bugout_journal_id_to_store_blockchain_data>"
export MOONSTREAM_DB_URI="postgresql://<username>:<password>@<db_host>:<db_port>/<db_name>"
export MOONSTREAM_POOL_SIZE=0
export MOONSTREAM_ADMIN_ACCESS_TOKEN="<Access token to application resources>"
export AWS_S3_SMARTCONTRACT_BUCKET=""
export AWS_S3_SMARTCONTRACT_BUCKET="<AWS S3 bucket to store smart contracts>"
export BUGOUT_BROOD_URL="https://auth.bugout.dev"
export BUGOUT_SPIRE_URL="https://spire.bugout.dev"
export BUGOUT_HUMBUG_TOKEN="<Bugout Humbug token for crash reports>"

10
backend/setup.py
Wyświetl plik

@ -10,7 +10,15 @@ setup(
name="moonstream",
version=MOONSTREAM_VERSION,
packages=find_packages(),
install_requires=["boto3", "bugout >= 0.1.17", "fastapi", "python-dateutil", "uvicorn", "types-python-dateutil"],
install_requires=[
"boto3",
"bugout >= 0.1.17",
"fastapi",
"humbug>=0.2.7",
"python-dateutil",
"uvicorn",
"types-python-dateutil",
],
extras_require={
"dev": ["black", "mypy"],
"distribute": ["setuptools", "twine", "wheel"],