diff --git a/backend/moonstream/__init__.py b/backend/moonstream/__init__.py index e69de29b..5df6f535 100644 --- a/backend/moonstream/__init__.py +++ b/backend/moonstream/__init__.py @@ -0,0 +1,7 @@ +from .reporter import reporter +from .version import MOONSTREAM_VERSION + +# Reporting +reporter.tags.append(f"version:{MOONSTREAM_VERSION}") +reporter.system_report(publish=True) +reporter.setup_excepthook(publish=True) diff --git a/backend/moonstream/actions.py b/backend/moonstream/actions.py index 65ef735b..7b75b5e1 100644 --- a/backend/moonstream/actions.py +++ b/backend/moonstream/actions.py @@ -1,16 +1,18 @@ import json import logging -from typing import Dict, Any, List, Optional +from typing import Optional from enum import Enum + import boto3 # type: ignore from moonstreamdb.models import ( EthereumAddress, EthereumLabel, ) from sqlalchemy import text -from sqlalchemy.orm import Session, query, query_expression +from sqlalchemy.orm import Session from . import data +from .reporter import reporter from .settings import ETHERSCAN_SMARTCONTRACTS_BUCKET import uuid from bugout.data import BugoutResource @@ -53,8 +55,9 @@ def get_contract_source_info( abi=obj_data["ABI"], ) return contract_source_info - except: + except Exception as e: logger.error(f"Failed to load smart contract {object_uri}") + reporter.error_report(e) return None diff --git a/backend/moonstream/middleware.py b/backend/moonstream/middleware.py index e2a15175..5e089aa6 100644 --- a/backend/moonstream/middleware.py +++ b/backend/moonstream/middleware.py @@ -1,16 +1,36 @@ import logging -from typing import Awaitable, Callable, Dict, Optional +from typing import Any, Awaitable, Callable, Dict, Optional from bugout.data import BugoutUser from bugout.exceptions import BugoutResponseException +from fastapi import HTTPException, Request, Response +from starlette.background import BackgroundTask from starlette.middleware.base import BaseHTTPMiddleware -from fastapi import Request, Response +from .reporter import reporter from .settings import MOONSTREAM_APPLICATION_ID, bugout_client as bc logger = logging.getLogger(__name__) +class MoonstreamHTTPException(HTTPException): + """ + Extended HTTPException to handle 500 Internal server errors + and send crash reports. + """ + + def __init__( + self, + status_code: int, + detail: Any = None, + headers: Optional[Dict[str, Any]] = None, + internal_error: Exception = None, + ): + super().__init__(status_code, detail, headers) + if internal_error is not None: + reporter.error_report(internal_error) + + class BroodAuthMiddleware(BaseHTTPMiddleware): """ Checks the authorization header on the request. If it represents a verified Brood user, @@ -61,6 +81,7 @@ class BroodAuthMiddleware(BaseHTTPMiddleware): return Response(status_code=e.status_code, content=e.detail) except Exception as e: logger.error(f"Error processing Brood response: {str(e)}") + reporter.error_report(e) return Response(status_code=500, content="Internal server error") request.state.user = user diff --git a/backend/moonstream/providers/__init__.py b/backend/moonstream/providers/__init__.py index fbb2ec7d..f884fe1b 100644 --- a/backend/moonstream/providers/__init__.py +++ b/backend/moonstream/providers/__init__.py @@ -39,6 +39,13 @@ from ..stream_queries import StreamQuery logger = logging.getLogger(__name__) logger.setLevel(logging.WARN) + +class ReceivingEventsException(Exception): + """ + Raised when error occurs during receiving events from provider. + """ + + event_providers: Dict[str, Any] = { ethereum_blockchain.event_type: ethereum_blockchain, bugout.whalewatch_provider.event_type: bugout.whalewatch_provider, @@ -91,7 +98,7 @@ def get_events( f"Error receiving events from provider: {provider_name}:\n{repr(e)}" ) else: - raise e + raise ReceivingEventsException(e) events = [event for _, event_list in results.values() for event in event_list] if sort_events: @@ -149,7 +156,7 @@ def latest_events( f"Error receiving events from provider: {provider_name}:\n{repr(e)}" ) else: - raise e + raise ReceivingEventsException(e) events = [event for event_list in results.values() for event in event_list] if sort_events: @@ -202,7 +209,7 @@ def next_event( f"Error receiving events from provider: {provider_name}:\n{repr(e)}" ) else: - raise e + raise ReceivingEventsException(e) event: Optional[data.Event] = None for candidate in results.values(): @@ -258,7 +265,7 @@ def previous_event( f"Error receiving events from provider: {provider_name}:\n{repr(e)}" ) else: - raise e + raise ReceivingEventsException(e) event: Optional[data.Event] = None for candidate in results.values(): diff --git a/backend/moonstream/reporter.py b/backend/moonstream/reporter.py new file mode 100644 index 00000000..1ba0997b --- /dev/null +++ b/backend/moonstream/reporter.py @@ -0,0 +1,18 @@ +import uuid + +from humbug.consent import HumbugConsent +from humbug.report import HumbugReporter + +from .settings import HUMBUG_REPORTER_BACKEND_TOKEN + +session_id = str(uuid.uuid4()) +client_id = "moonstream-backend" + +reporter = HumbugReporter( + name="moonstream", + consent=HumbugConsent(True), + client_id=client_id, + session_id=session_id, + bugout_token=HUMBUG_REPORTER_BACKEND_TOKEN, + tags=[], +) diff --git a/backend/moonstream/routes/address_info.py b/backend/moonstream/routes/address_info.py index 2b95bb92..c985c683 100644 --- a/backend/moonstream/routes/address_info.py +++ b/backend/moonstream/routes/address_info.py @@ -3,14 +3,14 @@ from typing import Dict, List, Optional from sqlalchemy.sql.expression import true -from fastapi import FastAPI, Depends, Query, HTTPException +from fastapi import FastAPI, Depends, Query from fastapi.middleware.cors import CORSMiddleware from moonstreamdb.db import yield_db_session from sqlalchemy.orm import Session from .. import actions from .. import data -from ..middleware import BroodAuthMiddleware +from ..middleware import BroodAuthMiddleware, MoonstreamHTTPException from ..settings import DOCS_TARGET_PATH, ORIGINS, DOCS_PATHS from ..version import MOONSTREAM_VERSION @@ -73,15 +73,15 @@ async def addresses_labels_bulk_handler( about known addresses. """ if limit > 100: - raise HTTPException( + raise MoonstreamHTTPException( status_code=406, detail="The limit cannot exceed 100 addresses" ) try: addresses_response = actions.get_address_labels( db_session=db_session, start=start, limit=limit, addresses=addresses ) - except Exception as err: - logger.error(f"Unable to get info about Ethereum addresses {err}") - raise HTTPException(status_code=500) + except Exception as e: + logger.error(f"Unable to get info about Ethereum addresses {e}") + raise MoonstreamHTTPException(status_code=500, internal_error=e) return addresses_response diff --git a/backend/moonstream/routes/streams.py b/backend/moonstream/routes/streams.py index be46bc58..10d0426c 100644 --- a/backend/moonstream/routes/streams.py +++ b/backend/moonstream/routes/streams.py @@ -5,15 +5,16 @@ import logging from typing import Dict, List, Optional from bugout.data import BugoutResource -from fastapi import FastAPI, HTTPException, Request, Query, Depends +from fastapi import FastAPI, Request, Query, Depends from fastapi.middleware.cors import CORSMiddleware from moonstreamdb import db from sqlalchemy.orm import Session from .. import data -from ..middleware import BroodAuthMiddleware +from ..middleware import BroodAuthMiddleware, MoonstreamHTTPException from ..providers import ( + ReceivingEventsException, event_providers, get_events, latest_events, @@ -121,17 +122,25 @@ async def stream_handler( if q.strip() != "": query = stream_queries.parse_query_string(q) - _, events = get_events( - db_session, - bc, - MOONSTREAM_DATA_JOURNAL_ID, - MOONSTREAM_ADMIN_ACCESS_TOKEN, - stream_boundary, - query, - user_subscriptions, - result_timeout=10.0, - raise_on_error=True, - ) + try: + _, events = get_events( + db_session, + bc, + MOONSTREAM_DATA_JOURNAL_ID, + MOONSTREAM_ADMIN_ACCESS_TOKEN, + stream_boundary, + query, + user_subscriptions, + result_timeout=10.0, + raise_on_error=True, + ) + except ReceivingEventsException as e: + logger.error("Error receiving events from provider") + raise MoonstreamHTTPException(status_code=500, internal_error=e) + except Exception as e: + logger.error("Unable to get events") + raise MoonstreamHTTPException(status_code=500, internal_error=e) + response = data.GetEventsResponse(stream_boundary=stream_boundary, events=events) return response @@ -155,18 +164,26 @@ async def latest_events_handler( if q.strip() != "": query = stream_queries.parse_query_string(q) - events = latest_events( - db_session, - bc, - MOONSTREAM_DATA_JOURNAL_ID, - MOONSTREAM_ADMIN_ACCESS_TOKEN, - query, - 1, - user_subscriptions, - result_timeout=6.0, - raise_on_error=True, - sort_events=True, - ) + try: + events = latest_events( + db_session, + bc, + MOONSTREAM_DATA_JOURNAL_ID, + MOONSTREAM_ADMIN_ACCESS_TOKEN, + query, + 1, + user_subscriptions, + result_timeout=6.0, + raise_on_error=True, + sort_events=True, + ) + except ReceivingEventsException as e: + logger.error("Error receiving events from provider") + raise MoonstreamHTTPException(status_code=500, internal_error=e) + except Exception as e: + logger.error("Unable to get latest events") + raise MoonstreamHTTPException(status_code=500, internal_error=e) + return events @@ -203,17 +220,24 @@ async def next_event_handler( if q.strip() != "": query = stream_queries.parse_query_string(q) - event = next_event( - db_session, - bc, - MOONSTREAM_DATA_JOURNAL_ID, - MOONSTREAM_ADMIN_ACCESS_TOKEN, - stream_boundary, - query, - user_subscriptions, - result_timeout=6.0, - raise_on_error=True, - ) + try: + event = next_event( + db_session, + bc, + MOONSTREAM_DATA_JOURNAL_ID, + MOONSTREAM_ADMIN_ACCESS_TOKEN, + stream_boundary, + query, + user_subscriptions, + result_timeout=6.0, + raise_on_error=True, + ) + except ReceivingEventsException as e: + logger.error("Error receiving events from provider") + raise MoonstreamHTTPException(status_code=500, internal_error=e) + except Exception as e: + logger.error("Unable to get next events") + raise MoonstreamHTTPException(status_code=500, internal_error=e) return event @@ -251,16 +275,23 @@ async def previous_event_handler( if q.strip() != "": query = stream_queries.parse_query_string(q) - event = previous_event( - db_session, - bc, - MOONSTREAM_DATA_JOURNAL_ID, - MOONSTREAM_ADMIN_ACCESS_TOKEN, - stream_boundary, - query, - user_subscriptions, - result_timeout=6.0, - raise_on_error=True, - ) + try: + event = previous_event( + db_session, + bc, + MOONSTREAM_DATA_JOURNAL_ID, + MOONSTREAM_ADMIN_ACCESS_TOKEN, + stream_boundary, + query, + user_subscriptions, + result_timeout=6.0, + raise_on_error=True, + ) + except ReceivingEventsException as e: + logger.error("Error receiving events from provider") + raise MoonstreamHTTPException(status_code=500, internal_error=e) + except Exception as e: + logger.error("Unable to get previous events") + raise MoonstreamHTTPException(status_code=500, internal_error=e) return event diff --git a/backend/moonstream/routes/subscriptions.py b/backend/moonstream/routes/subscriptions.py index 75c854dc..f86f4686 100644 --- a/backend/moonstream/routes/subscriptions.py +++ b/backend/moonstream/routes/subscriptions.py @@ -6,12 +6,13 @@ from typing import Dict, List, Optional from bugout.data import BugoutResource, BugoutResources from bugout.exceptions import BugoutResponseException -from fastapi import FastAPI, HTTPException, Request, Form +from fastapi import FastAPI, Request, Form from fastapi.middleware.cors import CORSMiddleware from ..admin import subscription_types from .. import data -from ..middleware import BroodAuthMiddleware +from ..middleware import BroodAuthMiddleware, MoonstreamHTTPException +from ..reporter import reporter from ..settings import ( DOCS_TARGET_PATH, DOCS_PATHS, @@ -77,7 +78,7 @@ async def add_subscription_handler( ] if subscription_type_id not in available_subscription_type_ids: - raise HTTPException( + raise MoonstreamHTTPException( status_code=404, detail=f"Invalid subscription type: {subscription_type_id}.", ) @@ -99,10 +100,11 @@ async def add_subscription_handler( application_id=MOONSTREAM_APPLICATION_ID, resource_data=resource_data, ) + except BugoutResponseException as e: + raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) except Exception as e: - logger.error("Error creating subscription resource:") - logger.error(e) - raise HTTPException(status_code=500) + logger.error(f"Error creating subscription resource: {str(e)}") + raise MoonstreamHTTPException(status_code=500, internal_error=e) return data.SubscriptionResourceData( id=str(resource.id), @@ -123,14 +125,14 @@ async def delete_subscription_handler(request: Request, subscription_id: str): """ Delete subscriptions. """ - token = request.state.token try: deleted_resource = bc.delete_resource(token=token, resource_id=subscription_id) except BugoutResponseException as e: - raise HTTPException(status_code=e.status_code, detail=e.detail) + raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) except Exception as e: - raise HTTPException(status_code=500) + logger.error(f"Error deleting subscription: {str(e)}") + raise MoonstreamHTTPException(status_code=500, internal_error=e) return data.SubscriptionResourceData( id=str(deleted_resource.id), @@ -154,12 +156,14 @@ async def get_subscriptions_handler(request: Request) -> data.SubscriptionsListR } try: resources: BugoutResources = bc.list_resources(token=token, params=params) + except BugoutResponseException as e: + raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) except Exception as e: logger.error( - f"Error listing subscriptions for user ({request.user.id}) with token ({request.state.token})" + f"Error listing subscriptions for user ({request.user.id}) with token ({request.state.token}), error: {str(e)}" ) - logger.error(e) - raise HTTPException(status_code=500) + reporter.error_report(e) + raise MoonstreamHTTPException(status_code=500, internal_error=e) return data.SubscriptionsListResponse( subscriptions=[ @@ -190,7 +194,6 @@ async def update_subscriptions_handler( """ Get user's subscriptions. """ - token = request.state.token update = {} @@ -210,9 +213,10 @@ async def update_subscriptions_handler( ).dict(), ) except BugoutResponseException as e: - raise HTTPException(status_code=e.status_code, detail=e.detail) + raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) except Exception as e: - raise HTTPException(status_code=500) + logger.error(f"Error getting user subscriptions: {str(e)}") + raise MoonstreamHTTPException(status_code=500, internal_error=e) return data.SubscriptionResourceData( id=str(resource.id), @@ -238,9 +242,10 @@ async def list_subscription_types() -> data.SubscriptionTypesListResponse: data.SubscriptionTypeResourceData.validate(resource.resource_data) for resource in response.resources ] + except BugoutResponseException as e: + raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) except Exception as e: - logger.error("Error reading subscription types from Brood API:") - logger.error(e) - raise HTTPException(status_code=500) + logger.error(f"Error reading subscription types from Brood API: {str(e)}") + raise MoonstreamHTTPException(status_code=500, internal_error=e) return data.SubscriptionTypesListResponse(subscription_types=results) diff --git a/backend/moonstream/routes/txinfo.py b/backend/moonstream/routes/txinfo.py index 20eec3c7..8ab9a5ec 100644 --- a/backend/moonstream/routes/txinfo.py +++ b/backend/moonstream/routes/txinfo.py @@ -6,9 +6,7 @@ transactions, etc.) with side information and return objects that are better sui end users. """ import logging -from typing import Dict, Optional - -from sqlalchemy.sql.expression import true +from typing import Dict from fastapi import FastAPI, Depends from fastapi.middleware.cors import CORSMiddleware @@ -54,6 +52,7 @@ app.add_middleware(BroodAuthMiddleware, whitelist=whitelist_paths) # TODO(zomglings): Factor out the enrichment logic into a separate action, because it may be useful # independently from serving API calls (e.g. data processing). +# TODO(kompotkot): Re-organize function to be able handle each steps with exceptions. @app.post( "/ethereum_blockchain", tags=["txinfo"], diff --git a/backend/moonstream/routes/users.py b/backend/moonstream/routes/users.py index cd30f75f..19cefc8f 100644 --- a/backend/moonstream/routes/users.py +++ b/backend/moonstream/routes/users.py @@ -7,17 +7,18 @@ import uuid from bugout.data import BugoutToken, BugoutUser, BugoutResource from bugout.exceptions import BugoutResponseException + from fastapi import ( Body, FastAPI, Form, - HTTPException, Request, ) from fastapi.middleware.cors import CORSMiddleware from .. import data -from ..middleware import BroodAuthMiddleware +from ..middleware import BroodAuthMiddleware, MoonstreamHTTPException + from ..settings import ( MOONSTREAM_APPLICATION_ID, DOCS_TARGET_PATH, @@ -79,9 +80,9 @@ async def create_user_handler( application_id=MOONSTREAM_APPLICATION_ID, ) except BugoutResponseException as e: - raise HTTPException(status_code=e.status_code, detail=e.detail) + raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) except Exception as e: - raise HTTPException(status_code=500) + raise MoonstreamHTTPException(status_code=500, internal_error=e) return user @@ -96,9 +97,9 @@ async def restore_password_handler(email: str = Form(...)) -> Dict[str, Any]: try: response = bc.restore_password(email=email) except BugoutResponseException as e: - raise HTTPException(status_code=e.status_code, detail=e.detail) + raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) except Exception as e: - raise HTTPException(status_code=500) + raise MoonstreamHTTPException(status_code=500, internal_error=e) return response @@ -109,9 +110,9 @@ async def reset_password_handler( try: response = bc.reset_password(reset_id=reset_id, new_password=new_password) except BugoutResponseException as e: - raise HTTPException(status_code=e.status_code, detail=e.detail) + raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) except Exception as e: - raise HTTPException(status_code=500) + raise MoonstreamHTTPException(status_code=500, internal_error=e) return response @@ -125,9 +126,9 @@ async def change_password_handler( token=token, current_password=current_password, new_password=new_password ) except BugoutResponseException as e: - raise HTTPException(status_code=e.status_code, detail=e.detail) + raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) except Exception as e: - raise HTTPException(status_code=500) + raise MoonstreamHTTPException(status_code=500, internal_error=e) return user @@ -140,9 +141,9 @@ async def delete_user_handler( try: user = bc.delete_user(token=token, user_id=user.id, password=password) except BugoutResponseException as e: - raise HTTPException(status_code=e.status_code, detail=e.detail) + raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) except Exception as e: - raise HTTPException(status_code=500) + raise MoonstreamHTTPException(status_code=500, internal_error=e) return user @@ -157,11 +158,11 @@ async def login_handler( application_id=MOONSTREAM_APPLICATION_ID, ) except BugoutResponseException as e: - raise HTTPException( + raise MoonstreamHTTPException( status_code=e.status_code, detail=f"Error from Brood API: {e.detail}" ) except Exception as e: - raise HTTPException(status_code=500) + raise MoonstreamHTTPException(status_code=500, internal_error=e) return token @@ -171,9 +172,9 @@ async def logout_handler(request: Request) -> uuid.UUID: try: token_id: uuid.UUID = bc.revoke_token(token=token) except BugoutResponseException as e: - raise HTTPException(status_code=e.status_code, detail=e.detail) + raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) except Exception as e: - raise HTTPException(status_code=500) + raise MoonstreamHTTPException(status_code=500, internal_error=e) return token_id @@ -203,9 +204,9 @@ async def set_onboarding_state( ) except BugoutResponseException as e: - raise HTTPException(status_code=e.status_code, detail=e.detail) + raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) except Exception as e: - raise HTTPException(status_code=500) + raise MoonstreamHTTPException(status_code=500) if ( resource.resource_data.get("is_complete") is None @@ -214,7 +215,7 @@ async def set_onboarding_state( logger.error( f"Resources did not return correct onboarding object. Resource id:{resource.id}" ) - raise HTTPException(status_code=500) + raise MoonstreamHTTPException(status_code=500) result = data.OnboardingState( is_complete=resource.resource_data.get("is_complete", False), @@ -238,10 +239,10 @@ async def get_onboarding_state(request: Request) -> data.OnboardingState: else: resource = create_onboarding_resource(token=token) except BugoutResponseException as e: - raise HTTPException(status_code=e.status_code, detail=e.detail) + raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) except Exception as e: - raise HTTPException(status_code=500) + raise MoonstreamHTTPException(status_code=500) if ( resource.resource_data.get("is_complete") is None @@ -250,7 +251,7 @@ async def get_onboarding_state(request: Request) -> data.OnboardingState: logger.error( f"Resources did not return correct onboarding object. Resource id:{resource.id}" ) - raise HTTPException(status_code=500) + raise MoonstreamHTTPException(status_code=500) result = data.OnboardingState( is_complete=resource.resource_data.get("is_complete", False), steps=resource.resource_data.get("steps", {}), @@ -268,7 +269,7 @@ async def delete_onboarding_state(request: Request) -> data.OnboardingState: timeout=BUGOUT_REQUEST_TIMEOUT_SECONDS, ) if not response.resources: - raise HTTPException(status_code=404, detail="not found") + raise MoonstreamHTTPException(status_code=404, detail="not found") if response.resources: resource: BugoutResource = bc.delete_resource( token=token, @@ -277,9 +278,9 @@ async def delete_onboarding_state(request: Request) -> data.OnboardingState: ) except BugoutResponseException as e: - raise HTTPException(status_code=e.status_code, detail=e.detail) + raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) except Exception as e: - raise HTTPException(status_code=500) + raise MoonstreamHTTPException(status_code=500) if ( resource.resource_data.get("is_complete") is None @@ -288,7 +289,7 @@ async def delete_onboarding_state(request: Request) -> data.OnboardingState: logger.error( f"Resources did not return correct onboarding object. Resource id:{resource.id}" ) - raise HTTPException(status_code=500) + raise MoonstreamHTTPException(status_code=500) result = data.OnboardingState( is_complete=resource.resource_data.get("is_complete", False), steps=resource.resource_data.get("steps", {}), diff --git a/backend/moonstream/settings.py b/backend/moonstream/settings.py index 6408b1d8..0a90d751 100644 --- a/backend/moonstream/settings.py +++ b/backend/moonstream/settings.py @@ -9,6 +9,8 @@ bugout_client = Bugout(brood_api_url=BUGOUT_BROOD_URL, spire_api_url=BUGOUT_SPIR BUGOUT_REQUEST_TIMEOUT_SECONDS = 5 +HUMBUG_REPORTER_BACKEND_TOKEN = os.environ.get("HUMBUG_REPORTER_BACKEND_TOKEN") + # Default value is "" instead of None so that mypy understands that MOONSTREAM_APPLICATION_ID is a string MOONSTREAM_APPLICATION_ID = os.environ.get("MOONSTREAM_APPLICATION_ID", "") if MOONSTREAM_APPLICATION_ID == "": diff --git a/backend/requirements.txt b/backend/requirements.txt index 44e49539..4cc7c386 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -11,6 +11,7 @@ fastapi==0.66.0 h11==0.12.0 idna==3.2 jmespath==0.10.0 +humbug==0.2.7 -e git+https://git@github.com/bugout-dev/moonstream.git@94135b054cabb9dc11b0a2406431619279979469#egg=moonstreamdb&subdirectory=db mypy==0.910 mypy-extensions==0.4.3 @@ -28,5 +29,6 @@ toml==0.10.2 tomli==1.0.4 types-python-dateutil==0.1.6 typing-extensions==3.10.0.0 +types-requests==2.25.6 urllib3==1.26.6 uvicorn==0.14.0 diff --git a/backend/sample.env b/backend/sample.env index 054e040d..779bbb0c 100644 --- a/backend/sample.env +++ b/backend/sample.env @@ -5,6 +5,7 @@ export MOONSTREAM_DATA_JOURNAL_ID="" export MOONSTREAM_DB_URI="postgresql://:@:/" export MOONSTREAM_POOL_SIZE=0 export MOONSTREAM_ADMIN_ACCESS_TOKEN="" -export AWS_S3_SMARTCONTRACT_BUCKET="" +export AWS_S3_SMARTCONTRACT_BUCKET="" export BUGOUT_BROOD_URL="https://auth.bugout.dev" export BUGOUT_SPIRE_URL="https://spire.bugout.dev" +export HUMBUG_REPORTER_BACKEND_TOKEN="" diff --git a/backend/setup.py b/backend/setup.py index 0cd9b6e5..06a2a807 100644 --- a/backend/setup.py +++ b/backend/setup.py @@ -10,7 +10,16 @@ setup( name="moonstream", version=MOONSTREAM_VERSION, packages=find_packages(), - install_requires=["boto3", "bugout >= 0.1.17", "fastapi", "python-dateutil", "uvicorn", "types-python-dateutil"], + install_requires=[ + "boto3", + "bugout >= 0.1.17", + "fastapi", + "humbug>=0.2.7", + "python-dateutil", + "uvicorn", + "types-python-dateutil", + "types-requests", + ], extras_require={ "dev": ["black", "mypy"], "distribute": ["setuptools", "twine", "wheel"], diff --git a/crawlers/ethtxpool/main.go b/crawlers/ethtxpool/main.go index 224ae7dc..e4aa4f55 100644 --- a/crawlers/ethtxpool/main.go +++ b/crawlers/ethtxpool/main.go @@ -10,23 +10,20 @@ import ( "encoding/json" "flag" "fmt" + "math/big" "os" "time" humbug "github.com/bugout-dev/humbug/go/pkg" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" - + "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rpc" "github.com/google/uuid" ) -// Generate humbug client to be able write data in Bugout journal. -func humbugClientFromEnv() (*humbug.HumbugReporter, error) { - clientID := os.Getenv("ETHTXPOOL_HUMBUG_CLIENT_ID") - humbugToken := os.Getenv("ETHTXPOOL_HUMBUG_TOKEN") - sessionID := uuid.New().String() - +// Generate humbug client +func humbugClient(sessionID string, clientID string, humbugToken string) (*humbug.HumbugReporter, error) { consent := humbug.CreateHumbugConsent(humbug.True) reporter, err := humbug.CreateHumbugReporter(consent, clientID, sessionID, humbugToken) return reporter, err @@ -124,11 +121,6 @@ func PollTxpoolContent(gethClient *rpc.Client, interval int, reporter *humbug.Hu continue } - // TODO(kompotkot, zomglings): Humbug API (on Spire) support bulk publication of reports. We should modify - // Humbug go client to use the bulk publish endpoint. Currently, if we have to publish all transactions - // pending in txpool, we *will* get rate limited. We may want to consider adding a publisher to the - // Humbug go client that can listen on a channel and will handle rate limiting, bulk publication etc. itself - // (without user having to worry about it). ReportTitle := "Ethereum: Pending transaction: " + transactionHash.String() ReportTags := []string{ "hash:" + transactionHash.String(), @@ -138,6 +130,7 @@ func PollTxpoolContent(gethClient *rpc.Client, interval int, reporter *humbug.Hu fmt.Sprintf("max_priority_fee_per_gas:%d", pendingTx.Transaction.MaxPriorityFeePerGas.ToInt()), fmt.Sprintf("max_fee_per_gas:%d", pendingTx.Transaction.MaxFeePerGas.ToInt()), fmt.Sprintf("gas:%d", pendingTx.Transaction.Gas), + fmt.Sprintf("value:%d", new(big.Float).Quo(new(big.Float).SetInt(transaction.Value.ToInt()), big.NewFloat(params.Ether))), "crawl_type:ethereum_txpool", } report := humbug.Report{ @@ -188,6 +181,23 @@ func main() { flag.IntVar(&intervalSeconds, "interval", 1, "Number of seconds to wait between RPC calls to query the transaction pool (default: 1)") flag.Parse() + sessionID := uuid.New().String() + + // Humbug crash client to collect errors + crashReporter, err := humbugClient(sessionID, "moonstream-crawlers", os.Getenv("HUMBUG_REPORTER_CRAWLERS_TOKEN")) + if err != nil { + panic(fmt.Sprintf("Invalid Humbug Crash configuration: %s", err.Error())) + } + crashReporter.Publish(humbug.SystemReport()) + + defer func() { + message := recover() + if message != nil { + fmt.Printf("Error: %s\n", message) + crashReporter.Publish(humbug.PanicReport(message)) + } + }() + // Set connection with Ethereum blockchain via geth gethClient, err := rpc.Dial(gethConnectionString) if err != nil { @@ -195,7 +205,8 @@ func main() { } defer gethClient.Close() - reporter, err := humbugClientFromEnv() + // Humbug client to be able write data in Bugout journal + reporter, err := humbugClient(sessionID, os.Getenv("ETHTXPOOL_HUMBUG_CLIENT_ID"), os.Getenv("ETHTXPOOL_HUMBUG_TOKEN")) if err != nil { panic(fmt.Sprintf("Invalid Humbug configuration: %s", err.Error())) } diff --git a/crawlers/ethtxpool/sample.env b/crawlers/ethtxpool/sample.env index abd4bc8e..809cc325 100644 --- a/crawlers/ethtxpool/sample.env +++ b/crawlers/ethtxpool/sample.env @@ -1,2 +1,3 @@ export ETHTXPOOL_HUMBUG_CLIENT_ID="" export ETHTXPOOL_HUMBUG_TOKEN="" +export HUMBUG_REPORTER_CRAWLERS_TOKEN="" diff --git a/crawlers/mooncrawl/mooncrawl/__init__.py b/crawlers/mooncrawl/mooncrawl/__init__.py index e69de29b..9548e297 100644 --- a/crawlers/mooncrawl/mooncrawl/__init__.py +++ b/crawlers/mooncrawl/mooncrawl/__init__.py @@ -0,0 +1,7 @@ +from .reporter import reporter +from .version import MOONCRAWL_VERSION + +# Reporting +reporter.tags.append(f"version:{MOONCRAWL_VERSION}") +reporter.system_report(publish=True) +reporter.setup_excepthook(publish=True) diff --git a/crawlers/mooncrawl/mooncrawl/ethcrawler.py b/crawlers/mooncrawl/mooncrawl/ethcrawler.py index 793ab3ea..5e9275c4 100644 --- a/crawlers/mooncrawl/mooncrawl/ethcrawler.py +++ b/crawlers/mooncrawl/mooncrawl/ethcrawler.py @@ -48,7 +48,7 @@ def yield_blocks_numbers_lists( print( "Wrong format provided, expected {bottom_block}-{top_block}, as ex. 105-340" ) - return + raise Exception starting_block = max(input_start_block, input_end_block) ending_block = min(input_start_block, input_end_block) diff --git a/crawlers/mooncrawl/mooncrawl/etherscan.py b/crawlers/mooncrawl/mooncrawl/etherscan.py index 6a1ead76..f3f61a83 100644 --- a/crawlers/mooncrawl/mooncrawl/etherscan.py +++ b/crawlers/mooncrawl/mooncrawl/etherscan.py @@ -1,24 +1,26 @@ import argparse +import codecs +import csv +from dataclasses import dataclass +from datetime import datetime +import json +import logging +import os import sys import time -from datetime import datetime from typing import Any, List, Optional, Dict -from dataclasses import dataclass -import csv -import codecs -import json -import os import boto3 # type: ignore from moonstreamdb.db import yield_db_session_ctx from moonstreamdb.models import EthereumAddress, EthereumLabel import requests from sqlalchemy.orm import Session -from sqlalchemy.sql.expression import text from .version import MOONCRAWL_VERSION from .settings import MOONSTREAM_ETHERSCAN_TOKEN +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) if MOONSTREAM_ETHERSCAN_TOKEN is None: raise Exception("MOONSTREAM_ETHERSCAN_TOKEN environment variable must be set") @@ -66,21 +68,16 @@ def get_address_id(db_session: Session, contract_address: str) -> int: if id is not None: return id[0] - latest_address_id = ( - db_session.query(EthereumAddress.id).order_by(text("id desc")).limit(1).one() - )[0] - - id = latest_address_id + 1 smart_contract = EthereumAddress( - id=id, address=contract_address, ) try: db_session.add(smart_contract) db_session.commit() - except: + return smart_contract.id + except Exception as e: db_session.rollback() - return id + raise e def crawl_step(db_session: Session, contract: VerifiedSmartContract, crawl_url: str): @@ -112,22 +109,27 @@ def crawl_step(db_session: Session, contract: VerifiedSmartContract, crawl_url: object_key = f"/etherscan/v1/{contract.address}.json" push_to_bucket(contract_info, object_key) - eth_address_id = get_address_id(db_session, contract.address) - - eth_label = EthereumLabel( - label=ETHERSCAN_SMARTCONTRACTS_LABEL_NAME, - address_id=eth_address_id, - label_data={ - "object_uri": f"s3://{bucket}/{object_key}", - "name": contract.name, - "tx_hash": contract.tx_hash, - }, - ) try: - db_session.add(eth_label) - db_session.commit() - except: - db_session.rollback() + eth_address_id = get_address_id(db_session, contract.address) + eth_label = EthereumLabel( + label=ETHERSCAN_SMARTCONTRACTS_LABEL_NAME, + address_id=eth_address_id, + label_data={ + "object_uri": f"s3://{bucket}/{object_key}", + "name": contract.name, + "tx_hash": contract.tx_hash, + }, + ) + try: + db_session.add(eth_label) + db_session.commit() + except Exception as e: + db_session.rollback() + raise e + except Exception as e: + logger.error( + f"Failed to add addresss label ${contract.address} to database\n{str(e)}" + ) def crawl( diff --git a/crawlers/mooncrawl/mooncrawl/reporter.py b/crawlers/mooncrawl/mooncrawl/reporter.py new file mode 100644 index 00000000..0cf170ac --- /dev/null +++ b/crawlers/mooncrawl/mooncrawl/reporter.py @@ -0,0 +1,18 @@ +import uuid + +from humbug.consent import HumbugConsent +from humbug.report import HumbugReporter + +from .settings import HUMBUG_REPORTER_CRAWLERS_TOKEN + +session_id = str(uuid.uuid4()) +client_id = "moonstream-crawlers" + +reporter = HumbugReporter( + name="moonstream-crawlers", + consent=HumbugConsent(True), + client_id=client_id, + session_id=session_id, + bugout_token=HUMBUG_REPORTER_CRAWLERS_TOKEN, + tags=[], +) diff --git a/crawlers/mooncrawl/mooncrawl/settings.py b/crawlers/mooncrawl/mooncrawl/settings.py index 82b13772..307f2e8a 100644 --- a/crawlers/mooncrawl/mooncrawl/settings.py +++ b/crawlers/mooncrawl/mooncrawl/settings.py @@ -1,5 +1,9 @@ import os +# Bugout +HUMBUG_REPORTER_CRAWLERS_TOKEN = os.environ.get("HUMBUG_REPORTER_CRAWLERS_TOKEN") + +# Geth MOONSTREAM_IPC_PATH = os.environ.get("MOONSTREAM_IPC_PATH", None) MOONSTREAM_CRAWL_WORKERS = 4 @@ -12,5 +16,5 @@ except: f"Could not parse MOONSTREAM_CRAWL_WORKERS as int: {MOONSTREAM_CRAWL_WORKERS_RAW}" ) - +# Etherscan MOONSTREAM_ETHERSCAN_TOKEN = os.environ.get("MOONSTREAM_ETHERSCAN_TOKEN") diff --git a/crawlers/mooncrawl/sample.env b/crawlers/mooncrawl/sample.env index 5ebad6e0..5d4ac548 100644 --- a/crawlers/mooncrawl/sample.env +++ b/crawlers/mooncrawl/sample.env @@ -6,3 +6,4 @@ export MOONSTREAM_ETHERSCAN_TOKEN="" export AWS_S3_SMARTCONTRACT_BUCKET="" export MOONSTREAM_HUMBUG_TOKEN="" export COINMARKETCAP_API_KEY="" +export HUMBUG_REPORTER_CRAWLERS_TOKEN="" diff --git a/crawlers/mooncrawl/setup.py b/crawlers/mooncrawl/setup.py index 1e893bd4..f26b82d5 100644 --- a/crawlers/mooncrawl/setup.py +++ b/crawlers/mooncrawl/setup.py @@ -1,6 +1,5 @@ from setuptools import find_packages, setup -from mooncrawl.version import MOONCRAWL_VERSION long_description = "" with open("README.md") as ifp: @@ -8,7 +7,7 @@ with open("README.md") as ifp: setup( name="mooncrawl", - version=MOONCRAWL_VERSION, + version="0.0.3", author="Bugout.dev", author_email="engineers@bugout.dev", license="Apache License 2.0", @@ -34,6 +33,7 @@ setup( zip_safe=False, install_requires=[ "moonstreamdb @ git+https://git@github.com/bugout-dev/moonstream.git@39d2b8e36a49958a9ae085ec2cc1be3fc732b9d0#egg=moonstreamdb&subdirectory=db", + "humbug", "python-dateutil", "requests", "tqdm", diff --git a/db/alembic/versions/ecb7817db377_add_opensea_state_table_and_add_index_.py b/db/alembic/versions/ecb7817db377_add_opensea_state_table_and_add_index_.py new file mode 100644 index 00000000..98a07eb0 --- /dev/null +++ b/db/alembic/versions/ecb7817db377_add_opensea_state_table_and_add_index_.py @@ -0,0 +1,49 @@ +"""Add opensea state table and add index by label_data ->> name + +Revision ID: ecb7817db377 +Revises: ea8185bd24c7 +Create Date: 2021-08-31 17:44:24.139028 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = "ecb7817db377" +down_revision = "ea8185bd24c7" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "opensea_crawler_state", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("query", sa.Text(), nullable=False), + sa.Column( + "crawled_at", + sa.DateTime(timezone=True), + server_default=sa.text("TIMEZONE('utc', statement_timestamp())"), + nullable=False, + ), + sa.Column("total_count", sa.Integer(), nullable=False), + sa.PrimaryKeyConstraint("id", name=op.f("pk_opensea_crawler_state")), + sa.UniqueConstraint("id", name=op.f("uq_opensea_crawler_state_id")), + ) + op.execute( + "ALTER TABLE ethereum_labels DROP CONSTRAINT IF EXISTS uq_ethereum_labels_label" + ) + + op.execute( + f"CREATE INDEX idx_ethereum_labels_opensea_nft_name ON ethereum_labels((label_data->>'name')) where label='opensea_nft';" + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table("opensea_crawler_state") + op.drop_index("idx_ethereum_labels_opensea_nft_name") + # ### end Alembic commands ### diff --git a/db/moonstreamdb/models.py b/db/moonstreamdb/models.py index 3114133f..7d7798a4 100644 --- a/db/moonstreamdb/models.py +++ b/db/moonstreamdb/models.py @@ -11,7 +11,6 @@ from sqlalchemy import ( Numeric, Text, VARCHAR, - UniqueConstraint, ) from sqlalchemy.dialects.postgresql import JSONB, UUID from sqlalchemy.sql import expression @@ -136,7 +135,6 @@ class EthereumLabel(Base): # type: ignore """ __tablename__ = "ethereum_labels" - __table_args__ = (UniqueConstraint("label", "address_id"),) id = Column( UUID(as_uuid=True), @@ -212,3 +210,22 @@ class ESDEventSignature(Base): # type: ignore created_at = Column( DateTime(timezone=True), server_default=utcnow(), nullable=False ) + + +class OpenSeaCrawlingState(Base): # type: ignore + """ + Model for control opeansea crawling state. + """ + + __tablename__ = "opensea_crawler_state" + + id = Column(Integer, primary_key=True, unique=True, nullable=False) + query = Column(Text, nullable=False) + crawled_at = Column( + DateTime(timezone=True), + server_default=utcnow(), + onupdate=utcnow(), + nullable=False, + ) + + total_count = Column(Integer, nullable=False) diff --git a/frontend/pages/index.js b/frontend/pages/index.js index c3e21079..0bcfdeaa 100644 --- a/frontend/pages/index.js +++ b/frontend/pages/index.js @@ -24,12 +24,15 @@ import { } from "@chakra-ui/react"; import dynamic from "next/dynamic"; import useUser from "../src/core/hooks/useUser"; -import useAnalytics from "../src/core/hooks/useAnalytics"; import useModals from "../src/core/hooks/useModals"; import useRouter from "../src/core/hooks/useRouter"; -import { MIXPANEL_PROPS } from "../src/core/providers/AnalyticsProvider/constants"; +import { + MIXPANEL_PROPS, + MIXPANEL_EVENTS, +} from "../src/core/providers/AnalyticsProvider/constants"; import UIContext from "../src/core/providers/UIProvider/context"; import { AWS_ASSETS_PATH } from "../src/core/constants"; +import mixpanel from "mixpanel-browser"; const SplitWithImage = dynamic( () => import("../src/components/SplitWithImage"), { @@ -105,7 +108,6 @@ const Homepage = () => { const router = useRouter(); const { isInit } = useUser(); - const { MIXPANEL_EVENTS, track } = useAnalytics(); const { toggleModal } = useModals(); const [ isLargerThan720px, @@ -379,27 +381,30 @@ const Homepage = () => { label: "Crypto trader", link: "/#cryptoTrader", onClick: () => { - track(`${MIXPANEL_EVENTS.BUTTON_CLICKED}`, { - [`${MIXPANEL_PROPS.BUTTON_CLICKED}`]: `scroll to CryptoTrader`, - }); + mixpanel.get_distinct_id() && + mixpanel.track(`${MIXPANEL_EVENTS.BUTTON_CLICKED}`, { + [`${MIXPANEL_PROPS.BUTTON_NAME}`]: `scroll to CryptoTrader`, + }); }, }} button2={{ label: "Algorithmic Fund", link: "/#algoFund", onClick: () => { - track(`${MIXPANEL_EVENTS.BUTTON_CLICKED}`, { - [`${MIXPANEL_PROPS.BUTTON_CLICKED}`]: `scroll to AlgoFund`, - }); + mixpanel.get_distinct_id() && + mixpanel.track(`${MIXPANEL_EVENTS.BUTTON_CLICKED}`, { + [`${MIXPANEL_PROPS.BUTTON_NAME}`]: `scroll to AlgoFund`, + }); }, }} button3={{ label: "Developer", link: "/#smartDeveloper", onClick: () => { - track(`${MIXPANEL_EVENTS.BUTTON_CLICKED}`, { - [`${MIXPANEL_PROPS.BUTTON_CLICKED}`]: `scroll to Developer`, - }); + mixpanel.get_distinct_id() && + mixpanel.track(`${MIXPANEL_EVENTS.BUTTON_CLICKED}`, { + [`${MIXPANEL_PROPS.BUTTON_NAME}`]: `scroll to Developer`, + }); }, }} /> @@ -417,9 +422,10 @@ const Homepage = () => { cta={{ label: "I want early access!", onClick: () => { - track(`${MIXPANEL_EVENTS.BUTTON_CLICKED}`, { - [`${MIXPANEL_PROPS.BUTTON_CLICKED}`]: `Early access CTA: Crypto trader`, - }); + mixpanel.get_distinct_id() && + mixpanel.track(`${MIXPANEL_EVENTS.BUTTON_CLICKED}`, { + [`${MIXPANEL_PROPS.BUTTON_NAME}`]: `Early access CTA: Crypto trader`, + }); toggleModal("hubspot-trader"); }, }} @@ -464,9 +470,10 @@ const Homepage = () => { cta={{ label: "I want early access!", onClick: () => { - track(`${MIXPANEL_EVENTS.BUTTON_CLICKED}`, { - [`${MIXPANEL_PROPS.BUTTON_CLICKED}`]: `Early access CTA: Algo fund`, - }); + mixpanel.get_distinct_id() && + mixpanel.track(`${MIXPANEL_EVENTS.BUTTON_CLICKED}`, { + [`${MIXPANEL_PROPS.BUTTON_NAME}`]: `Early access CTA: Algo fund`, + }); toggleModal("hubspot-fund"); }, }} @@ -509,9 +516,10 @@ const Homepage = () => { cta={{ label: "I want early access!", onClick: () => { - track(`${MIXPANEL_EVENTS.BUTTON_CLICKED}`, { - [`${MIXPANEL_PROPS.BUTTON_CLICKED}`]: `Early access CTA: developer`, - }); + mixpanel.get_distinct_id() && + mixpanel.track(`${MIXPANEL_EVENTS.BUTTON_CLICKED}`, { + [`${MIXPANEL_PROPS.BUTTON_NAME}`]: `Early access CTA: developer`, + }); toggleModal("hubspot-developer"); }, }} @@ -520,9 +528,10 @@ const Homepage = () => { network: "github", label: "See our github", onClick: () => { - track(`${MIXPANEL_EVENTS.BUTTON_CLICKED}`, { - [`${MIXPANEL_PROPS.BUTTON_CLICKED}`]: `Github link in landing page`, - }); + mixpanel.get_distinct_id() && + mixpanel.track(`${MIXPANEL_EVENTS.BUTTON_CLICKED}`, { + [`${MIXPANEL_PROPS.BUTTON_NAME}`]: `Github link in landing page`, + }); }, }} elementName={"element3"} @@ -568,9 +577,10 @@ const Homepage = () => { colorScheme="suggested" id="test" onClick={() => { - track(`${MIXPANEL_EVENTS.BUTTON_CLICKED}`, { - [`${MIXPANEL_PROPS.BUTTON_CLICKED}`]: `Join our discord`, - }); + mixpanel.get_distinct_id() && + mixpanel.track(`${MIXPANEL_EVENTS.BUTTON_CLICKED}`, { + [`${MIXPANEL_PROPS.BUTTON_NAME}`]: `Join our discord`, + }); toggleModal("hubspot"); }} > diff --git a/frontend/src/AppContext.js b/frontend/src/AppContext.js index 7a238760..f2c56e74 100644 --- a/frontend/src/AppContext.js +++ b/frontend/src/AppContext.js @@ -13,13 +13,13 @@ const AppContext = (props) => { return ( - - - - {props.children} - - - + + + + {props.children} + + + ); diff --git a/frontend/src/components/Footer.js b/frontend/src/components/Footer.js index d212befe..2b95a9f6 100644 --- a/frontend/src/components/Footer.js +++ b/frontend/src/components/Footer.js @@ -6,7 +6,7 @@ import RouterLink from "next/link"; const ICONS = [ { social: "discord", - link: "https://discord.gg/FetK5BxD", + link: "https://discord.gg/K56VNUQGvA", }, { social: "twit", link: "https://twitter.com/moonstreamto" }, diff --git a/frontend/src/components/Scrollable.js b/frontend/src/components/Scrollable.js index 4ab9f6a7..bdcdf0f0 100644 --- a/frontend/src/components/Scrollable.js +++ b/frontend/src/components/Scrollable.js @@ -1,13 +1,13 @@ import { Flex, Box } from "@chakra-ui/react"; import React, { useEffect, useRef, useState } from "react"; -import { useRouter, useAnalytics } from "../core/hooks"; +import { useRouter } from "../core/hooks"; +import mixpanel from "mixpanel-browser"; const Scrollable = (props) => { const scrollerRef = useRef(); const router = useRouter(); const [path, setPath] = useState(); const [scrollDepth, setScrollDepth] = useState(0); - const { mixpanel, isLoaded } = useAnalytics(); const getScrollPrecent = ({ currentTarget }) => { const scroll_level = @@ -20,7 +20,7 @@ const Scrollable = (props) => { const currentScroll = Math.ceil(getScrollPrecent(e) / 10); if (currentScroll > scrollDepth) { setScrollDepth(currentScroll); - isLoaded && + mixpanel.get_distinct_id() && mixpanel.people.increment({ [`Scroll depth at: ${router.nextRouter.pathname}`]: currentScroll, }); diff --git a/frontend/src/core/hooks/index.js b/frontend/src/core/hooks/index.js index 328a74da..5625f907 100644 --- a/frontend/src/core/hooks/index.js +++ b/frontend/src/core/hooks/index.js @@ -1,5 +1,4 @@ export { queryCacheProps as hookCommon } from "./hookCommon"; -export { default as useAnalytics } from "./useAnalytics"; export { default as useAuthResultHandler } from "./useAuthResultHandler"; export { default as useChangePassword } from "./useChangePassword"; export { default as useClientID } from "./useClientID"; diff --git a/frontend/src/core/hooks/useAnalytics.js b/frontend/src/core/hooks/useAnalytics.js deleted file mode 100644 index fee7e9eb..00000000 --- a/frontend/src/core/hooks/useAnalytics.js +++ /dev/null @@ -1,38 +0,0 @@ -import AnalyticsContext from "../providers/AnalyticsProvider/context"; -import { useContext } from "react"; -import { useState, useEffect, useCallback } from "react"; -const useAnalytics = () => { - const { mixpanel, isLoaded, MIXPANEL_EVENTS, MIXPANEL_PROPS } = - useContext(AnalyticsContext); - const [trackProps, setTrackProps] = useState({ - event: null, - props: null, - queued: false, - }); - - const track = useCallback((e, props) => { - setTrackProps({ event: e, props: props, queued: true }); - }, []); - - useEffect(() => { - if (isLoaded && trackProps.queued === true) { - mixpanel.track(trackProps.event, trackProps.props); - setTrackProps({ event: null, props: null, queued: false }); - } - }, [isLoaded, mixpanel, trackProps]); - - const withTracking = (fn, event, props) => { - track(event, props); - return fn; - }; - - return { - mixpanel, - isLoaded, - track, - MIXPANEL_PROPS, - MIXPANEL_EVENTS, - withTracking, - }; -}; -export default useAnalytics; diff --git a/frontend/src/core/hooks/useLogin.js b/frontend/src/core/hooks/useLogin.js index c8dd3bfc..b410cc9a 100644 --- a/frontend/src/core/hooks/useLogin.js +++ b/frontend/src/core/hooks/useLogin.js @@ -1,7 +1,6 @@ import { useMutation } from "react-query"; import { useToast, useUser, useInviteAccept } from "."; import { AuthService } from "../services"; -import { useAnalytics } from "."; const LOGIN_TYPES = { MANUAL: true, @@ -10,7 +9,6 @@ const LOGIN_TYPES = { const useLogin = (loginType) => { const { getUser } = useUser(); const toast = useToast(); - const analytics = useAnalytics(); const { inviteAccept } = useInviteAccept(); const { mutate: login, @@ -34,20 +32,6 @@ const useLogin = (loginType) => { inviteAccept(invite_code); } getUser(); - if (analytics.isLoaded) { - analytics.mixpanel.people.set_once({ - [`${analytics.MIXPANEL_EVENTS.FIRST_LOGIN_DATE}`]: - new Date().toISOString(), - }); - analytics.mixpanel.people.set({ - [`${analytics.MIXPANEL_EVENTS.LAST_LOGIN_DATE}`]: - new Date().toISOString(), - }); - analytics.mixpanel.track( - `${analytics.MIXPANEL_EVENTS.USER_LOGS_IN}`, - {} - ); - } } }, onError: (error) => { diff --git a/frontend/src/core/hooks/useLogout.js b/frontend/src/core/hooks/useLogout.js index ff605d80..c8be9a86 100644 --- a/frontend/src/core/hooks/useLogout.js +++ b/frontend/src/core/hooks/useLogout.js @@ -1,21 +1,14 @@ import { useCallback, useContext } from "react"; import { useMutation, useQueryClient } from "react-query"; -import { useUser, useRouter, useAnalytics } from "."; +import { useUser, useRouter } from "."; import UIContext from "../providers/UIProvider/context"; import { AuthService } from "../services"; const useLogout = () => { const { setLoggingOut } = useContext(UIContext); const router = useRouter(); - const analytics = useAnalytics(); const { mutate: revoke } = useMutation(AuthService.revoke, { onSuccess: () => { - if (analytics.isLoaded) { - analytics.mixpanel.track( - `${analytics.MIXPANEL_EVENTS.USER_LOGS_OUT}`, - {} - ); - } localStorage.removeItem("MOONSTREAM_ACCESS_TOKEN"); cache.clear(); setUser(null); diff --git a/frontend/src/core/hooks/useSignUp.js b/frontend/src/core/hooks/useSignUp.js index f6dca897..6ae77f7e 100644 --- a/frontend/src/core/hooks/useSignUp.js +++ b/frontend/src/core/hooks/useSignUp.js @@ -1,16 +1,18 @@ import { useContext } from "react"; import { useMutation } from "react-query"; import { AuthService } from "../services"; -import { useUser, useToast, useInviteAccept, useRouter, useAnalytics } from "."; +import { useUser, useToast, useInviteAccept, useRouter } from "."; import UIContext from "../providers/UIProvider/context"; +import mixpanel from "mixpanel-browser"; +import { MIXPANEL_EVENTS } from "../providers/AnalyticsProvider/constants"; const useSignUp = (source) => { const ui = useContext(UIContext); + const router = useRouter(); const { getUser } = useUser(); const toast = useToast(); const { inviteAccept } = useInviteAccept(); - const analytics = useAnalytics(); const { mutate: signUp, @@ -26,11 +28,11 @@ const useSignUp = (source) => { inviteAccept(invite_code); } - if (analytics.isLoaded) { - analytics.mixpanel.track( - `${analytics.MIXPANEL_EVENTS.CONVERT_TO_USER}`, - { full_url: router.nextRouter.asPath, code: source } - ); + if (mixpanel.get_distinct_id()) { + mixpanel.track(`${MIXPANEL_EVENTS.CONVERT_TO_USER}`, { + full_url: router.nextRouter.asPath, + code: source, + }); } getUser(); ui.setisOnboardingComplete(false); diff --git a/frontend/src/core/hooks/useToast.js b/frontend/src/core/hooks/useToast.js index eeae3e7e..6d78ed9a 100644 --- a/frontend/src/core/hooks/useToast.js +++ b/frontend/src/core/hooks/useToast.js @@ -1,21 +1,18 @@ import { useToast as useChakraToast, Box } from "@chakra-ui/react"; import React, { useCallback } from "react"; -import { useAnalytics } from "."; +import mixpanel from "mixpanel-browser"; +import { MIXPANEL_EVENTS } from "../providers/AnalyticsProvider/constants"; const useToast = () => { const chakraToast = useChakraToast(); - const analytics = useAnalytics(); const toast = useCallback( (message, type) => { - if (analytics.isLoaded && type === "error") { - analytics.mixpanel.track( - `${analytics.MIXPANEL_EVENTS.TOAST_ERROR_DISPLAYED}`, - { - status: message?.response?.status, - detail: message?.response?.data.detail, - } - ); + if (mixpanel.get_distinct_id() && type === "error") { + mixpanel.track(`${MIXPANEL_EVENTS.TOAST_ERROR_DISPLAYED}`, { + status: message?.response?.status, + detail: message?.response?.data.detail, + }); } const background = type === "error" ? "unsafe.500" : "suggested.500"; const userMessage = @@ -43,7 +40,7 @@ const useToast = () => { ), }); }, - [chakraToast, analytics] + [chakraToast] ); return toast; diff --git a/frontend/src/core/providers/AnalyticsProvider/constants.js b/frontend/src/core/providers/AnalyticsProvider/constants.js index 309532e1..d8dc2a09 100644 --- a/frontend/src/core/providers/AnalyticsProvider/constants.js +++ b/frontend/src/core/providers/AnalyticsProvider/constants.js @@ -8,6 +8,12 @@ export const MIXPANEL_PROPS = { USER_SPECIALITY: "user speciality", }; +export const MIXPANEL_GROUPS = { + DEVELOPERS: "developers", + TRADERS: "traders", + FUND: "funds", +}; + export const MIXPANEL_EVENTS = { FIRST_LOGIN_DATE: "First login date", LAST_LOGIN_DATE: "Last login date", @@ -20,7 +26,14 @@ export const MIXPANEL_EVENTS = { PAGEVIEW: "Page view", PRICING_PLAN_CLICKED: "Pricing Plan clicked", BUTTON_CLICKED: "Button clicked", - LEFT_PAGE: "Left page", + BEACON: "beacon", + ONBOARDING_COMPLETED: "Onbording complete", + SESSIONS_COUNT: "Sessions Counter", + ONBOARDING_STEP: "Onboarding step", + ONBOARDING_STATE: "Onboarding state", + TIMES_VISITED: "Page visit times", + FORM_SUBMITTED: "form submitted", + PAGEVIEW_DURATION: "Time spent on page", }; export default MIXPANEL_EVENTS; diff --git a/frontend/src/core/providers/AnalyticsProvider/index.js b/frontend/src/core/providers/AnalyticsProvider/index.js index eddffb8a..f05b2d85 100644 --- a/frontend/src/core/providers/AnalyticsProvider/index.js +++ b/frontend/src/core/providers/AnalyticsProvider/index.js @@ -1,66 +1,161 @@ -import React, { useEffect, useState } from "react"; +import React, { useContext, useEffect, useState } from "react"; import mixpanel from "mixpanel-browser"; import AnalyticsContext from "./context"; import { useClientID, useUser, useRouter } from "../../hooks"; import { MIXPANEL_EVENTS, MIXPANEL_PROPS } from "./constants"; +import UIContext from "../UIProvider/context"; const AnalyticsProvider = ({ children }) => { const clientID = useClientID(); const analytics = process.env.NEXT_PUBLIC_MIXPANEL_TOKEN; - const { user } = useUser(); - const [isLoaded, setIsLoaded] = useState(false); + const { user, isInit } = useUser(); + const [isMixpanelReady, setIsLoaded] = useState(false); const router = useRouter(); + const ui = useContext(UIContext); + // ********** OBOARDING STATE ************** + useEffect(() => { + if (ui.onboardingState && isMixpanelReady) { + mixpanel.people.set(MIXPANEL_EVENTS.ONBOARDING_STATE, { + state: { ...ui.onboardingState }, + }); + } + }, [ui.onboardingState, isMixpanelReady]); + + useEffect(() => { + if (ui.isOnboardingComplete && isMixpanelReady && user) { + mixpanel.people.set(MIXPANEL_EVENTS.ONBOARDING_COMPLETED, true); + } + }, [ui.isOnboardingComplete, isMixpanelReady, user]); + + // ********** ONBOARDING STEP and TIMING ************** + const [previousOnboardingStep, setPreviousOnboardingStep] = useState(false); + + useEffect(() => { + if (isMixpanelReady && router.nextRouter.pathname === "/welcome") { + if (!previousOnboardingStep) { + mixpanel.time_event(MIXPANEL_EVENTS.ONBOARDING_STEP); + setPreviousOnboardingStep(ui.onboardingStep); + } + if ( + previousOnboardingStep && + previousOnboardingStep !== ui.onboardingStep + ) { + mixpanel.track(MIXPANEL_EVENTS.ONBOARDING_STEP, { + step: previousOnboardingStep, + isBeforeUnload: false, + }); + setPreviousOnboardingStep(false); + } + } else if (previousOnboardingStep) { + mixpanel.track(MIXPANEL_EVENTS.ONBOARDING_STEP, { + step: previousOnboardingStep, + isBeforeUnload: false, + }); + setPreviousOnboardingStep(false); + } + }, [ + previousOnboardingStep, + ui.onboardingStep, + isMixpanelReady, + router.nextRouter.pathname, + ]); + + // ********** PING_PONG ************** useEffect(() => { let durationSeconds = 0; const intervalId = - isLoaded && + isMixpanelReady && setInterval(() => { - durationSeconds = durationSeconds + 1; + durationSeconds = durationSeconds + 30; mixpanel.track( - MIXPANEL_EVENTS.LEFT_PAGE, + MIXPANEL_EVENTS.BEACON, { duration_seconds: durationSeconds, url: router.nextRouter.pathname, - query: router.query, - pathParams: router.params, }, { transport: "sendBeacon" } ); }, 30000); return () => clearInterval(intervalId); - // eslint-disable-next-line - }, [isLoaded]); + }, [isMixpanelReady, router.nextRouter.pathname]); + + // ********** TIME SPENT ON PATH************** + + const [previousPathname, setPreviousPathname] = useState(false); useEffect(() => { - isLoaded && + if (isMixpanelReady) { + if (!previousPathname) { + mixpanel.time_event(MIXPANEL_EVENTS.PAGEVIEW_DURATION); + setPreviousPathname(router.nextRouter.pathname); + } + if (previousPathname && previousPathname !== router.nextRouter.pathname) { + mixpanel.track(MIXPANEL_EVENTS.PAGEVIEW_DURATION, { + url: previousPathname, + isBeforeUnload: false, + }); + setPreviousPathname(false); + } + } + }, [router.nextRouter.pathname, previousPathname, isMixpanelReady]); + + // ********** PAGES VIEW ************** + useEffect(() => { + if (isMixpanelReady && ui.sessionId && router.nextRouter.pathname) { mixpanel.track(MIXPANEL_EVENTS.PAGEVIEW, { url: router.nextRouter.pathname, - query: router.query, - pathParams: router.params, + sessionID: ui.sessionId, }); - }, [router.nextRouter.pathname, router.query, router.params, isLoaded]); - useEffect(() => { - try { - mixpanel.init(analytics, { - api_host: "https://api.mixpanel.com", - loaded: () => { - setIsLoaded(true); - mixpanel.identify(clientID); - }, + mixpanel.people.increment([ + `${MIXPANEL_EVENTS.TIMES_VISITED} ${router.nextRouter.pathname}`, + ]); + } + const urlForUnmount = router.nextRouter.pathname; + const closeListener = () => { + mixpanel.track(MIXPANEL_EVENTS.PAGEVIEW_DURATION, { + url: urlForUnmount, + isBeforeUnload: true, }); - } catch (error) { - console.warn("loading mixpanel failed:", error); + }; + window.addEventListener("beforeunload", closeListener); + //cleanup function fires on useEffect unmount + //https://reactjs.org/docs/hooks-effect.html + return () => { + window.removeEventListener("beforeunload", closeListener); + }; + }, [router.nextRouter.pathname, isMixpanelReady, ui.sessionId]); + + // ********** SESSION STATE ************** + useEffect(() => { + if (clientID) { + try { + mixpanel.init(analytics, { + api_host: "https://api.mixpanel.com", + loaded: () => { + setIsLoaded(true); + mixpanel.identify(clientID); + }, + }); + } catch (error) { + console.warn("loading mixpanel failed:", error); + } } }, [analytics, clientID]); + useEffect(() => { + isMixpanelReady && mixpanel.register("sessionId", ui.sessionId); + }, [ui.sessionId, isMixpanelReady]); + + // ********** USER STATE ************** + useEffect(() => { if (user) { try { - if (isLoaded) { + if (isMixpanelReady) { mixpanel.people.set({ [`${MIXPANEL_EVENTS.LAST_VISITED}`]: new Date().toISOString(), }); @@ -74,11 +169,36 @@ const AnalyticsProvider = ({ children }) => { console.error("could not set up people in mixpanel:", err); } } - }, [user, isLoaded, clientID]); + }, [user, isMixpanelReady, clientID]); + + useEffect(() => { + if (isMixpanelReady && user) { + mixpanel.people.set_once({ + [`${MIXPANEL_EVENTS.FIRST_LOGIN_DATE}`]: new Date().toISOString(), + }); + mixpanel.people.set({ + [`${MIXPANEL_EVENTS.LAST_LOGIN_DATE}`]: new Date().toISOString(), + }); + mixpanel.track(`${MIXPANEL_EVENTS.USER_LOGS_IN}`, {}); + } + }, [user, isMixpanelReady]); + + useEffect(() => { + if (isMixpanelReady && ui.isLoggingOut) { + mixpanel.track(`${MIXPANEL_EVENTS.USER_LOGS_OUT}`, {}); + } + }, [ui.isLoggingOut, isMixpanelReady]); + + // ********** USER BOUNCE TIME ************** + useEffect(() => { + if (!user && isInit && isMixpanelReady) { + mixpanel.time_event(MIXPANEL_EVENTS.CONVERT_TO_USER); + } + }, [user, isInit, isMixpanelReady]); return ( {children} diff --git a/frontend/src/core/providers/UIProvider/index.js b/frontend/src/core/providers/UIProvider/index.js index c9af6911..d4d73a58 100644 --- a/frontend/src/core/providers/UIProvider/index.js +++ b/frontend/src/core/providers/UIProvider/index.js @@ -298,6 +298,8 @@ const UIProvider = ({ children }) => { setOnboardingComplete, onboardingSteps, setOnboardingState, + onboardingState, + isLoggingOut, }} > {children}