mirror
/
moonstream
kopia lustrzana https://github.com/bugout-dev/moonstream
1
0
Forkuj 0
Kod Zgłoszenia Projekty Wydania Wiki Aktywność

Merge branch 'main' into store-onbarding-on-backend

pull/227/head
Tim Pechersky 2021-09-06 19:24:37 +02:00
rodzic 5a2b88d86d fb9f2ed24e
commit eb1fd2b057
38 zmienionych plików z 605 dodań i 307 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

7
backend/moonstream/__init__.py
Wyświetl plik

@ -0,0 +1,7 @@
from .reporter import reporter
from .version import MOONSTREAM_VERSION
# Reporting
reporter.tags.append(f"version:{MOONSTREAM_VERSION}")
reporter.system_report(publish=True)
reporter.setup_excepthook(publish=True)

9
backend/moonstream/actions.py
Wyświetl plik

@ -1,16 +1,18 @@
import json
import logging
from typing import Dict, Any, List, Optional
from typing import Optional
from enum import Enum
import boto3 # type: ignore
from moonstreamdb.models import (
EthereumAddress,
EthereumLabel,
)
from sqlalchemy import text
from sqlalchemy.orm import Session, query, query_expression
from sqlalchemy.orm import Session
from . import data
from .reporter import reporter
from .settings import ETHERSCAN_SMARTCONTRACTS_BUCKET
import uuid
from bugout.data import BugoutResource
@ -53,8 +55,9 @@ def get_contract_source_info(
abi=obj_data["ABI"],
)
return contract_source_info
except:
except Exception as e:
logger.error(f"Failed to load smart contract {object_uri}")
reporter.error_report(e)
return None

25
backend/moonstream/middleware.py
Wyświetl plik

@ -1,16 +1,36 @@
import logging
from typing import Awaitable, Callable, Dict, Optional
from typing import Any, Awaitable, Callable, Dict, Optional
from bugout.data import BugoutUser
from bugout.exceptions import BugoutResponseException
from fastapi import HTTPException, Request, Response
from starlette.background import BackgroundTask
from starlette.middleware.base import BaseHTTPMiddleware
from fastapi import Request, Response
from .reporter import reporter
from .settings import MOONSTREAM_APPLICATION_ID, bugout_client as bc
logger = logging.getLogger(__name__)
class MoonstreamHTTPException(HTTPException):
"""
Extended HTTPException to handle 500 Internal server errors
and send crash reports.
"""
def __init__(
self,
status_code: int,
detail: Any = None,
headers: Optional[Dict[str, Any]] = None,
internal_error: Exception = None,
):
super().__init__(status_code, detail, headers)
if internal_error is not None:
reporter.error_report(internal_error)
class BroodAuthMiddleware(BaseHTTPMiddleware):
"""
Checks the authorization header on the request. If it represents a verified Brood user,
@ -61,6 +81,7 @@ class BroodAuthMiddleware(BaseHTTPMiddleware):
return Response(status_code=e.status_code, content=e.detail)
except Exception as e:
logger.error(f"Error processing Brood response: {str(e)}")
reporter.error_report(e)
return Response(status_code=500, content="Internal server error")
request.state.user = user

15
backend/moonstream/providers/__init__.py
Wyświetl plik

@ -39,6 +39,13 @@ from ..stream_queries import StreamQuery
logger = logging.getLogger(__name__)
logger.setLevel(logging.WARN)
class ReceivingEventsException(Exception):
"""
Raised when error occurs during receiving events from provider.
"""
event_providers: Dict[str, Any] = {
ethereum_blockchain.event_type: ethereum_blockchain,
bugout.whalewatch_provider.event_type: bugout.whalewatch_provider,
@ -91,7 +98,7 @@ def get_events(
f"Error receiving events from provider: {provider_name}:\n{repr(e)}"
)
else:
raise e
raise ReceivingEventsException(e)
events = [event for _, event_list in results.values() for event in event_list]
if sort_events:
@ -149,7 +156,7 @@ def latest_events(
f"Error receiving events from provider: {provider_name}:\n{repr(e)}"
)
else:
raise e
raise ReceivingEventsException(e)
events = [event for event_list in results.values() for event in event_list]
if sort_events:
@ -202,7 +209,7 @@ def next_event(
f"Error receiving events from provider: {provider_name}:\n{repr(e)}"
)
else:
raise e
raise ReceivingEventsException(e)
event: Optional[data.Event] = None
for candidate in results.values():
@ -258,7 +265,7 @@ def previous_event(
f"Error receiving events from provider: {provider_name}:\n{repr(e)}"
)
else:
raise e
raise ReceivingEventsException(e)
event: Optional[data.Event] = None
for candidate in results.values():

18
backend/moonstream/reporter.py 100644
Wyświetl plik

@ -0,0 +1,18 @@
import uuid
from humbug.consent import HumbugConsent
from humbug.report import HumbugReporter
from .settings import HUMBUG_REPORTER_BACKEND_TOKEN
session_id = str(uuid.uuid4())
client_id = "moonstream-backend"
reporter = HumbugReporter(
name="moonstream",
consent=HumbugConsent(True),
client_id=client_id,
session_id=session_id,
bugout_token=HUMBUG_REPORTER_BACKEND_TOKEN,
tags=[],
)

12
backend/moonstream/routes/address_info.py
Wyświetl plik

@ -3,14 +3,14 @@ from typing import Dict, List, Optional
from sqlalchemy.sql.expression import true
from fastapi import FastAPI, Depends, Query, HTTPException
from fastapi import FastAPI, Depends, Query
from fastapi.middleware.cors import CORSMiddleware
from moonstreamdb.db import yield_db_session
from sqlalchemy.orm import Session
from .. import actions
from .. import data
from ..middleware import BroodAuthMiddleware
from ..middleware import BroodAuthMiddleware, MoonstreamHTTPException
from ..settings import DOCS_TARGET_PATH, ORIGINS, DOCS_PATHS
from ..version import MOONSTREAM_VERSION
@ -73,15 +73,15 @@ async def addresses_labels_bulk_handler(
about known addresses.
"""
if limit > 100:
raise HTTPException(
raise MoonstreamHTTPException(
status_code=406, detail="The limit cannot exceed 100 addresses"
)
try:
addresses_response = actions.get_address_labels(
db_session=db_session, start=start, limit=limit, addresses=addresses
)
except Exception as err:
logger.error(f"Unable to get info about Ethereum addresses {err}")
raise HTTPException(status_code=500)
except Exception as e:
logger.error(f"Unable to get info about Ethereum addresses {e}")
raise MoonstreamHTTPException(status_code=500, internal_error=e)
return addresses_response

125
backend/moonstream/routes/streams.py
Wyświetl plik

@ -5,15 +5,16 @@ import logging
from typing import Dict, List, Optional
from bugout.data import BugoutResource
from fastapi import FastAPI, HTTPException, Request, Query, Depends
from fastapi import FastAPI, Request, Query, Depends
from fastapi.middleware.cors import CORSMiddleware
from moonstreamdb import db
from sqlalchemy.orm import Session
from .. import data
from ..middleware import BroodAuthMiddleware
from ..middleware import BroodAuthMiddleware, MoonstreamHTTPException
from ..providers import (
ReceivingEventsException,
event_providers,
get_events,
latest_events,
@ -121,17 +122,25 @@ async def stream_handler(
if q.strip() != "":
query = stream_queries.parse_query_string(q)
_, events = get_events(
db_session,
bc,
MOONSTREAM_DATA_JOURNAL_ID,
MOONSTREAM_ADMIN_ACCESS_TOKEN,
stream_boundary,
query,
user_subscriptions,
result_timeout=10.0,
raise_on_error=True,
)
try:
_, events = get_events(
db_session,
bc,
MOONSTREAM_DATA_JOURNAL_ID,
MOONSTREAM_ADMIN_ACCESS_TOKEN,
stream_boundary,
query,
user_subscriptions,
result_timeout=10.0,
raise_on_error=True,
)
except ReceivingEventsException as e:
logger.error("Error receiving events from provider")
raise MoonstreamHTTPException(status_code=500, internal_error=e)
except Exception as e:
logger.error("Unable to get events")
raise MoonstreamHTTPException(status_code=500, internal_error=e)
response = data.GetEventsResponse(stream_boundary=stream_boundary, events=events)
return response
@ -155,18 +164,26 @@ async def latest_events_handler(
if q.strip() != "":
query = stream_queries.parse_query_string(q)
events = latest_events(
db_session,
bc,
MOONSTREAM_DATA_JOURNAL_ID,
MOONSTREAM_ADMIN_ACCESS_TOKEN,
query,
1,
user_subscriptions,
result_timeout=6.0,
raise_on_error=True,
sort_events=True,
)
try:
events = latest_events(
db_session,
bc,
MOONSTREAM_DATA_JOURNAL_ID,
MOONSTREAM_ADMIN_ACCESS_TOKEN,
query,
1,
user_subscriptions,
result_timeout=6.0,
raise_on_error=True,
sort_events=True,
)
except ReceivingEventsException as e:
logger.error("Error receiving events from provider")
raise MoonstreamHTTPException(status_code=500, internal_error=e)
except Exception as e:
logger.error("Unable to get latest events")
raise MoonstreamHTTPException(status_code=500, internal_error=e)
return events
@ -203,17 +220,24 @@ async def next_event_handler(
if q.strip() != "":
query = stream_queries.parse_query_string(q)
event = next_event(
db_session,
bc,
MOONSTREAM_DATA_JOURNAL_ID,
MOONSTREAM_ADMIN_ACCESS_TOKEN,
stream_boundary,
query,
user_subscriptions,
result_timeout=6.0,
raise_on_error=True,
)
try:
event = next_event(
db_session,
bc,
MOONSTREAM_DATA_JOURNAL_ID,
MOONSTREAM_ADMIN_ACCESS_TOKEN,
stream_boundary,
query,
user_subscriptions,
result_timeout=6.0,
raise_on_error=True,
)
except ReceivingEventsException as e:
logger.error("Error receiving events from provider")
raise MoonstreamHTTPException(status_code=500, internal_error=e)
except Exception as e:
logger.error("Unable to get next events")
raise MoonstreamHTTPException(status_code=500, internal_error=e)
return event
@ -251,16 +275,23 @@ async def previous_event_handler(
if q.strip() != "":
query = stream_queries.parse_query_string(q)
event = previous_event(
db_session,
bc,
MOONSTREAM_DATA_JOURNAL_ID,
MOONSTREAM_ADMIN_ACCESS_TOKEN,
stream_boundary,
query,
user_subscriptions,
result_timeout=6.0,
raise_on_error=True,
)
try:
event = previous_event(
db_session,
bc,
MOONSTREAM_DATA_JOURNAL_ID,
MOONSTREAM_ADMIN_ACCESS_TOKEN,
stream_boundary,
query,
user_subscriptions,
result_timeout=6.0,
raise_on_error=True,
)
except ReceivingEventsException as e:
logger.error("Error receiving events from provider")
raise MoonstreamHTTPException(status_code=500, internal_error=e)
except Exception as e:
logger.error("Unable to get previous events")
raise MoonstreamHTTPException(status_code=500, internal_error=e)
return event

41
backend/moonstream/routes/subscriptions.py
Wyświetl plik

@ -6,12 +6,13 @@ from typing import Dict, List, Optional
from bugout.data import BugoutResource, BugoutResources
from bugout.exceptions import BugoutResponseException
from fastapi import FastAPI, HTTPException, Request, Form
from fastapi import FastAPI, Request, Form
from fastapi.middleware.cors import CORSMiddleware
from ..admin import subscription_types
from .. import data
from ..middleware import BroodAuthMiddleware
from ..middleware import BroodAuthMiddleware, MoonstreamHTTPException
from ..reporter import reporter
from ..settings import (
DOCS_TARGET_PATH,
DOCS_PATHS,
@ -77,7 +78,7 @@ async def add_subscription_handler(
]
if subscription_type_id not in available_subscription_type_ids:
raise HTTPException(
raise MoonstreamHTTPException(
status_code=404,
detail=f"Invalid subscription type: {subscription_type_id}.",
)
@ -99,10 +100,11 @@ async def add_subscription_handler(
application_id=MOONSTREAM_APPLICATION_ID,
resource_data=resource_data,
)
except BugoutResponseException as e:
raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail)
except Exception as e:
logger.error("Error creating subscription resource:")
logger.error(e)
raise HTTPException(status_code=500)
logger.error(f"Error creating subscription resource: {str(e)}")
raise MoonstreamHTTPException(status_code=500, internal_error=e)
return data.SubscriptionResourceData(
id=str(resource.id),
@ -123,14 +125,14 @@ async def delete_subscription_handler(request: Request, subscription_id: str):
"""
Delete subscriptions.
"""
token = request.state.token
try:
deleted_resource = bc.delete_resource(token=token, resource_id=subscription_id)
except BugoutResponseException as e:
raise HTTPException(status_code=e.status_code, detail=e.detail)
raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail)
except Exception as e:
raise HTTPException(status_code=500)
logger.error(f"Error deleting subscription: {str(e)}")
raise MoonstreamHTTPException(status_code=500, internal_error=e)
return data.SubscriptionResourceData(
id=str(deleted_resource.id),
@ -154,12 +156,14 @@ async def get_subscriptions_handler(request: Request) -> data.SubscriptionsListR
}
try:
resources: BugoutResources = bc.list_resources(token=token, params=params)
except BugoutResponseException as e:
raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail)
except Exception as e:
logger.error(
f"Error listing subscriptions for user ({request.user.id}) with token ({request.state.token})"
f"Error listing subscriptions for user ({request.user.id}) with token ({request.state.token}), error: {str(e)}"
)
logger.error(e)
raise HTTPException(status_code=500)
reporter.error_report(e)
raise MoonstreamHTTPException(status_code=500, internal_error=e)
return data.SubscriptionsListResponse(
subscriptions=[
@ -190,7 +194,6 @@ async def update_subscriptions_handler(
"""
Get user's subscriptions.
"""
token = request.state.token
update = {}
@ -210,9 +213,10 @@ async def update_subscriptions_handler(
).dict(),
)
except BugoutResponseException as e:
raise HTTPException(status_code=e.status_code, detail=e.detail)
raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail)
except Exception as e:
raise HTTPException(status_code=500)
logger.error(f"Error getting user subscriptions: {str(e)}")
raise MoonstreamHTTPException(status_code=500, internal_error=e)
return data.SubscriptionResourceData(
id=str(resource.id),
@ -238,9 +242,10 @@ async def list_subscription_types() -> data.SubscriptionTypesListResponse:
data.SubscriptionTypeResourceData.validate(resource.resource_data)
for resource in response.resources
]
except BugoutResponseException as e:
raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail)
except Exception as e:
logger.error("Error reading subscription types from Brood API:")
logger.error(e)
raise HTTPException(status_code=500)
logger.error(f"Error reading subscription types from Brood API: {str(e)}")
raise MoonstreamHTTPException(status_code=500, internal_error=e)
return data.SubscriptionTypesListResponse(subscription_types=results)

5
backend/moonstream/routes/txinfo.py
Wyświetl plik

@ -6,9 +6,7 @@ transactions, etc.) with side information and return objects that are better sui
end users.
"""
import logging
from typing import Dict, Optional
from sqlalchemy.sql.expression import true
from typing import Dict
from fastapi import FastAPI, Depends
from fastapi.middleware.cors import CORSMiddleware
@ -54,6 +52,7 @@ app.add_middleware(BroodAuthMiddleware, whitelist=whitelist_paths)
# TODO(zomglings): Factor out the enrichment logic into a separate action, because it may be useful
# independently from serving API calls (e.g. data processing).
# TODO(kompotkot): Re-organize function to be able handle each steps with exceptions.
@app.post(
"/ethereum_blockchain",
tags=["txinfo"],

53
backend/moonstream/routes/users.py
Wyświetl plik

@ -7,17 +7,18 @@ import uuid
from bugout.data import BugoutToken, BugoutUser, BugoutResource
from bugout.exceptions import BugoutResponseException
from fastapi import (
Body,
FastAPI,
Form,
HTTPException,
Request,
)
from fastapi.middleware.cors import CORSMiddleware
from .. import data
from ..middleware import BroodAuthMiddleware
from ..middleware import BroodAuthMiddleware, MoonstreamHTTPException
from ..settings import (
MOONSTREAM_APPLICATION_ID,
DOCS_TARGET_PATH,
@ -79,9 +80,9 @@ async def create_user_handler(
application_id=MOONSTREAM_APPLICATION_ID,
)
except BugoutResponseException as e:
raise HTTPException(status_code=e.status_code, detail=e.detail)
raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail)
except Exception as e:
raise HTTPException(status_code=500)
raise MoonstreamHTTPException(status_code=500, internal_error=e)
return user
@ -96,9 +97,9 @@ async def restore_password_handler(email: str = Form(...)) -> Dict[str, Any]:
try:
response = bc.restore_password(email=email)
except BugoutResponseException as e:
raise HTTPException(status_code=e.status_code, detail=e.detail)
raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail)
except Exception as e:
raise HTTPException(status_code=500)
raise MoonstreamHTTPException(status_code=500, internal_error=e)
return response
@ -109,9 +110,9 @@ async def reset_password_handler(
try:
response = bc.reset_password(reset_id=reset_id, new_password=new_password)
except BugoutResponseException as e:
raise HTTPException(status_code=e.status_code, detail=e.detail)
raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail)
except Exception as e:
raise HTTPException(status_code=500)
raise MoonstreamHTTPException(status_code=500, internal_error=e)
return response
@ -125,9 +126,9 @@ async def change_password_handler(
token=token, current_password=current_password, new_password=new_password
)
except BugoutResponseException as e:
raise HTTPException(status_code=e.status_code, detail=e.detail)
raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail)
except Exception as e:
raise HTTPException(status_code=500)
raise MoonstreamHTTPException(status_code=500, internal_error=e)
return user
@ -140,9 +141,9 @@ async def delete_user_handler(
try:
user = bc.delete_user(token=token, user_id=user.id, password=password)
except BugoutResponseException as e:
raise HTTPException(status_code=e.status_code, detail=e.detail)
raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail)
except Exception as e:
raise HTTPException(status_code=500)
raise MoonstreamHTTPException(status_code=500, internal_error=e)
return user
@ -157,11 +158,11 @@ async def login_handler(
application_id=MOONSTREAM_APPLICATION_ID,
)
except BugoutResponseException as e:
raise HTTPException(
raise MoonstreamHTTPException(
status_code=e.status_code, detail=f"Error from Brood API: {e.detail}"
)
except Exception as e:
raise HTTPException(status_code=500)
raise MoonstreamHTTPException(status_code=500, internal_error=e)
return token
@ -171,9 +172,9 @@ async def logout_handler(request: Request) -> uuid.UUID:
try:
token_id: uuid.UUID = bc.revoke_token(token=token)
except BugoutResponseException as e:
raise HTTPException(status_code=e.status_code, detail=e.detail)
raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail)
except Exception as e:
raise HTTPException(status_code=500)
raise MoonstreamHTTPException(status_code=500, internal_error=e)
return token_id
@ -203,9 +204,9 @@ async def set_onboarding_state(
)
except BugoutResponseException as e:
raise HTTPException(status_code=e.status_code, detail=e.detail)
raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail)
except Exception as e:
raise HTTPException(status_code=500)
raise MoonstreamHTTPException(status_code=500)
if (
resource.resource_data.get("is_complete") is None
@ -214,7 +215,7 @@ async def set_onboarding_state(
logger.error(
f"Resources did not return correct onboarding object. Resource id:{resource.id}"
)
raise HTTPException(status_code=500)
raise MoonstreamHTTPException(status_code=500)
result = data.OnboardingState(
is_complete=resource.resource_data.get("is_complete", False),
@ -238,10 +239,10 @@ async def get_onboarding_state(request: Request) -> data.OnboardingState:
else:
resource = create_onboarding_resource(token=token)
except BugoutResponseException as e:
raise HTTPException(status_code=e.status_code, detail=e.detail)
raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail)
except Exception as e:
raise HTTPException(status_code=500)
raise MoonstreamHTTPException(status_code=500)
if (
resource.resource_data.get("is_complete") is None
@ -250,7 +251,7 @@ async def get_onboarding_state(request: Request) -> data.OnboardingState:
logger.error(
f"Resources did not return correct onboarding object. Resource id:{resource.id}"
)
raise HTTPException(status_code=500)
raise MoonstreamHTTPException(status_code=500)
result = data.OnboardingState(
is_complete=resource.resource_data.get("is_complete", False),
steps=resource.resource_data.get("steps", {}),
@ -268,7 +269,7 @@ async def delete_onboarding_state(request: Request) -> data.OnboardingState:
timeout=BUGOUT_REQUEST_TIMEOUT_SECONDS,
)
if not response.resources:
raise HTTPException(status_code=404, detail="not found")
raise MoonstreamHTTPException(status_code=404, detail="not found")
if response.resources:
resource: BugoutResource = bc.delete_resource(
token=token,
@ -277,9 +278,9 @@ async def delete_onboarding_state(request: Request) -> data.OnboardingState:
)
except BugoutResponseException as e:
raise HTTPException(status_code=e.status_code, detail=e.detail)
raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail)
except Exception as e:
raise HTTPException(status_code=500)
raise MoonstreamHTTPException(status_code=500)
if (
resource.resource_data.get("is_complete") is None
@ -288,7 +289,7 @@ async def delete_onboarding_state(request: Request) -> data.OnboardingState:
logger.error(
f"Resources did not return correct onboarding object. Resource id:{resource.id}"
)
raise HTTPException(status_code=500)
raise MoonstreamHTTPException(status_code=500)
result = data.OnboardingState(
is_complete=resource.resource_data.get("is_complete", False),
steps=resource.resource_data.get("steps", {}),

2
backend/moonstream/settings.py
Wyświetl plik

@ -9,6 +9,8 @@ bugout_client = Bugout(brood_api_url=BUGOUT_BROOD_URL, spire_api_url=BUGOUT_SPIR
BUGOUT_REQUEST_TIMEOUT_SECONDS = 5
HUMBUG_REPORTER_BACKEND_TOKEN = os.environ.get("HUMBUG_REPORTER_BACKEND_TOKEN")
# Default value is "" instead of None so that mypy understands that MOONSTREAM_APPLICATION_ID is a string
MOONSTREAM_APPLICATION_ID = os.environ.get("MOONSTREAM_APPLICATION_ID", "")
if MOONSTREAM_APPLICATION_ID == "":

2
backend/requirements.txt
Wyświetl plik

@ -11,6 +11,7 @@ fastapi==0.66.0
h11==0.12.0
idna==3.2
jmespath==0.10.0
humbug==0.2.7
-e git+https://git@github.com/bugout-dev/moonstream.git@94135b054cabb9dc11b0a2406431619279979469#egg=moonstreamdb&subdirectory=db
mypy==0.910
mypy-extensions==0.4.3
@ -28,5 +29,6 @@ toml==0.10.2
tomli==1.0.4
types-python-dateutil==0.1.6
typing-extensions==3.10.0.0
types-requests==2.25.6
urllib3==1.26.6
uvicorn==0.14.0

3
backend/sample.env
Wyświetl plik

@ -5,6 +5,7 @@ export MOONSTREAM_DATA_JOURNAL_ID="<bugout_journal_id_to_store_blockchain_data>"
export MOONSTREAM_DB_URI="postgresql://<username>:<password>@<db_host>:<db_port>/<db_name>"
export MOONSTREAM_POOL_SIZE=0
export MOONSTREAM_ADMIN_ACCESS_TOKEN="<Access token to application resources>"
export AWS_S3_SMARTCONTRACT_BUCKET=""
export AWS_S3_SMARTCONTRACT_BUCKET="<AWS S3 bucket to store smart contracts>"
export BUGOUT_BROOD_URL="https://auth.bugout.dev"
export BUGOUT_SPIRE_URL="https://spire.bugout.dev"
export HUMBUG_REPORTER_BACKEND_TOKEN="<Bugout Humbug token for crash reports>"

11
backend/setup.py
Wyświetl plik

@ -10,7 +10,16 @@ setup(
name="moonstream",
version=MOONSTREAM_VERSION,
packages=find_packages(),
install_requires=["boto3", "bugout >= 0.1.17", "fastapi", "python-dateutil", "uvicorn", "types-python-dateutil"],
install_requires=[
"boto3",
"bugout >= 0.1.17",
"fastapi",
"humbug>=0.2.7",
"python-dateutil",
"uvicorn",
"types-python-dateutil",
"types-requests",
],
extras_require={
"dev": ["black", "mypy"],
"distribute": ["setuptools", "twine", "wheel"],

37
crawlers/ethtxpool/main.go
Wyświetl plik

@ -10,23 +10,20 @@ import (
"encoding/json"
"flag"
"fmt"
"math/big"
"os"
"time"
humbug "github.com/bugout-dev/humbug/go/pkg"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/rpc"
"github.com/google/uuid"
)
// Generate humbug client to be able write data in Bugout journal.
func humbugClientFromEnv() (*humbug.HumbugReporter, error) {
clientID := os.Getenv("ETHTXPOOL_HUMBUG_CLIENT_ID")
humbugToken := os.Getenv("ETHTXPOOL_HUMBUG_TOKEN")
sessionID := uuid.New().String()
// Generate humbug client
func humbugClient(sessionID string, clientID string, humbugToken string) (*humbug.HumbugReporter, error) {
consent := humbug.CreateHumbugConsent(humbug.True)
reporter, err := humbug.CreateHumbugReporter(consent, clientID, sessionID, humbugToken)
return reporter, err
@ -124,11 +121,6 @@ func PollTxpoolContent(gethClient *rpc.Client, interval int, reporter *humbug.Hu
continue
}
// TODO(kompotkot, zomglings): Humbug API (on Spire) support bulk publication of reports. We should modify
// Humbug go client to use the bulk publish endpoint. Currently, if we have to publish all transactions
// pending in txpool, we *will* get rate limited. We may want to consider adding a publisher to the
// Humbug go client that can listen on a channel and will handle rate limiting, bulk publication etc. itself
// (without user having to worry about it).
ReportTitle := "Ethereum: Pending transaction: " + transactionHash.String()
ReportTags := []string{
"hash:" + transactionHash.String(),
@ -138,6 +130,7 @@ func PollTxpoolContent(gethClient *rpc.Client, interval int, reporter *humbug.Hu
fmt.Sprintf("max_priority_fee_per_gas:%d", pendingTx.Transaction.MaxPriorityFeePerGas.ToInt()),
fmt.Sprintf("max_fee_per_gas:%d", pendingTx.Transaction.MaxFeePerGas.ToInt()),
fmt.Sprintf("gas:%d", pendingTx.Transaction.Gas),
fmt.Sprintf("value:%d", new(big.Float).Quo(new(big.Float).SetInt(transaction.Value.ToInt()), big.NewFloat(params.Ether))),
"crawl_type:ethereum_txpool",
}
report := humbug.Report{
@ -188,6 +181,23 @@ func main() {
flag.IntVar(&intervalSeconds, "interval", 1, "Number of seconds to wait between RPC calls to query the transaction pool (default: 1)")
flag.Parse()
sessionID := uuid.New().String()
// Humbug crash client to collect errors
crashReporter, err := humbugClient(sessionID, "moonstream-crawlers", os.Getenv("HUMBUG_REPORTER_CRAWLERS_TOKEN"))
if err != nil {
panic(fmt.Sprintf("Invalid Humbug Crash configuration: %s", err.Error()))
}
crashReporter.Publish(humbug.SystemReport())
defer func() {
message := recover()
if message != nil {
fmt.Printf("Error: %s\n", message)
crashReporter.Publish(humbug.PanicReport(message))
}
}()
// Set connection with Ethereum blockchain via geth
gethClient, err := rpc.Dial(gethConnectionString)
if err != nil {
@ -195,7 +205,8 @@ func main() {
}
defer gethClient.Close()
reporter, err := humbugClientFromEnv()
// Humbug client to be able write data in Bugout journal
reporter, err := humbugClient(sessionID, os.Getenv("ETHTXPOOL_HUMBUG_CLIENT_ID"), os.Getenv("ETHTXPOOL_HUMBUG_TOKEN"))
if err != nil {
panic(fmt.Sprintf("Invalid Humbug configuration: %s", err.Error()))
}

1
crawlers/ethtxpool/sample.env
Wyświetl plik

@ -1,2 +1,3 @@
export ETHTXPOOL_HUMBUG_CLIENT_ID="<client id for the crawling machine>"
export ETHTXPOOL_HUMBUG_TOKEN="<Generate an integration and a Humbug token from https://bugout.dev/account/teams>"
export HUMBUG_REPORTER_CRAWLERS_TOKEN="<Bugout Humbug token for crash reports>"

7
crawlers/mooncrawl/mooncrawl/__init__.py
Wyświetl plik

@ -0,0 +1,7 @@
from .reporter import reporter
from .version import MOONCRAWL_VERSION
# Reporting
reporter.tags.append(f"version:{MOONCRAWL_VERSION}")
reporter.system_report(publish=True)
reporter.setup_excepthook(publish=True)

2
crawlers/mooncrawl/mooncrawl/ethcrawler.py
Wyświetl plik

@ -48,7 +48,7 @@ def yield_blocks_numbers_lists(
print(
"Wrong format provided, expected {bottom_block}-{top_block}, as ex. 105-340"
)
return
raise Exception
starting_block = max(input_start_block, input_end_block)
ending_block = min(input_start_block, input_end_block)

62
crawlers/mooncrawl/mooncrawl/etherscan.py
Wyświetl plik

@ -1,24 +1,26 @@
import argparse
import codecs
import csv
from dataclasses import dataclass
from datetime import datetime
import json
import logging
import os
import sys
import time
from datetime import datetime
from typing import Any, List, Optional, Dict
from dataclasses import dataclass
import csv
import codecs
import json
import os
import boto3 # type: ignore
from moonstreamdb.db import yield_db_session_ctx
from moonstreamdb.models import EthereumAddress, EthereumLabel
import requests
from sqlalchemy.orm import Session
from sqlalchemy.sql.expression import text
from .version import MOONCRAWL_VERSION
from .settings import MOONSTREAM_ETHERSCAN_TOKEN
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
if MOONSTREAM_ETHERSCAN_TOKEN is None:
raise Exception("MOONSTREAM_ETHERSCAN_TOKEN environment variable must be set")
@ -66,21 +68,16 @@ def get_address_id(db_session: Session, contract_address: str) -> int:
if id is not None:
return id[0]
latest_address_id = (
db_session.query(EthereumAddress.id).order_by(text("id desc")).limit(1).one()
)[0]
id = latest_address_id + 1
smart_contract = EthereumAddress(
id=id,
address=contract_address,
)
try:
db_session.add(smart_contract)
db_session.commit()
except:
return smart_contract.id
except Exception as e:
db_session.rollback()
return id
raise e
def crawl_step(db_session: Session, contract: VerifiedSmartContract, crawl_url: str):
@ -112,22 +109,27 @@ def crawl_step(db_session: Session, contract: VerifiedSmartContract, crawl_url:
object_key = f"/etherscan/v1/{contract.address}.json"
push_to_bucket(contract_info, object_key)
eth_address_id = get_address_id(db_session, contract.address)
eth_label = EthereumLabel(
label=ETHERSCAN_SMARTCONTRACTS_LABEL_NAME,
address_id=eth_address_id,
label_data={
"object_uri": f"s3://{bucket}/{object_key}",
"name": contract.name,
"tx_hash": contract.tx_hash,
},
)
try:
db_session.add(eth_label)
db_session.commit()
except:
db_session.rollback()
eth_address_id = get_address_id(db_session, contract.address)
eth_label = EthereumLabel(
label=ETHERSCAN_SMARTCONTRACTS_LABEL_NAME,
address_id=eth_address_id,
label_data={
"object_uri": f"s3://{bucket}/{object_key}",
"name": contract.name,
"tx_hash": contract.tx_hash,
},
)
try:
db_session.add(eth_label)
db_session.commit()
except Exception as e:
db_session.rollback()
raise e
except Exception as e:
logger.error(
f"Failed to add addresss label ${contract.address} to database\n{str(e)}"
)
def crawl(

18
crawlers/mooncrawl/mooncrawl/reporter.py 100644
Wyświetl plik

@ -0,0 +1,18 @@
import uuid
from humbug.consent import HumbugConsent
from humbug.report import HumbugReporter
from .settings import HUMBUG_REPORTER_CRAWLERS_TOKEN
session_id = str(uuid.uuid4())
client_id = "moonstream-crawlers"
reporter = HumbugReporter(
name="moonstream-crawlers",
consent=HumbugConsent(True),
client_id=client_id,
session_id=session_id,
bugout_token=HUMBUG_REPORTER_CRAWLERS_TOKEN,
tags=[],
)

6
crawlers/mooncrawl/mooncrawl/settings.py
Wyświetl plik

@ -1,5 +1,9 @@
import os
# Bugout
HUMBUG_REPORTER_CRAWLERS_TOKEN = os.environ.get("HUMBUG_REPORTER_CRAWLERS_TOKEN")
# Geth
MOONSTREAM_IPC_PATH = os.environ.get("MOONSTREAM_IPC_PATH", None)
MOONSTREAM_CRAWL_WORKERS = 4
@ -12,5 +16,5 @@ except:
f"Could not parse MOONSTREAM_CRAWL_WORKERS as int: {MOONSTREAM_CRAWL_WORKERS_RAW}"
)
# Etherscan
MOONSTREAM_ETHERSCAN_TOKEN = os.environ.get("MOONSTREAM_ETHERSCAN_TOKEN")

1
crawlers/mooncrawl/sample.env
Wyświetl plik

@ -6,3 +6,4 @@ export MOONSTREAM_ETHERSCAN_TOKEN="<Token for etherscan>"
export AWS_S3_SMARTCONTRACT_BUCKET="<AWS S3 bucket for smart contracts>"
export MOONSTREAM_HUMBUG_TOKEN="<Token for crawlers store data via Humbug>"
export COINMARKETCAP_API_KEY="<API key to parse conmarketcap>"
export HUMBUG_REPORTER_CRAWLERS_TOKEN="<Bugout Humbug token for crash reports>"

4
crawlers/mooncrawl/setup.py
Wyświetl plik

@ -1,6 +1,5 @@
from setuptools import find_packages, setup
from mooncrawl.version import MOONCRAWL_VERSION
long_description = ""
with open("README.md") as ifp:
@ -8,7 +7,7 @@ with open("README.md") as ifp:
setup(
name="mooncrawl",
version=MOONCRAWL_VERSION,
version="0.0.3",
author="Bugout.dev",
author_email="engineers@bugout.dev",
license="Apache License 2.0",
@ -34,6 +33,7 @@ setup(
zip_safe=False,
install_requires=[
"moonstreamdb @ git+https://git@github.com/bugout-dev/moonstream.git@39d2b8e36a49958a9ae085ec2cc1be3fc732b9d0#egg=moonstreamdb&subdirectory=db",
"humbug",
"python-dateutil",
"requests",
"tqdm",

49
db/alembic/versions/ecb7817db377_add_opensea_state_table_and_add_index_.py 100644
Wyświetl plik

@ -0,0 +1,49 @@
"""Add opensea state table and add index by label_data ->> name
Revision ID: ecb7817db377
Revises: ea8185bd24c7
Create Date: 2021-08-31 17:44:24.139028
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = "ecb7817db377"
down_revision = "ea8185bd24c7"
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.create_table(
"opensea_crawler_state",
sa.Column("id", sa.Integer(), nullable=False),
sa.Column("query", sa.Text(), nullable=False),
sa.Column(
"crawled_at",
sa.DateTime(timezone=True),
server_default=sa.text("TIMEZONE('utc', statement_timestamp())"),
nullable=False,
),
sa.Column("total_count", sa.Integer(), nullable=False),
sa.PrimaryKeyConstraint("id", name=op.f("pk_opensea_crawler_state")),
sa.UniqueConstraint("id", name=op.f("uq_opensea_crawler_state_id")),
)
op.execute(
"ALTER TABLE ethereum_labels DROP CONSTRAINT IF EXISTS uq_ethereum_labels_label"
)
op.execute(
f"CREATE INDEX idx_ethereum_labels_opensea_nft_name ON ethereum_labels((label_data->>'name')) where label='opensea_nft';"
)
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.drop_table("opensea_crawler_state")
op.drop_index("idx_ethereum_labels_opensea_nft_name")
# ### end Alembic commands ###

21
db/moonstreamdb/models.py
Wyświetl plik

@ -11,7 +11,6 @@ from sqlalchemy import (
Numeric,
Text,
VARCHAR,
UniqueConstraint,
)
from sqlalchemy.dialects.postgresql import JSONB, UUID
from sqlalchemy.sql import expression
@ -136,7 +135,6 @@ class EthereumLabel(Base): # type: ignore
"""
__tablename__ = "ethereum_labels"
__table_args__ = (UniqueConstraint("label", "address_id"),)
id = Column(
UUID(as_uuid=True),
@ -212,3 +210,22 @@ class ESDEventSignature(Base): # type: ignore
created_at = Column(
DateTime(timezone=True), server_default=utcnow(), nullable=False
)
class OpenSeaCrawlingState(Base): # type: ignore
"""
Model for control opeansea crawling state.
"""
__tablename__ = "opensea_crawler_state"
id = Column(Integer, primary_key=True, unique=True, nullable=False)
query = Column(Text, nullable=False)
crawled_at = Column(
DateTime(timezone=True),
server_default=utcnow(),
onupdate=utcnow(),
nullable=False,
)
total_count = Column(Integer, nullable=False)

64
frontend/pages/index.js
Wyświetl plik

@ -24,12 +24,15 @@ import {
} from "@chakra-ui/react";
import dynamic from "next/dynamic";
import useUser from "../src/core/hooks/useUser";
import useAnalytics from "../src/core/hooks/useAnalytics";
import useModals from "../src/core/hooks/useModals";
import useRouter from "../src/core/hooks/useRouter";
import { MIXPANEL_PROPS } from "../src/core/providers/AnalyticsProvider/constants";
import {
MIXPANEL_PROPS,
MIXPANEL_EVENTS,
} from "../src/core/providers/AnalyticsProvider/constants";
import UIContext from "../src/core/providers/UIProvider/context";
import { AWS_ASSETS_PATH } from "../src/core/constants";
import mixpanel from "mixpanel-browser";
const SplitWithImage = dynamic(
() => import("../src/components/SplitWithImage"),
{
@ -105,7 +108,6 @@ const Homepage = () => {
const router = useRouter();
const { isInit } = useUser();
const { MIXPANEL_EVENTS, track } = useAnalytics();
const { toggleModal } = useModals();
const [
isLargerThan720px,
@ -379,27 +381,30 @@ const Homepage = () => {
label: "Crypto trader",
link: "/#cryptoTrader",
onClick: () => {
track(`${MIXPANEL_EVENTS.BUTTON_CLICKED}`, {
[`${MIXPANEL_PROPS.BUTTON_CLICKED}`]: `scroll to CryptoTrader`,
});
mixpanel.get_distinct_id() &&
mixpanel.track(`${MIXPANEL_EVENTS.BUTTON_CLICKED}`, {
[`${MIXPANEL_PROPS.BUTTON_NAME}`]: `scroll to CryptoTrader`,
});
},
}}
button2={{
label: "Algorithmic Fund",
link: "/#algoFund",
onClick: () => {
track(`${MIXPANEL_EVENTS.BUTTON_CLICKED}`, {
[`${MIXPANEL_PROPS.BUTTON_CLICKED}`]: `scroll to AlgoFund`,
});
mixpanel.get_distinct_id() &&
mixpanel.track(`${MIXPANEL_EVENTS.BUTTON_CLICKED}`, {
[`${MIXPANEL_PROPS.BUTTON_NAME}`]: `scroll to AlgoFund`,
});
},
}}
button3={{
label: "Developer",
link: "/#smartDeveloper",
onClick: () => {
track(`${MIXPANEL_EVENTS.BUTTON_CLICKED}`, {
[`${MIXPANEL_PROPS.BUTTON_CLICKED}`]: `scroll to Developer`,
});
mixpanel.get_distinct_id() &&
mixpanel.track(`${MIXPANEL_EVENTS.BUTTON_CLICKED}`, {
[`${MIXPANEL_PROPS.BUTTON_NAME}`]: `scroll to Developer`,
});
},
}}
/>
@ -417,9 +422,10 @@ const Homepage = () => {
cta={{
label: "I want early access!",
onClick: () => {
track(`${MIXPANEL_EVENTS.BUTTON_CLICKED}`, {
[`${MIXPANEL_PROPS.BUTTON_CLICKED}`]: `Early access CTA: Crypto trader`,
});
mixpanel.get_distinct_id() &&
mixpanel.track(`${MIXPANEL_EVENTS.BUTTON_CLICKED}`, {
[`${MIXPANEL_PROPS.BUTTON_NAME}`]: `Early access CTA: Crypto trader`,
});
toggleModal("hubspot-trader");
},
}}
@ -464,9 +470,10 @@ const Homepage = () => {
cta={{
label: "I want early access!",
onClick: () => {
track(`${MIXPANEL_EVENTS.BUTTON_CLICKED}`, {
[`${MIXPANEL_PROPS.BUTTON_CLICKED}`]: `Early access CTA: Algo fund`,
});
mixpanel.get_distinct_id() &&
mixpanel.track(`${MIXPANEL_EVENTS.BUTTON_CLICKED}`, {
[`${MIXPANEL_PROPS.BUTTON_NAME}`]: `Early access CTA: Algo fund`,
});
toggleModal("hubspot-fund");
},
}}
@ -509,9 +516,10 @@ const Homepage = () => {
cta={{
label: "I want early access!",
onClick: () => {
track(`${MIXPANEL_EVENTS.BUTTON_CLICKED}`, {
[`${MIXPANEL_PROPS.BUTTON_CLICKED}`]: `Early access CTA: developer`,
});
mixpanel.get_distinct_id() &&
mixpanel.track(`${MIXPANEL_EVENTS.BUTTON_CLICKED}`, {
[`${MIXPANEL_PROPS.BUTTON_NAME}`]: `Early access CTA: developer`,
});
toggleModal("hubspot-developer");
},
}}
@ -520,9 +528,10 @@ const Homepage = () => {
network: "github",
label: "See our github",
onClick: () => {
track(`${MIXPANEL_EVENTS.BUTTON_CLICKED}`, {
[`${MIXPANEL_PROPS.BUTTON_CLICKED}`]: `Github link in landing page`,
});
mixpanel.get_distinct_id() &&
mixpanel.track(`${MIXPANEL_EVENTS.BUTTON_CLICKED}`, {
[`${MIXPANEL_PROPS.BUTTON_NAME}`]: `Github link in landing page`,
});
},
}}
elementName={"element3"}
@ -568,9 +577,10 @@ const Homepage = () => {
colorScheme="suggested"
id="test"
onClick={() => {
track(`${MIXPANEL_EVENTS.BUTTON_CLICKED}`, {
[`${MIXPANEL_PROPS.BUTTON_CLICKED}`]: `Join our discord`,
});
mixpanel.get_distinct_id() &&
mixpanel.track(`${MIXPANEL_EVENTS.BUTTON_CLICKED}`, {
[`${MIXPANEL_PROPS.BUTTON_NAME}`]: `Join our discord`,
});
toggleModal("hubspot");
}}
>

14
frontend/src/AppContext.js
Wyświetl plik

@ -13,13 +13,13 @@ const AppContext = (props) => {
return (
<UserProvider>
<ModalProvider>
<AnalyticsProvider>
<StripeProvider>
<ChakraProvider theme={theme}>
<UIProvider>{props.children}</UIProvider>
</ChakraProvider>
</StripeProvider>
</AnalyticsProvider>
<StripeProvider>
<ChakraProvider theme={theme}>
<UIProvider>
<AnalyticsProvider>{props.children}</AnalyticsProvider>
</UIProvider>
</ChakraProvider>
</StripeProvider>
</ModalProvider>
</UserProvider>
);

2
frontend/src/components/Footer.js
Wyświetl plik

@ -6,7 +6,7 @@ import RouterLink from "next/link";
const ICONS = [
{
social: "discord",
link: "https://discord.gg/FetK5BxD",
link: "https://discord.gg/K56VNUQGvA",
},
{ social: "twit", link: "https://twitter.com/moonstreamto" },

6
frontend/src/components/Scrollable.js
Wyświetl plik

@ -1,13 +1,13 @@
import { Flex, Box } from "@chakra-ui/react";
import React, { useEffect, useRef, useState } from "react";
import { useRouter, useAnalytics } from "../core/hooks";
import { useRouter } from "../core/hooks";
import mixpanel from "mixpanel-browser";
const Scrollable = (props) => {
const scrollerRef = useRef();
const router = useRouter();
const [path, setPath] = useState();
const [scrollDepth, setScrollDepth] = useState(0);
const { mixpanel, isLoaded } = useAnalytics();
const getScrollPrecent = ({ currentTarget }) => {
const scroll_level =
@ -20,7 +20,7 @@ const Scrollable = (props) => {
const currentScroll = Math.ceil(getScrollPrecent(e) / 10);
if (currentScroll > scrollDepth) {
setScrollDepth(currentScroll);
isLoaded &&
mixpanel.get_distinct_id() &&
mixpanel.people.increment({
[`Scroll depth at: ${router.nextRouter.pathname}`]: currentScroll,
});

1
frontend/src/core/hooks/index.js
Wyświetl plik

@ -1,5 +1,4 @@
export { queryCacheProps as hookCommon } from "./hookCommon";
export { default as useAnalytics } from "./useAnalytics";
export { default as useAuthResultHandler } from "./useAuthResultHandler";
export { default as useChangePassword } from "./useChangePassword";
export { default as useClientID } from "./useClientID";

38
frontend/src/core/hooks/useAnalytics.js
Wyświetl plik

@ -1,38 +0,0 @@
import AnalyticsContext from "../providers/AnalyticsProvider/context";
import { useContext } from "react";
import { useState, useEffect, useCallback } from "react";
const useAnalytics = () => {
const { mixpanel, isLoaded, MIXPANEL_EVENTS, MIXPANEL_PROPS } =
useContext(AnalyticsContext);
const [trackProps, setTrackProps] = useState({
event: null,
props: null,
queued: false,
});
const track = useCallback((e, props) => {
setTrackProps({ event: e, props: props, queued: true });
}, []);
useEffect(() => {
if (isLoaded && trackProps.queued === true) {
mixpanel.track(trackProps.event, trackProps.props);
setTrackProps({ event: null, props: null, queued: false });
}
}, [isLoaded, mixpanel, trackProps]);
const withTracking = (fn, event, props) => {
track(event, props);
return fn;
};
return {
mixpanel,
isLoaded,
track,
MIXPANEL_PROPS,
MIXPANEL_EVENTS,
withTracking,
};
};
export default useAnalytics;

16
frontend/src/core/hooks/useLogin.js
Wyświetl plik

@ -1,7 +1,6 @@
import { useMutation } from "react-query";
import { useToast, useUser, useInviteAccept } from ".";
import { AuthService } from "../services";
import { useAnalytics } from ".";
const LOGIN_TYPES = {
MANUAL: true,
@ -10,7 +9,6 @@ const LOGIN_TYPES = {
const useLogin = (loginType) => {
const { getUser } = useUser();
const toast = useToast();
const analytics = useAnalytics();
const { inviteAccept } = useInviteAccept();
const {
mutate: login,
@ -34,20 +32,6 @@ const useLogin = (loginType) => {
inviteAccept(invite_code);
}
getUser();
if (analytics.isLoaded) {
analytics.mixpanel.people.set_once({
[`${analytics.MIXPANEL_EVENTS.FIRST_LOGIN_DATE}`]:
new Date().toISOString(),
});
analytics.mixpanel.people.set({
[`${analytics.MIXPANEL_EVENTS.LAST_LOGIN_DATE}`]:
new Date().toISOString(),
});
analytics.mixpanel.track(
`${analytics.MIXPANEL_EVENTS.USER_LOGS_IN}`,
{}
);
}
}
},
onError: (error) => {

9
frontend/src/core/hooks/useLogout.js
Wyświetl plik

@ -1,21 +1,14 @@
import { useCallback, useContext } from "react";
import { useMutation, useQueryClient } from "react-query";
import { useUser, useRouter, useAnalytics } from ".";
import { useUser, useRouter } from ".";
import UIContext from "../providers/UIProvider/context";
import { AuthService } from "../services";
const useLogout = () => {
const { setLoggingOut } = useContext(UIContext);
const router = useRouter();
const analytics = useAnalytics();
const { mutate: revoke } = useMutation(AuthService.revoke, {
onSuccess: () => {
if (analytics.isLoaded) {
analytics.mixpanel.track(
`${analytics.MIXPANEL_EVENTS.USER_LOGS_OUT}`,
{}
);
}
localStorage.removeItem("MOONSTREAM_ACCESS_TOKEN");
cache.clear();
setUser(null);

16
frontend/src/core/hooks/useSignUp.js
Wyświetl plik

@ -1,16 +1,18 @@
import { useContext } from "react";
import { useMutation } from "react-query";
import { AuthService } from "../services";
import { useUser, useToast, useInviteAccept, useRouter, useAnalytics } from ".";
import { useUser, useToast, useInviteAccept, useRouter } from ".";
import UIContext from "../providers/UIProvider/context";
import mixpanel from "mixpanel-browser";
import { MIXPANEL_EVENTS } from "../providers/AnalyticsProvider/constants";
const useSignUp = (source) => {
const ui = useContext(UIContext);
const router = useRouter();
const { getUser } = useUser();
const toast = useToast();
const { inviteAccept } = useInviteAccept();
const analytics = useAnalytics();
const {
mutate: signUp,
@ -26,11 +28,11 @@ const useSignUp = (source) => {
inviteAccept(invite_code);
}
if (analytics.isLoaded) {
analytics.mixpanel.track(
`${analytics.MIXPANEL_EVENTS.CONVERT_TO_USER}`,
{ full_url: router.nextRouter.asPath, code: source }
);
if (mixpanel.get_distinct_id()) {
mixpanel.track(`${MIXPANEL_EVENTS.CONVERT_TO_USER}`, {
full_url: router.nextRouter.asPath,
code: source,
});
}
getUser();
ui.setisOnboardingComplete(false);

19
frontend/src/core/hooks/useToast.js
Wyświetl plik

@ -1,21 +1,18 @@
import { useToast as useChakraToast, Box } from "@chakra-ui/react";
import React, { useCallback } from "react";
import { useAnalytics } from ".";
import mixpanel from "mixpanel-browser";
import { MIXPANEL_EVENTS } from "../providers/AnalyticsProvider/constants";
const useToast = () => {
const chakraToast = useChakraToast();
const analytics = useAnalytics();
const toast = useCallback(
(message, type) => {
if (analytics.isLoaded && type === "error") {
analytics.mixpanel.track(
`${analytics.MIXPANEL_EVENTS.TOAST_ERROR_DISPLAYED}`,
{
status: message?.response?.status,
detail: message?.response?.data.detail,
}
);
if (mixpanel.get_distinct_id() && type === "error") {
mixpanel.track(`${MIXPANEL_EVENTS.TOAST_ERROR_DISPLAYED}`, {
status: message?.response?.status,
detail: message?.response?.data.detail,
});
}
const background = type === "error" ? "unsafe.500" : "suggested.500";
const userMessage =
@ -43,7 +40,7 @@ const useToast = () => {
),
});
},
[chakraToast, analytics]
[chakraToast]
);
return toast;

15
frontend/src/core/providers/AnalyticsProvider/constants.js
Wyświetl plik

@ -8,6 +8,12 @@ export const MIXPANEL_PROPS = {
USER_SPECIALITY: "user speciality",
};
export const MIXPANEL_GROUPS = {
DEVELOPERS: "developers",
TRADERS: "traders",
FUND: "funds",
};
export const MIXPANEL_EVENTS = {
FIRST_LOGIN_DATE: "First login date",
LAST_LOGIN_DATE: "Last login date",
@ -20,7 +26,14 @@ export const MIXPANEL_EVENTS = {
PAGEVIEW: "Page view",
PRICING_PLAN_CLICKED: "Pricing Plan clicked",
BUTTON_CLICKED: "Button clicked",
LEFT_PAGE: "Left page",
BEACON: "beacon",
ONBOARDING_COMPLETED: "Onbording complete",
SESSIONS_COUNT: "Sessions Counter",
ONBOARDING_STEP: "Onboarding step",
ONBOARDING_STATE: "Onboarding state",
TIMES_VISITED: "Page visit times",
FORM_SUBMITTED: "form submitted",
PAGEVIEW_DURATION: "Time spent on page",
};
export default MIXPANEL_EVENTS;

174
frontend/src/core/providers/AnalyticsProvider/index.js
Wyświetl plik

@ -1,66 +1,161 @@
import React, { useEffect, useState } from "react";
import React, { useContext, useEffect, useState } from "react";
import mixpanel from "mixpanel-browser";
import AnalyticsContext from "./context";
import { useClientID, useUser, useRouter } from "../../hooks";
import { MIXPANEL_EVENTS, MIXPANEL_PROPS } from "./constants";
import UIContext from "../UIProvider/context";
const AnalyticsProvider = ({ children }) => {
const clientID = useClientID();
const analytics = process.env.NEXT_PUBLIC_MIXPANEL_TOKEN;
const { user } = useUser();
const [isLoaded, setIsLoaded] = useState(false);
const { user, isInit } = useUser();
const [isMixpanelReady, setIsLoaded] = useState(false);
const router = useRouter();
const ui = useContext(UIContext);
// ********** OBOARDING STATE **************
useEffect(() => {
if (ui.onboardingState && isMixpanelReady) {
mixpanel.people.set(MIXPANEL_EVENTS.ONBOARDING_STATE, {
state: { ...ui.onboardingState },
});
}
}, [ui.onboardingState, isMixpanelReady]);
useEffect(() => {
if (ui.isOnboardingComplete && isMixpanelReady && user) {
mixpanel.people.set(MIXPANEL_EVENTS.ONBOARDING_COMPLETED, true);
}
}, [ui.isOnboardingComplete, isMixpanelReady, user]);
// ********** ONBOARDING STEP and TIMING **************
const [previousOnboardingStep, setPreviousOnboardingStep] = useState(false);
useEffect(() => {
if (isMixpanelReady && router.nextRouter.pathname === "/welcome") {
if (!previousOnboardingStep) {
mixpanel.time_event(MIXPANEL_EVENTS.ONBOARDING_STEP);
setPreviousOnboardingStep(ui.onboardingStep);
}
if (
previousOnboardingStep &&
previousOnboardingStep !== ui.onboardingStep
) {
mixpanel.track(MIXPANEL_EVENTS.ONBOARDING_STEP, {
step: previousOnboardingStep,
isBeforeUnload: false,
});
setPreviousOnboardingStep(false);
}
} else if (previousOnboardingStep) {
mixpanel.track(MIXPANEL_EVENTS.ONBOARDING_STEP, {
step: previousOnboardingStep,
isBeforeUnload: false,
});
setPreviousOnboardingStep(false);
}
}, [
previousOnboardingStep,
ui.onboardingStep,
isMixpanelReady,
router.nextRouter.pathname,
]);
// ********** PING_PONG **************
useEffect(() => {
let durationSeconds = 0;
const intervalId =
isLoaded &&
isMixpanelReady &&
setInterval(() => {
durationSeconds = durationSeconds + 1;
durationSeconds = durationSeconds + 30;
mixpanel.track(
MIXPANEL_EVENTS.LEFT_PAGE,
MIXPANEL_EVENTS.BEACON,
{
duration_seconds: durationSeconds,
url: router.nextRouter.pathname,
query: router.query,
pathParams: router.params,
},
{ transport: "sendBeacon" }
);
}, 30000);
return () => clearInterval(intervalId);
// eslint-disable-next-line
}, [isLoaded]);
}, [isMixpanelReady, router.nextRouter.pathname]);
// ********** TIME SPENT ON PATH**************
const [previousPathname, setPreviousPathname] = useState(false);
useEffect(() => {
isLoaded &&
if (isMixpanelReady) {
if (!previousPathname) {
mixpanel.time_event(MIXPANEL_EVENTS.PAGEVIEW_DURATION);
setPreviousPathname(router.nextRouter.pathname);
}
if (previousPathname && previousPathname !== router.nextRouter.pathname) {
mixpanel.track(MIXPANEL_EVENTS.PAGEVIEW_DURATION, {
url: previousPathname,
isBeforeUnload: false,
});
setPreviousPathname(false);
}
}
}, [router.nextRouter.pathname, previousPathname, isMixpanelReady]);
// ********** PAGES VIEW **************
useEffect(() => {
if (isMixpanelReady && ui.sessionId && router.nextRouter.pathname) {
mixpanel.track(MIXPANEL_EVENTS.PAGEVIEW, {
url: router.nextRouter.pathname,
query: router.query,
pathParams: router.params,
sessionID: ui.sessionId,
});
}, [router.nextRouter.pathname, router.query, router.params, isLoaded]);
useEffect(() => {
try {
mixpanel.init(analytics, {
api_host: "https://api.mixpanel.com",
loaded: () => {
setIsLoaded(true);
mixpanel.identify(clientID);
},
mixpanel.people.increment([
`${MIXPANEL_EVENTS.TIMES_VISITED} ${router.nextRouter.pathname}`,
]);
}
const urlForUnmount = router.nextRouter.pathname;
const closeListener = () => {
mixpanel.track(MIXPANEL_EVENTS.PAGEVIEW_DURATION, {
url: urlForUnmount,
isBeforeUnload: true,
});
} catch (error) {
console.warn("loading mixpanel failed:", error);
};
window.addEventListener("beforeunload", closeListener);
//cleanup function fires on useEffect unmount
//https://reactjs.org/docs/hooks-effect.html
return () => {
window.removeEventListener("beforeunload", closeListener);
};
}, [router.nextRouter.pathname, isMixpanelReady, ui.sessionId]);
// ********** SESSION STATE **************
useEffect(() => {
if (clientID) {
try {
mixpanel.init(analytics, {
api_host: "https://api.mixpanel.com",
loaded: () => {
setIsLoaded(true);
mixpanel.identify(clientID);
},
});
} catch (error) {
console.warn("loading mixpanel failed:", error);
}
}
}, [analytics, clientID]);
useEffect(() => {
isMixpanelReady && mixpanel.register("sessionId", ui.sessionId);
}, [ui.sessionId, isMixpanelReady]);
// ********** USER STATE **************
useEffect(() => {
if (user) {
try {
if (isLoaded) {
if (isMixpanelReady) {
mixpanel.people.set({
[`${MIXPANEL_EVENTS.LAST_VISITED}`]: new Date().toISOString(),
});
@ -74,11 +169,36 @@ const AnalyticsProvider = ({ children }) => {
console.error("could not set up people in mixpanel:", err);
}
}
}, [user, isLoaded, clientID]);
}, [user, isMixpanelReady, clientID]);
useEffect(() => {
if (isMixpanelReady && user) {
mixpanel.people.set_once({
[`${MIXPANEL_EVENTS.FIRST_LOGIN_DATE}`]: new Date().toISOString(),
});
mixpanel.people.set({
[`${MIXPANEL_EVENTS.LAST_LOGIN_DATE}`]: new Date().toISOString(),
});
mixpanel.track(`${MIXPANEL_EVENTS.USER_LOGS_IN}`, {});
}
}, [user, isMixpanelReady]);
useEffect(() => {
if (isMixpanelReady && ui.isLoggingOut) {
mixpanel.track(`${MIXPANEL_EVENTS.USER_LOGS_OUT}`, {});
}
}, [ui.isLoggingOut, isMixpanelReady]);
// ********** USER BOUNCE TIME **************
useEffect(() => {
if (!user && isInit && isMixpanelReady) {
mixpanel.time_event(MIXPANEL_EVENTS.CONVERT_TO_USER);
}
}, [user, isInit, isMixpanelReady]);
return (
<AnalyticsContext.Provider
value={{ mixpanel, isLoaded, MIXPANEL_EVENTS, MIXPANEL_PROPS }}
value={{ mixpanel, isMixpanelReady, MIXPANEL_EVENTS, MIXPANEL_PROPS }}
>
{children}
</AnalyticsContext.Provider>

2
frontend/src/core/providers/UIProvider/index.js
Wyświetl plik

@ -298,6 +298,8 @@ const UIProvider = ({ children }) => {
setOnboardingComplete,
onboardingSteps,
setOnboardingState,
onboardingState,
isLoggingOut,
}}
>
{children}