From db809568f22831036b7834d9c2f91d27b2c587a0 Mon Sep 17 00:00:00 2001 From: Tim Pechersky Date: Mon, 30 Aug 2021 18:19:46 +0200 Subject: [PATCH 01/32] change onboarding state var for better tracking --- frontend/src/core/providers/UIProvider/index.js | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/frontend/src/core/providers/UIProvider/index.js b/frontend/src/core/providers/UIProvider/index.js index 9c92d494..40e20b62 100644 --- a/frontend/src/core/providers/UIProvider/index.js +++ b/frontend/src/core/providers/UIProvider/index.js @@ -181,10 +181,19 @@ const UIProvider = ({ children }) => { ); const [onboardingStep, setOnboardingStep] = useState(() => { - const step = onboardingSteps.findIndex( - (event) => onboardingState[event.step] === 0 + //First find onboarding step that was viewed once (resume where left) + // If none - find step that was never viewed + // if none - set onboarding to zero + let step = onboardingSteps.findIndex( + (event) => onboardingState[event.step] === 1 ); - if (step === -1 && isOnboardingComplete) return onboardingSteps.length - 1; + step = + step === -1 + ? onboardingSteps.findIndex( + (event) => onboardingState[event.step] === 0 + ) + : step; + if (step === -1 && isOnboardingComplete) return 0; else if (step === -1 && !isOnboardingComplete) return 0; else return step; }); From e25828b5af0d0c03dfa63a9a95eed249e845cf49 Mon Sep 17 00:00:00 2001 From: Tim Pechersky Date: Mon, 30 Aug 2021 18:20:13 +0200 Subject: [PATCH 02/32] add new constants to analytics --- .../core/providers/AnalyticsProvider/constants.js | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/frontend/src/core/providers/AnalyticsProvider/constants.js b/frontend/src/core/providers/AnalyticsProvider/constants.js index 309532e1..e39e354a 100644 --- a/frontend/src/core/providers/AnalyticsProvider/constants.js +++ b/frontend/src/core/providers/AnalyticsProvider/constants.js @@ -8,6 +8,12 @@ export const MIXPANEL_PROPS = { USER_SPECIALITY: "user speciality", }; +export const MIXPANEL_GROUPS = { + DEVELOPERS: "developers", + TRADERS: "traders", + FUND: "funds", +}; + export const MIXPANEL_EVENTS = { FIRST_LOGIN_DATE: "First login date", LAST_LOGIN_DATE: "Last login date", @@ -20,7 +26,13 @@ export const MIXPANEL_EVENTS = { PAGEVIEW: "Page view", PRICING_PLAN_CLICKED: "Pricing Plan clicked", BUTTON_CLICKED: "Button clicked", - LEFT_PAGE: "Left page", + BEACON: "beacon", + ONBOARDING_COMPLETED: "Onbording complete", + SESSIONS_COUNT: "Sessions Counter", + ONBOARDING_STEPS: "Onboarding Steps", + PAGES_VISITED: "pages visited", + FORM_SUBMITTED: "form submitted", + PAGEVIEW_DURATION: "Time spent on page", }; export default MIXPANEL_EVENTS; From 6de180663b81ba6ba6282177a58c52f16e35a146 Mon Sep 17 00:00:00 2001 From: Tim Pechersky Date: Mon, 30 Aug 2021 18:20:24 +0200 Subject: [PATCH 03/32] fix button clicked bug --- frontend/pages/index.js | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/frontend/pages/index.js b/frontend/pages/index.js index c3e21079..5c83aea4 100644 --- a/frontend/pages/index.js +++ b/frontend/pages/index.js @@ -380,7 +380,7 @@ const Homepage = () => { link: "/#cryptoTrader", onClick: () => { track(`${MIXPANEL_EVENTS.BUTTON_CLICKED}`, { - [`${MIXPANEL_PROPS.BUTTON_CLICKED}`]: `scroll to CryptoTrader`, + [`${MIXPANEL_PROPS.BUTTON_NAME}`]: `scroll to CryptoTrader`, }); }, }} @@ -389,7 +389,7 @@ const Homepage = () => { link: "/#algoFund", onClick: () => { track(`${MIXPANEL_EVENTS.BUTTON_CLICKED}`, { - [`${MIXPANEL_PROPS.BUTTON_CLICKED}`]: `scroll to AlgoFund`, + [`${MIXPANEL_PROPS.BUTTON_NAME}`]: `scroll to AlgoFund`, }); }, }} @@ -398,7 +398,7 @@ const Homepage = () => { link: "/#smartDeveloper", onClick: () => { track(`${MIXPANEL_EVENTS.BUTTON_CLICKED}`, { - [`${MIXPANEL_PROPS.BUTTON_CLICKED}`]: `scroll to Developer`, + [`${MIXPANEL_PROPS.BUTTON_NAME}`]: `scroll to Developer`, }); }, }} @@ -418,7 +418,7 @@ const Homepage = () => { label: "I want early access!", onClick: () => { track(`${MIXPANEL_EVENTS.BUTTON_CLICKED}`, { - [`${MIXPANEL_PROPS.BUTTON_CLICKED}`]: `Early access CTA: Crypto trader`, + [`${MIXPANEL_PROPS.BUTTON_NAME}`]: `Early access CTA: Crypto trader`, }); toggleModal("hubspot-trader"); }, @@ -465,7 +465,7 @@ const Homepage = () => { label: "I want early access!", onClick: () => { track(`${MIXPANEL_EVENTS.BUTTON_CLICKED}`, { - [`${MIXPANEL_PROPS.BUTTON_CLICKED}`]: `Early access CTA: Algo fund`, + [`${MIXPANEL_PROPS.BUTTON_NAME}`]: `Early access CTA: Algo fund`, }); toggleModal("hubspot-fund"); }, @@ -510,7 +510,7 @@ const Homepage = () => { label: "I want early access!", onClick: () => { track(`${MIXPANEL_EVENTS.BUTTON_CLICKED}`, { - [`${MIXPANEL_PROPS.BUTTON_CLICKED}`]: `Early access CTA: developer`, + [`${MIXPANEL_PROPS.BUTTON_NAME}`]: `Early access CTA: developer`, }); toggleModal("hubspot-developer"); }, @@ -521,7 +521,7 @@ const Homepage = () => { label: "See our github", onClick: () => { track(`${MIXPANEL_EVENTS.BUTTON_CLICKED}`, { - [`${MIXPANEL_PROPS.BUTTON_CLICKED}`]: `Github link in landing page`, + [`${MIXPANEL_PROPS.BUTTON_NAME}`]: `Github link in landing page`, }); }, }} @@ -569,7 +569,7 @@ const Homepage = () => { id="test" onClick={() => { track(`${MIXPANEL_EVENTS.BUTTON_CLICKED}`, { - [`${MIXPANEL_PROPS.BUTTON_CLICKED}`]: `Join our discord`, + [`${MIXPANEL_PROPS.BUTTON_NAME}`]: `Join our discord`, }); toggleModal("hubspot"); }} From b69b6d031aae518a2d17d2db3ec1536f60877796 Mon Sep 17 00:00:00 2001 From: Tim Pechersky Date: Mon, 30 Aug 2021 18:20:46 +0200 Subject: [PATCH 04/32] move analytics below UI provider in context --- frontend/src/AppContext.js | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/frontend/src/AppContext.js b/frontend/src/AppContext.js index 7a238760..f2c56e74 100644 --- a/frontend/src/AppContext.js +++ b/frontend/src/AppContext.js @@ -13,13 +13,13 @@ const AppContext = (props) => { return ( - - - - {props.children} - - - + + + + {props.children} + + + ); From 604d0b76b181f9c18af043aaf9cc31bfed1a5a5a Mon Sep 17 00:00:00 2001 From: Tim Pechersky Date: Mon, 30 Aug 2021 18:34:47 +0200 Subject: [PATCH 05/32] provider improvements --- .../core/providers/AnalyticsProvider/index.js | 102 +++++++++++++----- 1 file changed, 76 insertions(+), 26 deletions(-) diff --git a/frontend/src/core/providers/AnalyticsProvider/index.js b/frontend/src/core/providers/AnalyticsProvider/index.js index eddffb8a..95b91de7 100644 --- a/frontend/src/core/providers/AnalyticsProvider/index.js +++ b/frontend/src/core/providers/AnalyticsProvider/index.js @@ -1,66 +1,116 @@ -import React, { useEffect, useState } from "react"; +import React, { useContext, useEffect, useState } from "react"; import mixpanel from "mixpanel-browser"; import AnalyticsContext from "./context"; import { useClientID, useUser, useRouter } from "../../hooks"; import { MIXPANEL_EVENTS, MIXPANEL_PROPS } from "./constants"; +import UIContext from "../UIProvider/context"; const AnalyticsProvider = ({ children }) => { const clientID = useClientID(); const analytics = process.env.NEXT_PUBLIC_MIXPANEL_TOKEN; const { user } = useUser(); - const [isLoaded, setIsLoaded] = useState(false); + const [isMixpanelReady, setIsLoaded] = useState(false); const router = useRouter(); + const ui = useContext(UIContext); + useEffect(() => { + if (ui.isOnboardingComplete && isMixpanelReady && user) { + mixpanel.people.set(MIXPANEL_EVENTS.ONBOARDING_COMPLETED, true); + } + }, [ui.isOnboardingComplete, isMixpanelReady, user]); + + useEffect(() => { + if (ui.onboardingStep && isMixpanelReady) { + mixpanel.people.set(MIXPANEL_EVENTS.ONBOARDING_STEPS, ui.onboardingStep); + } + }, [ui.onboardingStep, isMixpanelReady]); + useEffect(() => { let durationSeconds = 0; const intervalId = - isLoaded && + isMixpanelReady && setInterval(() => { - durationSeconds = durationSeconds + 1; + durationSeconds = durationSeconds + 30; mixpanel.track( - MIXPANEL_EVENTS.LEFT_PAGE, + MIXPANEL_EVENTS.BEACON, { duration_seconds: durationSeconds, url: router.nextRouter.pathname, - query: router.query, - pathParams: router.params, }, { transport: "sendBeacon" } ); }, 30000); return () => clearInterval(intervalId); - // eslint-disable-next-line - }, [isLoaded]); + }, [isMixpanelReady, router.nextRouter.pathname]); + + const [previousPathname, setPreviousPathname] = useState(false); useEffect(() => { - isLoaded && + if (isMixpanelReady) { + if (!previousPathname) { + mixpanel.time_event(MIXPANEL_EVENTS.PAGEVIEW_DURATION); + setPreviousPathname(router.nextRouter.pathname); + } + if (previousPathname && previousPathname !== router.nextRouter.pathname) { + mixpanel.track(MIXPANEL_EVENTS.PAGEVIEW_DURATION, { + url: previousPathname, + isBeforeUnload: false, + }); + setPreviousPathname(false); + } + } + }, [router.nextRouter.pathname, previousPathname, isMixpanelReady]); + + useEffect(() => { + if (isMixpanelReady && ui.sessionId && router.nextRouter.pathname) { mixpanel.track(MIXPANEL_EVENTS.PAGEVIEW, { url: router.nextRouter.pathname, - query: router.query, - pathParams: router.params, + sessionID: ui.sessionId, }); - }, [router.nextRouter.pathname, router.query, router.params, isLoaded]); + + mixpanel.people.set(MIXPANEL_EVENTS.PAGES_VISITED, { + [`${router.nextRouter.pathname}`]: true, + }); + } + const urlForUnmount = router.nextRouter.pathname; + const closeListener = () => { + mixpanel.track(MIXPANEL_EVENTS.PAGEVIEW_DURATION, { + url: urlForUnmount, + isBeforeUnload: true, + }); + }; + window.addEventListener("beforeunload", closeListener); + return () => { + window.removeEventListener("beforeunload", closeListener); + }; + }, [router.nextRouter.pathname, isMixpanelReady, ui.sessionId]); useEffect(() => { - try { - mixpanel.init(analytics, { - api_host: "https://api.mixpanel.com", - loaded: () => { - setIsLoaded(true); - mixpanel.identify(clientID); - }, - }); - } catch (error) { - console.warn("loading mixpanel failed:", error); + isMixpanelReady && mixpanel.register("sessionId", ui.sessionId); + }, [ui.sessionId, isMixpanelReady]); + + useEffect(() => { + if (clientID) { + try { + mixpanel.init(analytics, { + api_host: "https://api.mixpanel.com", + loaded: () => { + setIsLoaded(true); + mixpanel.identify(clientID); + }, + }); + } catch (error) { + console.warn("loading mixpanel failed:", error); + } } }, [analytics, clientID]); useEffect(() => { if (user) { try { - if (isLoaded) { + if (isMixpanelReady) { mixpanel.people.set({ [`${MIXPANEL_EVENTS.LAST_VISITED}`]: new Date().toISOString(), }); @@ -74,11 +124,11 @@ const AnalyticsProvider = ({ children }) => { console.error("could not set up people in mixpanel:", err); } } - }, [user, isLoaded, clientID]); + }, [user, isMixpanelReady, clientID]); return ( {children} From 9c852fa8d84156038e7117a98a4e2ad7f525107c Mon Sep 17 00:00:00 2001 From: Tim Pechersky Date: Mon, 30 Aug 2021 18:35:50 +0200 Subject: [PATCH 06/32] track form submited on analytics page --- frontend/pages/analytics.js | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/frontend/pages/analytics.js b/frontend/pages/analytics.js index fb320883..a1666c37 100644 --- a/frontend/pages/analytics.js +++ b/frontend/pages/analytics.js @@ -3,6 +3,9 @@ import HubspotForm from "react-hubspot-form"; import { getLayout } from "../src/layouts/AppLayout"; import { Spinner, Flex, Heading } from "@chakra-ui/react"; import Scrollable from "../src/components/Scrollable"; +import mixpanel from "mixpanel-browser"; +import MIXPANEL_EVENTS from "../src/core/providers/AnalyticsProvider/constants"; +import { useUser } from "../src/core/hooks"; const Analytics = () => { useEffect(() => { @@ -11,6 +14,8 @@ const Analytics = () => { } }, []); + const { user } = useUser(); + return ( { portalId="8018701" formId="39bc0fbe-41c4-430a-b885-46eba66c59c2" loading={} + onSubmit={() => + mixpanel.track(MIXPANEL_EVENTS.FORM_SUBMITTED, { + formType: "Hubspot analytics", + user: user, + }) + } /> From 9fbf1a43d7608439bc62988dc988a5926ac3ebec Mon Sep 17 00:00:00 2001 From: Tim Pechersky Date: Mon, 30 Aug 2021 19:17:31 +0200 Subject: [PATCH 07/32] remove useAnalytics hook --- frontend/pages/index.js | 64 +++++++++++-------- frontend/src/components/Scrollable.js | 6 +- frontend/src/core/hooks/index.js | 1 - frontend/src/core/hooks/useAnalytics.js | 40 +++++++----- frontend/src/core/hooks/useLogin.js | 16 ----- frontend/src/core/hooks/useLogout.js | 9 +-- frontend/src/core/hooks/useSignUp.js | 16 +++-- frontend/src/core/hooks/useToast.js | 19 +++--- .../core/providers/AnalyticsProvider/index.js | 20 ++++++ .../src/core/providers/UIProvider/index.js | 1 + 10 files changed, 103 insertions(+), 89 deletions(-) diff --git a/frontend/pages/index.js b/frontend/pages/index.js index 5c83aea4..0bcfdeaa 100644 --- a/frontend/pages/index.js +++ b/frontend/pages/index.js @@ -24,12 +24,15 @@ import { } from "@chakra-ui/react"; import dynamic from "next/dynamic"; import useUser from "../src/core/hooks/useUser"; -import useAnalytics from "../src/core/hooks/useAnalytics"; import useModals from "../src/core/hooks/useModals"; import useRouter from "../src/core/hooks/useRouter"; -import { MIXPANEL_PROPS } from "../src/core/providers/AnalyticsProvider/constants"; +import { + MIXPANEL_PROPS, + MIXPANEL_EVENTS, +} from "../src/core/providers/AnalyticsProvider/constants"; import UIContext from "../src/core/providers/UIProvider/context"; import { AWS_ASSETS_PATH } from "../src/core/constants"; +import mixpanel from "mixpanel-browser"; const SplitWithImage = dynamic( () => import("../src/components/SplitWithImage"), { @@ -105,7 +108,6 @@ const Homepage = () => { const router = useRouter(); const { isInit } = useUser(); - const { MIXPANEL_EVENTS, track } = useAnalytics(); const { toggleModal } = useModals(); const [ isLargerThan720px, @@ -379,27 +381,30 @@ const Homepage = () => { label: "Crypto trader", link: "/#cryptoTrader", onClick: () => { - track(`${MIXPANEL_EVENTS.BUTTON_CLICKED}`, { - [`${MIXPANEL_PROPS.BUTTON_NAME}`]: `scroll to CryptoTrader`, - }); + mixpanel.get_distinct_id() && + mixpanel.track(`${MIXPANEL_EVENTS.BUTTON_CLICKED}`, { + [`${MIXPANEL_PROPS.BUTTON_NAME}`]: `scroll to CryptoTrader`, + }); }, }} button2={{ label: "Algorithmic Fund", link: "/#algoFund", onClick: () => { - track(`${MIXPANEL_EVENTS.BUTTON_CLICKED}`, { - [`${MIXPANEL_PROPS.BUTTON_NAME}`]: `scroll to AlgoFund`, - }); + mixpanel.get_distinct_id() && + mixpanel.track(`${MIXPANEL_EVENTS.BUTTON_CLICKED}`, { + [`${MIXPANEL_PROPS.BUTTON_NAME}`]: `scroll to AlgoFund`, + }); }, }} button3={{ label: "Developer", link: "/#smartDeveloper", onClick: () => { - track(`${MIXPANEL_EVENTS.BUTTON_CLICKED}`, { - [`${MIXPANEL_PROPS.BUTTON_NAME}`]: `scroll to Developer`, - }); + mixpanel.get_distinct_id() && + mixpanel.track(`${MIXPANEL_EVENTS.BUTTON_CLICKED}`, { + [`${MIXPANEL_PROPS.BUTTON_NAME}`]: `scroll to Developer`, + }); }, }} /> @@ -417,9 +422,10 @@ const Homepage = () => { cta={{ label: "I want early access!", onClick: () => { - track(`${MIXPANEL_EVENTS.BUTTON_CLICKED}`, { - [`${MIXPANEL_PROPS.BUTTON_NAME}`]: `Early access CTA: Crypto trader`, - }); + mixpanel.get_distinct_id() && + mixpanel.track(`${MIXPANEL_EVENTS.BUTTON_CLICKED}`, { + [`${MIXPANEL_PROPS.BUTTON_NAME}`]: `Early access CTA: Crypto trader`, + }); toggleModal("hubspot-trader"); }, }} @@ -464,9 +470,10 @@ const Homepage = () => { cta={{ label: "I want early access!", onClick: () => { - track(`${MIXPANEL_EVENTS.BUTTON_CLICKED}`, { - [`${MIXPANEL_PROPS.BUTTON_NAME}`]: `Early access CTA: Algo fund`, - }); + mixpanel.get_distinct_id() && + mixpanel.track(`${MIXPANEL_EVENTS.BUTTON_CLICKED}`, { + [`${MIXPANEL_PROPS.BUTTON_NAME}`]: `Early access CTA: Algo fund`, + }); toggleModal("hubspot-fund"); }, }} @@ -509,9 +516,10 @@ const Homepage = () => { cta={{ label: "I want early access!", onClick: () => { - track(`${MIXPANEL_EVENTS.BUTTON_CLICKED}`, { - [`${MIXPANEL_PROPS.BUTTON_NAME}`]: `Early access CTA: developer`, - }); + mixpanel.get_distinct_id() && + mixpanel.track(`${MIXPANEL_EVENTS.BUTTON_CLICKED}`, { + [`${MIXPANEL_PROPS.BUTTON_NAME}`]: `Early access CTA: developer`, + }); toggleModal("hubspot-developer"); }, }} @@ -520,9 +528,10 @@ const Homepage = () => { network: "github", label: "See our github", onClick: () => { - track(`${MIXPANEL_EVENTS.BUTTON_CLICKED}`, { - [`${MIXPANEL_PROPS.BUTTON_NAME}`]: `Github link in landing page`, - }); + mixpanel.get_distinct_id() && + mixpanel.track(`${MIXPANEL_EVENTS.BUTTON_CLICKED}`, { + [`${MIXPANEL_PROPS.BUTTON_NAME}`]: `Github link in landing page`, + }); }, }} elementName={"element3"} @@ -568,9 +577,10 @@ const Homepage = () => { colorScheme="suggested" id="test" onClick={() => { - track(`${MIXPANEL_EVENTS.BUTTON_CLICKED}`, { - [`${MIXPANEL_PROPS.BUTTON_NAME}`]: `Join our discord`, - }); + mixpanel.get_distinct_id() && + mixpanel.track(`${MIXPANEL_EVENTS.BUTTON_CLICKED}`, { + [`${MIXPANEL_PROPS.BUTTON_NAME}`]: `Join our discord`, + }); toggleModal("hubspot"); }} > diff --git a/frontend/src/components/Scrollable.js b/frontend/src/components/Scrollable.js index 4ab9f6a7..bdcdf0f0 100644 --- a/frontend/src/components/Scrollable.js +++ b/frontend/src/components/Scrollable.js @@ -1,13 +1,13 @@ import { Flex, Box } from "@chakra-ui/react"; import React, { useEffect, useRef, useState } from "react"; -import { useRouter, useAnalytics } from "../core/hooks"; +import { useRouter } from "../core/hooks"; +import mixpanel from "mixpanel-browser"; const Scrollable = (props) => { const scrollerRef = useRef(); const router = useRouter(); const [path, setPath] = useState(); const [scrollDepth, setScrollDepth] = useState(0); - const { mixpanel, isLoaded } = useAnalytics(); const getScrollPrecent = ({ currentTarget }) => { const scroll_level = @@ -20,7 +20,7 @@ const Scrollable = (props) => { const currentScroll = Math.ceil(getScrollPrecent(e) / 10); if (currentScroll > scrollDepth) { setScrollDepth(currentScroll); - isLoaded && + mixpanel.get_distinct_id() && mixpanel.people.increment({ [`Scroll depth at: ${router.nextRouter.pathname}`]: currentScroll, }); diff --git a/frontend/src/core/hooks/index.js b/frontend/src/core/hooks/index.js index 328a74da..5625f907 100644 --- a/frontend/src/core/hooks/index.js +++ b/frontend/src/core/hooks/index.js @@ -1,5 +1,4 @@ export { queryCacheProps as hookCommon } from "./hookCommon"; -export { default as useAnalytics } from "./useAnalytics"; export { default as useAuthResultHandler } from "./useAuthResultHandler"; export { default as useChangePassword } from "./useChangePassword"; export { default as useClientID } from "./useClientID"; diff --git a/frontend/src/core/hooks/useAnalytics.js b/frontend/src/core/hooks/useAnalytics.js index fee7e9eb..59b955d9 100644 --- a/frontend/src/core/hooks/useAnalytics.js +++ b/frontend/src/core/hooks/useAnalytics.js @@ -1,25 +1,33 @@ import AnalyticsContext from "../providers/AnalyticsProvider/context"; import { useContext } from "react"; -import { useState, useEffect, useCallback } from "react"; -const useAnalytics = () => { - const { mixpanel, isLoaded, MIXPANEL_EVENTS, MIXPANEL_PROPS } = - useContext(AnalyticsContext); - const [trackProps, setTrackProps] = useState({ - event: null, - props: null, - queued: false, - }); +import { useState, useEffect } from "react"; +import { + MIXPANEL_PROPS, + MIXPANEL_EVENTS, +} from "../providers/AnalyticsProvider/constants"; - const track = useCallback((e, props) => { - setTrackProps({ event: e, props: props, queued: true }); - }, []); +const useAnalytics = () => { + const { mixpanel, isLoaded } = useContext(AnalyticsContext); + const [eventsQueue, setEventsQueue] = useState([]); + + const track = (e, props) => { + setEventsQueue((trackingQueue) => [ + ...trackingQueue, + { + event: e, + props: props, + }, + ]); + }; useEffect(() => { - if (isLoaded && trackProps.queued === true) { - mixpanel.track(trackProps.event, trackProps.props); - setTrackProps({ event: null, props: null, queued: false }); + if (isLoaded && eventsQueue.length > 0 && mixpanel) { + const newTrackingQueue = [...eventsQueue]; + const newTrackEvent = newTrackingQueue.pop(); + mixpanel.track(newTrackEvent.event, newTrackEvent.props); + setEventsQueue(newTrackingQueue); } - }, [isLoaded, mixpanel, trackProps]); + }, [isLoaded, mixpanel, eventsQueue]); const withTracking = (fn, event, props) => { track(event, props); diff --git a/frontend/src/core/hooks/useLogin.js b/frontend/src/core/hooks/useLogin.js index c8dd3bfc..b410cc9a 100644 --- a/frontend/src/core/hooks/useLogin.js +++ b/frontend/src/core/hooks/useLogin.js @@ -1,7 +1,6 @@ import { useMutation } from "react-query"; import { useToast, useUser, useInviteAccept } from "."; import { AuthService } from "../services"; -import { useAnalytics } from "."; const LOGIN_TYPES = { MANUAL: true, @@ -10,7 +9,6 @@ const LOGIN_TYPES = { const useLogin = (loginType) => { const { getUser } = useUser(); const toast = useToast(); - const analytics = useAnalytics(); const { inviteAccept } = useInviteAccept(); const { mutate: login, @@ -34,20 +32,6 @@ const useLogin = (loginType) => { inviteAccept(invite_code); } getUser(); - if (analytics.isLoaded) { - analytics.mixpanel.people.set_once({ - [`${analytics.MIXPANEL_EVENTS.FIRST_LOGIN_DATE}`]: - new Date().toISOString(), - }); - analytics.mixpanel.people.set({ - [`${analytics.MIXPANEL_EVENTS.LAST_LOGIN_DATE}`]: - new Date().toISOString(), - }); - analytics.mixpanel.track( - `${analytics.MIXPANEL_EVENTS.USER_LOGS_IN}`, - {} - ); - } } }, onError: (error) => { diff --git a/frontend/src/core/hooks/useLogout.js b/frontend/src/core/hooks/useLogout.js index ff605d80..c8be9a86 100644 --- a/frontend/src/core/hooks/useLogout.js +++ b/frontend/src/core/hooks/useLogout.js @@ -1,21 +1,14 @@ import { useCallback, useContext } from "react"; import { useMutation, useQueryClient } from "react-query"; -import { useUser, useRouter, useAnalytics } from "."; +import { useUser, useRouter } from "."; import UIContext from "../providers/UIProvider/context"; import { AuthService } from "../services"; const useLogout = () => { const { setLoggingOut } = useContext(UIContext); const router = useRouter(); - const analytics = useAnalytics(); const { mutate: revoke } = useMutation(AuthService.revoke, { onSuccess: () => { - if (analytics.isLoaded) { - analytics.mixpanel.track( - `${analytics.MIXPANEL_EVENTS.USER_LOGS_OUT}`, - {} - ); - } localStorage.removeItem("MOONSTREAM_ACCESS_TOKEN"); cache.clear(); setUser(null); diff --git a/frontend/src/core/hooks/useSignUp.js b/frontend/src/core/hooks/useSignUp.js index f6dca897..6ae77f7e 100644 --- a/frontend/src/core/hooks/useSignUp.js +++ b/frontend/src/core/hooks/useSignUp.js @@ -1,16 +1,18 @@ import { useContext } from "react"; import { useMutation } from "react-query"; import { AuthService } from "../services"; -import { useUser, useToast, useInviteAccept, useRouter, useAnalytics } from "."; +import { useUser, useToast, useInviteAccept, useRouter } from "."; import UIContext from "../providers/UIProvider/context"; +import mixpanel from "mixpanel-browser"; +import { MIXPANEL_EVENTS } from "../providers/AnalyticsProvider/constants"; const useSignUp = (source) => { const ui = useContext(UIContext); + const router = useRouter(); const { getUser } = useUser(); const toast = useToast(); const { inviteAccept } = useInviteAccept(); - const analytics = useAnalytics(); const { mutate: signUp, @@ -26,11 +28,11 @@ const useSignUp = (source) => { inviteAccept(invite_code); } - if (analytics.isLoaded) { - analytics.mixpanel.track( - `${analytics.MIXPANEL_EVENTS.CONVERT_TO_USER}`, - { full_url: router.nextRouter.asPath, code: source } - ); + if (mixpanel.get_distinct_id()) { + mixpanel.track(`${MIXPANEL_EVENTS.CONVERT_TO_USER}`, { + full_url: router.nextRouter.asPath, + code: source, + }); } getUser(); ui.setisOnboardingComplete(false); diff --git a/frontend/src/core/hooks/useToast.js b/frontend/src/core/hooks/useToast.js index eeae3e7e..6d78ed9a 100644 --- a/frontend/src/core/hooks/useToast.js +++ b/frontend/src/core/hooks/useToast.js @@ -1,21 +1,18 @@ import { useToast as useChakraToast, Box } from "@chakra-ui/react"; import React, { useCallback } from "react"; -import { useAnalytics } from "."; +import mixpanel from "mixpanel-browser"; +import { MIXPANEL_EVENTS } from "../providers/AnalyticsProvider/constants"; const useToast = () => { const chakraToast = useChakraToast(); - const analytics = useAnalytics(); const toast = useCallback( (message, type) => { - if (analytics.isLoaded && type === "error") { - analytics.mixpanel.track( - `${analytics.MIXPANEL_EVENTS.TOAST_ERROR_DISPLAYED}`, - { - status: message?.response?.status, - detail: message?.response?.data.detail, - } - ); + if (mixpanel.get_distinct_id() && type === "error") { + mixpanel.track(`${MIXPANEL_EVENTS.TOAST_ERROR_DISPLAYED}`, { + status: message?.response?.status, + detail: message?.response?.data.detail, + }); } const background = type === "error" ? "unsafe.500" : "suggested.500"; const userMessage = @@ -43,7 +40,7 @@ const useToast = () => { ), }); }, - [chakraToast, analytics] + [chakraToast] ); return toast; diff --git a/frontend/src/core/providers/AnalyticsProvider/index.js b/frontend/src/core/providers/AnalyticsProvider/index.js index 95b91de7..a5f9e3ca 100644 --- a/frontend/src/core/providers/AnalyticsProvider/index.js +++ b/frontend/src/core/providers/AnalyticsProvider/index.js @@ -82,6 +82,8 @@ const AnalyticsProvider = ({ children }) => { }); }; window.addEventListener("beforeunload", closeListener); + //cleanup function fires on useEffect unmount + //https://reactjs.org/docs/hooks-effect.html return () => { window.removeEventListener("beforeunload", closeListener); }; @@ -126,6 +128,24 @@ const AnalyticsProvider = ({ children }) => { } }, [user, isMixpanelReady, clientID]); + useEffect(() => { + if (isMixpanelReady && user) { + mixpanel.people.set_once({ + [`${MIXPANEL_EVENTS.FIRST_LOGIN_DATE}`]: new Date().toISOString(), + }); + mixpanel.people.set({ + [`${MIXPANEL_EVENTS.LAST_LOGIN_DATE}`]: new Date().toISOString(), + }); + mixpanel.track(`${MIXPANEL_EVENTS.USER_LOGS_IN}`, {}); + } + }, [user, isMixpanelReady]); + + useEffect(() => { + if (isMixpanelReady && ui.isLoggingOut) { + mixpanel.track(`${MIXPANEL_EVENTS.USER_LOGS_OUT}`, {}); + } + }, [ui.isLoggingOut, isMixpanelReady]); + return ( { setisOnboardingComplete, onboardingSteps, setOnboardingState, + isLoggingOut, }} > {children} From 2d8ee15ae11c87e93ee94b156a160bc7d8071b57 Mon Sep 17 00:00:00 2001 From: Tim Pechersky Date: Mon, 30 Aug 2021 19:18:30 +0200 Subject: [PATCH 08/32] delete useAnalytics hook --- frontend/src/core/hooks/useAnalytics.js | 46 ------------------------- 1 file changed, 46 deletions(-) delete mode 100644 frontend/src/core/hooks/useAnalytics.js diff --git a/frontend/src/core/hooks/useAnalytics.js b/frontend/src/core/hooks/useAnalytics.js deleted file mode 100644 index 59b955d9..00000000 --- a/frontend/src/core/hooks/useAnalytics.js +++ /dev/null @@ -1,46 +0,0 @@ -import AnalyticsContext from "../providers/AnalyticsProvider/context"; -import { useContext } from "react"; -import { useState, useEffect } from "react"; -import { - MIXPANEL_PROPS, - MIXPANEL_EVENTS, -} from "../providers/AnalyticsProvider/constants"; - -const useAnalytics = () => { - const { mixpanel, isLoaded } = useContext(AnalyticsContext); - const [eventsQueue, setEventsQueue] = useState([]); - - const track = (e, props) => { - setEventsQueue((trackingQueue) => [ - ...trackingQueue, - { - event: e, - props: props, - }, - ]); - }; - - useEffect(() => { - if (isLoaded && eventsQueue.length > 0 && mixpanel) { - const newTrackingQueue = [...eventsQueue]; - const newTrackEvent = newTrackingQueue.pop(); - mixpanel.track(newTrackEvent.event, newTrackEvent.props); - setEventsQueue(newTrackingQueue); - } - }, [isLoaded, mixpanel, eventsQueue]); - - const withTracking = (fn, event, props) => { - track(event, props); - return fn; - }; - - return { - mixpanel, - isLoaded, - track, - MIXPANEL_PROPS, - MIXPANEL_EVENTS, - withTracking, - }; -}; -export default useAnalytics; From 11766df9cd1e57b6f2b9a40453192a04680fef5f Mon Sep 17 00:00:00 2001 From: kompotkot Date: Mon, 30 Aug 2021 20:28:29 +0000 Subject: [PATCH 09/32] Added value to tags in ether --- crawlers/ethtxpool/main.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/crawlers/ethtxpool/main.go b/crawlers/ethtxpool/main.go index 224ae7dc..a98d4eb4 100644 --- a/crawlers/ethtxpool/main.go +++ b/crawlers/ethtxpool/main.go @@ -10,13 +10,14 @@ import ( "encoding/json" "flag" "fmt" + "math/big" "os" "time" humbug "github.com/bugout-dev/humbug/go/pkg" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" - + "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rpc" "github.com/google/uuid" ) @@ -124,11 +125,6 @@ func PollTxpoolContent(gethClient *rpc.Client, interval int, reporter *humbug.Hu continue } - // TODO(kompotkot, zomglings): Humbug API (on Spire) support bulk publication of reports. We should modify - // Humbug go client to use the bulk publish endpoint. Currently, if we have to publish all transactions - // pending in txpool, we *will* get rate limited. We may want to consider adding a publisher to the - // Humbug go client that can listen on a channel and will handle rate limiting, bulk publication etc. itself - // (without user having to worry about it). ReportTitle := "Ethereum: Pending transaction: " + transactionHash.String() ReportTags := []string{ "hash:" + transactionHash.String(), @@ -138,6 +134,7 @@ func PollTxpoolContent(gethClient *rpc.Client, interval int, reporter *humbug.Hu fmt.Sprintf("max_priority_fee_per_gas:%d", pendingTx.Transaction.MaxPriorityFeePerGas.ToInt()), fmt.Sprintf("max_fee_per_gas:%d", pendingTx.Transaction.MaxFeePerGas.ToInt()), fmt.Sprintf("gas:%d", pendingTx.Transaction.Gas), + fmt.Sprintf("value:%d", new(big.Float).Quo(new(big.Float).SetInt(transaction.Value.ToInt()), big.NewFloat(params.Ether))), "crawl_type:ethereum_txpool", } report := humbug.Report{ From 5896c0405142376958897c44fc62a04ccf7fd18a Mon Sep 17 00:00:00 2001 From: Tim Pechersky Date: Tue, 31 Aug 2021 13:31:47 +0200 Subject: [PATCH 10/32] page visit count by user --- frontend/src/core/providers/AnalyticsProvider/constants.js | 2 +- frontend/src/core/providers/AnalyticsProvider/index.js | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/frontend/src/core/providers/AnalyticsProvider/constants.js b/frontend/src/core/providers/AnalyticsProvider/constants.js index e39e354a..e8b55c73 100644 --- a/frontend/src/core/providers/AnalyticsProvider/constants.js +++ b/frontend/src/core/providers/AnalyticsProvider/constants.js @@ -30,7 +30,7 @@ export const MIXPANEL_EVENTS = { ONBOARDING_COMPLETED: "Onbording complete", SESSIONS_COUNT: "Sessions Counter", ONBOARDING_STEPS: "Onboarding Steps", - PAGES_VISITED: "pages visited", + TIMES_VISITED: "Page visit times", FORM_SUBMITTED: "form submitted", PAGEVIEW_DURATION: "Time spent on page", }; diff --git a/frontend/src/core/providers/AnalyticsProvider/index.js b/frontend/src/core/providers/AnalyticsProvider/index.js index a5f9e3ca..cd15579b 100644 --- a/frontend/src/core/providers/AnalyticsProvider/index.js +++ b/frontend/src/core/providers/AnalyticsProvider/index.js @@ -70,9 +70,9 @@ const AnalyticsProvider = ({ children }) => { sessionID: ui.sessionId, }); - mixpanel.people.set(MIXPANEL_EVENTS.PAGES_VISITED, { - [`${router.nextRouter.pathname}`]: true, - }); + mixpanel.people.increment([ + `${MIXPANEL_EVENTS.TIMES_VISITED} ${router.nextRouter.pathname}`, + ]); } const urlForUnmount = router.nextRouter.pathname; const closeListener = () => { From e2a0822f790a3d9ba56b03a071f15281270163a2 Mon Sep 17 00:00:00 2001 From: Tim Pechersky Date: Tue, 31 Aug 2021 13:36:21 +0200 Subject: [PATCH 11/32] time to convert to user --- frontend/src/core/providers/AnalyticsProvider/index.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/frontend/src/core/providers/AnalyticsProvider/index.js b/frontend/src/core/providers/AnalyticsProvider/index.js index cd15579b..daefb962 100644 --- a/frontend/src/core/providers/AnalyticsProvider/index.js +++ b/frontend/src/core/providers/AnalyticsProvider/index.js @@ -8,7 +8,7 @@ import UIContext from "../UIProvider/context"; const AnalyticsProvider = ({ children }) => { const clientID = useClientID(); const analytics = process.env.NEXT_PUBLIC_MIXPANEL_TOKEN; - const { user } = useUser(); + const { user, isInit } = useUser(); const [isMixpanelReady, setIsLoaded] = useState(false); const router = useRouter(); @@ -19,6 +19,12 @@ const AnalyticsProvider = ({ children }) => { } }, [ui.isOnboardingComplete, isMixpanelReady, user]); + useEffect(() => { + if (!user && isInit && isMixpanelReady) { + mixpanel.time_event(MIXPANEL_EVENTS.CONVERT_TO_USER); + } + }, [user, isInit, isMixpanelReady]); + useEffect(() => { if (ui.onboardingStep && isMixpanelReady) { mixpanel.people.set(MIXPANEL_EVENTS.ONBOARDING_STEPS, ui.onboardingStep); From e71a41bed2b043a7c7a0c1fcacabfbb5e68072ea Mon Sep 17 00:00:00 2001 From: Tim Pechersky Date: Tue, 31 Aug 2021 15:14:33 +0200 Subject: [PATCH 12/32] add files to previous commit --- .../providers/AnalyticsProvider/constants.js | 3 +- .../core/providers/AnalyticsProvider/index.js | 70 +++++++++++++++---- .../src/core/providers/UIProvider/index.js | 1 + 3 files changed, 60 insertions(+), 14 deletions(-) diff --git a/frontend/src/core/providers/AnalyticsProvider/constants.js b/frontend/src/core/providers/AnalyticsProvider/constants.js index e8b55c73..d8dc2a09 100644 --- a/frontend/src/core/providers/AnalyticsProvider/constants.js +++ b/frontend/src/core/providers/AnalyticsProvider/constants.js @@ -29,7 +29,8 @@ export const MIXPANEL_EVENTS = { BEACON: "beacon", ONBOARDING_COMPLETED: "Onbording complete", SESSIONS_COUNT: "Sessions Counter", - ONBOARDING_STEPS: "Onboarding Steps", + ONBOARDING_STEP: "Onboarding step", + ONBOARDING_STATE: "Onboarding state", TIMES_VISITED: "Page visit times", FORM_SUBMITTED: "form submitted", PAGEVIEW_DURATION: "Time spent on page", diff --git a/frontend/src/core/providers/AnalyticsProvider/index.js b/frontend/src/core/providers/AnalyticsProvider/index.js index daefb962..f05b2d85 100644 --- a/frontend/src/core/providers/AnalyticsProvider/index.js +++ b/frontend/src/core/providers/AnalyticsProvider/index.js @@ -11,26 +11,57 @@ const AnalyticsProvider = ({ children }) => { const { user, isInit } = useUser(); const [isMixpanelReady, setIsLoaded] = useState(false); const router = useRouter(); - const ui = useContext(UIContext); + + // ********** OBOARDING STATE ************** + useEffect(() => { + if (ui.onboardingState && isMixpanelReady) { + mixpanel.people.set(MIXPANEL_EVENTS.ONBOARDING_STATE, { + state: { ...ui.onboardingState }, + }); + } + }, [ui.onboardingState, isMixpanelReady]); + useEffect(() => { if (ui.isOnboardingComplete && isMixpanelReady && user) { mixpanel.people.set(MIXPANEL_EVENTS.ONBOARDING_COMPLETED, true); } }, [ui.isOnboardingComplete, isMixpanelReady, user]); - useEffect(() => { - if (!user && isInit && isMixpanelReady) { - mixpanel.time_event(MIXPANEL_EVENTS.CONVERT_TO_USER); - } - }, [user, isInit, isMixpanelReady]); + // ********** ONBOARDING STEP and TIMING ************** + const [previousOnboardingStep, setPreviousOnboardingStep] = useState(false); useEffect(() => { - if (ui.onboardingStep && isMixpanelReady) { - mixpanel.people.set(MIXPANEL_EVENTS.ONBOARDING_STEPS, ui.onboardingStep); + if (isMixpanelReady && router.nextRouter.pathname === "/welcome") { + if (!previousOnboardingStep) { + mixpanel.time_event(MIXPANEL_EVENTS.ONBOARDING_STEP); + setPreviousOnboardingStep(ui.onboardingStep); + } + if ( + previousOnboardingStep && + previousOnboardingStep !== ui.onboardingStep + ) { + mixpanel.track(MIXPANEL_EVENTS.ONBOARDING_STEP, { + step: previousOnboardingStep, + isBeforeUnload: false, + }); + setPreviousOnboardingStep(false); + } + } else if (previousOnboardingStep) { + mixpanel.track(MIXPANEL_EVENTS.ONBOARDING_STEP, { + step: previousOnboardingStep, + isBeforeUnload: false, + }); + setPreviousOnboardingStep(false); } - }, [ui.onboardingStep, isMixpanelReady]); + }, [ + previousOnboardingStep, + ui.onboardingStep, + isMixpanelReady, + router.nextRouter.pathname, + ]); + // ********** PING_PONG ************** useEffect(() => { let durationSeconds = 0; @@ -51,6 +82,8 @@ const AnalyticsProvider = ({ children }) => { return () => clearInterval(intervalId); }, [isMixpanelReady, router.nextRouter.pathname]); + // ********** TIME SPENT ON PATH************** + const [previousPathname, setPreviousPathname] = useState(false); useEffect(() => { @@ -69,6 +102,7 @@ const AnalyticsProvider = ({ children }) => { } }, [router.nextRouter.pathname, previousPathname, isMixpanelReady]); + // ********** PAGES VIEW ************** useEffect(() => { if (isMixpanelReady && ui.sessionId && router.nextRouter.pathname) { mixpanel.track(MIXPANEL_EVENTS.PAGEVIEW, { @@ -95,10 +129,7 @@ const AnalyticsProvider = ({ children }) => { }; }, [router.nextRouter.pathname, isMixpanelReady, ui.sessionId]); - useEffect(() => { - isMixpanelReady && mixpanel.register("sessionId", ui.sessionId); - }, [ui.sessionId, isMixpanelReady]); - + // ********** SESSION STATE ************** useEffect(() => { if (clientID) { try { @@ -115,6 +146,12 @@ const AnalyticsProvider = ({ children }) => { } }, [analytics, clientID]); + useEffect(() => { + isMixpanelReady && mixpanel.register("sessionId", ui.sessionId); + }, [ui.sessionId, isMixpanelReady]); + + // ********** USER STATE ************** + useEffect(() => { if (user) { try { @@ -152,6 +189,13 @@ const AnalyticsProvider = ({ children }) => { } }, [ui.isLoggingOut, isMixpanelReady]); + // ********** USER BOUNCE TIME ************** + useEffect(() => { + if (!user && isInit && isMixpanelReady) { + mixpanel.time_event(MIXPANEL_EVENTS.CONVERT_TO_USER); + } + }, [user, isInit, isMixpanelReady]); + return ( { setisOnboardingComplete, onboardingSteps, setOnboardingState, + onboardingState, isLoggingOut, }} > From a0ecb8f64a5110bb5ae1f2f869d588c988e98720 Mon Sep 17 00:00:00 2001 From: Andrey Dolgolev Date: Tue, 31 Aug 2021 17:17:36 +0300 Subject: [PATCH 13/32] Add table and index. --- db/moonstreamdb/models.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/db/moonstreamdb/models.py b/db/moonstreamdb/models.py index 3114133f..f9b4dd14 100644 --- a/db/moonstreamdb/models.py +++ b/db/moonstreamdb/models.py @@ -12,9 +12,10 @@ from sqlalchemy import ( Text, VARCHAR, UniqueConstraint, + Index, ) from sqlalchemy.dialects.postgresql import JSONB, UUID -from sqlalchemy.sql import expression +from sqlalchemy.sql import expression, text from sqlalchemy.ext.compiler import compiles """ @@ -136,7 +137,6 @@ class EthereumLabel(Base): # type: ignore """ __tablename__ = "ethereum_labels" - __table_args__ = (UniqueConstraint("label", "address_id"),) id = Column( UUID(as_uuid=True), @@ -156,6 +156,7 @@ class EthereumLabel(Base): # type: ignore created_at = Column( DateTime(timezone=True), server_default=utcnow(), nullable=False ) + __table_args__ = (Index("ix_label_name", text("label_data->>'name'")),) class EthereumPendingTransaction(Base): # type: ignore @@ -212,3 +213,19 @@ class ESDEventSignature(Base): # type: ignore created_at = Column( DateTime(timezone=True), server_default=utcnow(), nullable=False ) + + +class OpenSeaCrawlingState(Base): # type: ignore + """ + Model for control opeansea crawling state. + """ + + __tablename__ = "opensea_crawler_state" + + id = Column(Integer, primary_key=True, unique=True, nullable=False, index=True) + query = Column(Text, nullable=False) + updated_at = Column( + DateTime(timezone=True), server_default=utcnow(), nullable=False + ) + + total_count = Column(Integer, nullable=False) From 2a3d6dc4bab079d5d9ff44496b1e0f933c806005 Mon Sep 17 00:00:00 2001 From: Andrey Dolgolev Date: Tue, 31 Aug 2021 18:01:23 +0300 Subject: [PATCH 14/32] Add opensea table and index for jsonb column. --- db/moonstreamdb/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/db/moonstreamdb/models.py b/db/moonstreamdb/models.py index f9b4dd14..f917db55 100644 --- a/db/moonstreamdb/models.py +++ b/db/moonstreamdb/models.py @@ -156,7 +156,7 @@ class EthereumLabel(Base): # type: ignore created_at = Column( DateTime(timezone=True), server_default=utcnow(), nullable=False ) - __table_args__ = (Index("ix_label_name", text("label_data->>'name'")),) + __table_args__ = (Index("ix_label_name", text("(label_data->>'name')")),) class EthereumPendingTransaction(Base): # type: ignore @@ -222,7 +222,7 @@ class OpenSeaCrawlingState(Base): # type: ignore __tablename__ = "opensea_crawler_state" - id = Column(Integer, primary_key=True, unique=True, nullable=False, index=True) + id = Column(Integer, primary_key=True, unique=True, nullable=False) query = Column(Text, nullable=False) updated_at = Column( DateTime(timezone=True), server_default=utcnow(), nullable=False From 8d16c68a108eb3571b1de71ac95bcd7439023331 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Wed, 1 Sep 2021 12:03:44 +0000 Subject: [PATCH 15/32] Humbug reporter for moonstream backend --- backend/moonstream/__init__.py | 7 ++ backend/moonstream/actions.py | 9 +- backend/moonstream/middleware.py | 2 + backend/moonstream/providers/__init__.py | 15 ++- backend/moonstream/reporter.py | 18 +++ backend/moonstream/routes/address_info.py | 2 + backend/moonstream/routes/streams.py | 130 ++++++++++++++------- backend/moonstream/routes/subscriptions.py | 19 +-- backend/moonstream/routes/txinfo.py | 5 +- backend/moonstream/routes/users.py | 8 ++ backend/moonstream/settings.py | 2 + backend/requirements.txt | 1 + backend/sample.env | 3 +- backend/setup.py | 10 +- 14 files changed, 166 insertions(+), 65 deletions(-) create mode 100644 backend/moonstream/reporter.py diff --git a/backend/moonstream/__init__.py b/backend/moonstream/__init__.py index e69de29b..5df6f535 100644 --- a/backend/moonstream/__init__.py +++ b/backend/moonstream/__init__.py @@ -0,0 +1,7 @@ +from .reporter import reporter +from .version import MOONSTREAM_VERSION + +# Reporting +reporter.tags.append(f"version:{MOONSTREAM_VERSION}") +reporter.system_report(publish=True) +reporter.setup_excepthook(publish=True) diff --git a/backend/moonstream/actions.py b/backend/moonstream/actions.py index 22612cd8..4ecf8bd4 100644 --- a/backend/moonstream/actions.py +++ b/backend/moonstream/actions.py @@ -1,16 +1,18 @@ import json import logging -from typing import Dict, Any, List, Optional +from typing import Optional from enum import Enum + import boto3 # type: ignore from moonstreamdb.models import ( EthereumAddress, EthereumLabel, ) from sqlalchemy import text -from sqlalchemy.orm import Session, query, query_expression +from sqlalchemy.orm import Session from . import data +from .reporter import reporter from .settings import ETHERSCAN_SMARTCONTRACTS_BUCKET logger = logging.getLogger(__name__) @@ -46,8 +48,9 @@ def get_contract_source_info( abi=obj_data["ABI"], ) return contract_source_info - except: + except Exception as e: logger.error(f"Failed to load smart contract {object_uri}") + reporter.error_report(e) return None diff --git a/backend/moonstream/middleware.py b/backend/moonstream/middleware.py index e2a15175..a872095a 100644 --- a/backend/moonstream/middleware.py +++ b/backend/moonstream/middleware.py @@ -6,6 +6,7 @@ from bugout.exceptions import BugoutResponseException from starlette.middleware.base import BaseHTTPMiddleware from fastapi import Request, Response +from .reporter import reporter from .settings import MOONSTREAM_APPLICATION_ID, bugout_client as bc logger = logging.getLogger(__name__) @@ -61,6 +62,7 @@ class BroodAuthMiddleware(BaseHTTPMiddleware): return Response(status_code=e.status_code, content=e.detail) except Exception as e: logger.error(f"Error processing Brood response: {str(e)}") + reporter.error_report(e) return Response(status_code=500, content="Internal server error") request.state.user = user diff --git a/backend/moonstream/providers/__init__.py b/backend/moonstream/providers/__init__.py index fbb2ec7d..f884fe1b 100644 --- a/backend/moonstream/providers/__init__.py +++ b/backend/moonstream/providers/__init__.py @@ -39,6 +39,13 @@ from ..stream_queries import StreamQuery logger = logging.getLogger(__name__) logger.setLevel(logging.WARN) + +class ReceivingEventsException(Exception): + """ + Raised when error occurs during receiving events from provider. + """ + + event_providers: Dict[str, Any] = { ethereum_blockchain.event_type: ethereum_blockchain, bugout.whalewatch_provider.event_type: bugout.whalewatch_provider, @@ -91,7 +98,7 @@ def get_events( f"Error receiving events from provider: {provider_name}:\n{repr(e)}" ) else: - raise e + raise ReceivingEventsException(e) events = [event for _, event_list in results.values() for event in event_list] if sort_events: @@ -149,7 +156,7 @@ def latest_events( f"Error receiving events from provider: {provider_name}:\n{repr(e)}" ) else: - raise e + raise ReceivingEventsException(e) events = [event for event_list in results.values() for event in event_list] if sort_events: @@ -202,7 +209,7 @@ def next_event( f"Error receiving events from provider: {provider_name}:\n{repr(e)}" ) else: - raise e + raise ReceivingEventsException(e) event: Optional[data.Event] = None for candidate in results.values(): @@ -258,7 +265,7 @@ def previous_event( f"Error receiving events from provider: {provider_name}:\n{repr(e)}" ) else: - raise e + raise ReceivingEventsException(e) event: Optional[data.Event] = None for candidate in results.values(): diff --git a/backend/moonstream/reporter.py b/backend/moonstream/reporter.py new file mode 100644 index 00000000..0943c327 --- /dev/null +++ b/backend/moonstream/reporter.py @@ -0,0 +1,18 @@ +import uuid + +from humbug.consent import HumbugConsent +from humbug.report import HumbugReporter + +from .settings import BUGOUT_HUMBUG_TOKEN + +session_id = str(uuid.uuid4()) +client_id = "moonstream-backend" + +reporter = HumbugReporter( + name="moonstream", + consent=HumbugConsent(True), + client_id=client_id, + session_id=session_id, + bugout_token=BUGOUT_HUMBUG_TOKEN, + tags=[], +) diff --git a/backend/moonstream/routes/address_info.py b/backend/moonstream/routes/address_info.py index 2b95bb92..b9794937 100644 --- a/backend/moonstream/routes/address_info.py +++ b/backend/moonstream/routes/address_info.py @@ -11,6 +11,7 @@ from sqlalchemy.orm import Session from .. import actions from .. import data from ..middleware import BroodAuthMiddleware +from ..reporter import reporter from ..settings import DOCS_TARGET_PATH, ORIGINS, DOCS_PATHS from ..version import MOONSTREAM_VERSION @@ -82,6 +83,7 @@ async def addresses_labels_bulk_handler( ) except Exception as err: logger.error(f"Unable to get info about Ethereum addresses {err}") + reporter.error_report(err) raise HTTPException(status_code=500) return addresses_response diff --git a/backend/moonstream/routes/streams.py b/backend/moonstream/routes/streams.py index be46bc58..35bfe0a2 100644 --- a/backend/moonstream/routes/streams.py +++ b/backend/moonstream/routes/streams.py @@ -14,12 +14,14 @@ from sqlalchemy.orm import Session from .. import data from ..middleware import BroodAuthMiddleware from ..providers import ( + ReceivingEventsException, event_providers, get_events, latest_events, next_event, previous_event, ) +from ..reporter import reporter from ..settings import ( DOCS_TARGET_PATH, MOONSTREAM_ADMIN_ACCESS_TOKEN, @@ -121,17 +123,27 @@ async def stream_handler( if q.strip() != "": query = stream_queries.parse_query_string(q) - _, events = get_events( - db_session, - bc, - MOONSTREAM_DATA_JOURNAL_ID, - MOONSTREAM_ADMIN_ACCESS_TOKEN, - stream_boundary, - query, - user_subscriptions, - result_timeout=10.0, - raise_on_error=True, - ) + try: + _, events = get_events( + db_session, + bc, + MOONSTREAM_DATA_JOURNAL_ID, + MOONSTREAM_ADMIN_ACCESS_TOKEN, + stream_boundary, + query, + user_subscriptions, + result_timeout=10.0, + raise_on_error=True, + ) + except ReceivingEventsException as e: + logger.error("Error receiving events from provider") + reporter.error_report(e) + raise HTTPException(status_code=500) + except Exception as e: + logger.error("Unable to get events") + reporter.error_report(e) + raise HTTPException(status_code=500) + response = data.GetEventsResponse(stream_boundary=stream_boundary, events=events) return response @@ -155,18 +167,28 @@ async def latest_events_handler( if q.strip() != "": query = stream_queries.parse_query_string(q) - events = latest_events( - db_session, - bc, - MOONSTREAM_DATA_JOURNAL_ID, - MOONSTREAM_ADMIN_ACCESS_TOKEN, - query, - 1, - user_subscriptions, - result_timeout=6.0, - raise_on_error=True, - sort_events=True, - ) + try: + events = latest_events( + db_session, + bc, + MOONSTREAM_DATA_JOURNAL_ID, + MOONSTREAM_ADMIN_ACCESS_TOKEN, + query, + 1, + user_subscriptions, + result_timeout=6.0, + raise_on_error=True, + sort_events=True, + ) + except ReceivingEventsException as e: + logger.error("Error receiving events from provider") + reporter.error_report(e) + raise HTTPException(status_code=500) + except Exception as e: + logger.error("Unable to get latest events") + reporter.error_report(e) + raise HTTPException(status_code=500) + return events @@ -203,17 +225,26 @@ async def next_event_handler( if q.strip() != "": query = stream_queries.parse_query_string(q) - event = next_event( - db_session, - bc, - MOONSTREAM_DATA_JOURNAL_ID, - MOONSTREAM_ADMIN_ACCESS_TOKEN, - stream_boundary, - query, - user_subscriptions, - result_timeout=6.0, - raise_on_error=True, - ) + try: + event = next_event( + db_session, + bc, + MOONSTREAM_DATA_JOURNAL_ID, + MOONSTREAM_ADMIN_ACCESS_TOKEN, + stream_boundary, + query, + user_subscriptions, + result_timeout=6.0, + raise_on_error=True, + ) + except ReceivingEventsException as e: + logger.error("Error receiving events from provider") + reporter.error_report(e) + raise HTTPException(status_code=500) + except Exception as e: + logger.error("Unable to get next events") + reporter.error_report(e) + raise HTTPException(status_code=500) return event @@ -251,16 +282,25 @@ async def previous_event_handler( if q.strip() != "": query = stream_queries.parse_query_string(q) - event = previous_event( - db_session, - bc, - MOONSTREAM_DATA_JOURNAL_ID, - MOONSTREAM_ADMIN_ACCESS_TOKEN, - stream_boundary, - query, - user_subscriptions, - result_timeout=6.0, - raise_on_error=True, - ) + try: + event = previous_event( + db_session, + bc, + MOONSTREAM_DATA_JOURNAL_ID, + MOONSTREAM_ADMIN_ACCESS_TOKEN, + stream_boundary, + query, + user_subscriptions, + result_timeout=6.0, + raise_on_error=True, + ) + except ReceivingEventsException as e: + logger.error("Error receiving events from provider") + reporter.error_report(e) + raise HTTPException(status_code=500) + except Exception as e: + logger.error("Unable to get previous events") + reporter.error_report(e) + raise HTTPException(status_code=500) return event diff --git a/backend/moonstream/routes/subscriptions.py b/backend/moonstream/routes/subscriptions.py index 75c854dc..f05eea0e 100644 --- a/backend/moonstream/routes/subscriptions.py +++ b/backend/moonstream/routes/subscriptions.py @@ -12,6 +12,7 @@ from fastapi.middleware.cors import CORSMiddleware from ..admin import subscription_types from .. import data from ..middleware import BroodAuthMiddleware +from ..reporter import reporter from ..settings import ( DOCS_TARGET_PATH, DOCS_PATHS, @@ -100,8 +101,8 @@ async def add_subscription_handler( resource_data=resource_data, ) except Exception as e: - logger.error("Error creating subscription resource:") - logger.error(e) + logger.error(f"Error creating subscription resource: {str(e)}") + reporter.error_report(e) raise HTTPException(status_code=500) return data.SubscriptionResourceData( @@ -123,13 +124,14 @@ async def delete_subscription_handler(request: Request, subscription_id: str): """ Delete subscriptions. """ - token = request.state.token try: deleted_resource = bc.delete_resource(token=token, resource_id=subscription_id) except BugoutResponseException as e: raise HTTPException(status_code=e.status_code, detail=e.detail) except Exception as e: + logger.error(f"Error deleting subscription: {str(e)}") + reporter.error_report(e) raise HTTPException(status_code=500) return data.SubscriptionResourceData( @@ -156,9 +158,9 @@ async def get_subscriptions_handler(request: Request) -> data.SubscriptionsListR resources: BugoutResources = bc.list_resources(token=token, params=params) except Exception as e: logger.error( - f"Error listing subscriptions for user ({request.user.id}) with token ({request.state.token})" + f"Error listing subscriptions for user ({request.user.id}) with token ({request.state.token}), error: {str(e)}" ) - logger.error(e) + reporter.error_report(e) raise HTTPException(status_code=500) return data.SubscriptionsListResponse( @@ -190,7 +192,6 @@ async def update_subscriptions_handler( """ Get user's subscriptions. """ - token = request.state.token update = {} @@ -212,6 +213,8 @@ async def update_subscriptions_handler( except BugoutResponseException as e: raise HTTPException(status_code=e.status_code, detail=e.detail) except Exception as e: + logger.error(f"Error getting user subscriptions: {str(e)}") + reporter.error_report(e) raise HTTPException(status_code=500) return data.SubscriptionResourceData( @@ -239,8 +242,8 @@ async def list_subscription_types() -> data.SubscriptionTypesListResponse: for resource in response.resources ] except Exception as e: - logger.error("Error reading subscription types from Brood API:") - logger.error(e) + logger.error(f"Error reading subscription types from Brood API: {str(e)}") + reporter.error_report(e) raise HTTPException(status_code=500) return data.SubscriptionTypesListResponse(subscription_types=results) diff --git a/backend/moonstream/routes/txinfo.py b/backend/moonstream/routes/txinfo.py index 20eec3c7..8ab9a5ec 100644 --- a/backend/moonstream/routes/txinfo.py +++ b/backend/moonstream/routes/txinfo.py @@ -6,9 +6,7 @@ transactions, etc.) with side information and return objects that are better sui end users. """ import logging -from typing import Dict, Optional - -from sqlalchemy.sql.expression import true +from typing import Dict from fastapi import FastAPI, Depends from fastapi.middleware.cors import CORSMiddleware @@ -54,6 +52,7 @@ app.add_middleware(BroodAuthMiddleware, whitelist=whitelist_paths) # TODO(zomglings): Factor out the enrichment logic into a separate action, because it may be useful # independently from serving API calls (e.g. data processing). +# TODO(kompotkot): Re-organize function to be able handle each steps with exceptions. @app.post( "/ethereum_blockchain", tags=["txinfo"], diff --git a/backend/moonstream/routes/users.py b/backend/moonstream/routes/users.py index 5101a25c..60090740 100644 --- a/backend/moonstream/routes/users.py +++ b/backend/moonstream/routes/users.py @@ -16,6 +16,7 @@ from fastapi import ( from fastapi.middleware.cors import CORSMiddleware from ..middleware import BroodAuthMiddleware +from ..reporter import reporter from ..settings import ( MOONSTREAM_APPLICATION_ID, DOCS_TARGET_PATH, @@ -77,6 +78,7 @@ async def create_user_handler( except BugoutResponseException as e: raise HTTPException(status_code=e.status_code, detail=e.detail) except Exception as e: + reporter.error_report(e) raise HTTPException(status_code=500) return user @@ -94,6 +96,7 @@ async def restore_password_handler(email: str = Form(...)) -> Dict[str, Any]: except BugoutResponseException as e: raise HTTPException(status_code=e.status_code, detail=e.detail) except Exception as e: + reporter.error_report(e) raise HTTPException(status_code=500) return response @@ -107,6 +110,7 @@ async def reset_password_handler( except BugoutResponseException as e: raise HTTPException(status_code=e.status_code, detail=e.detail) except Exception as e: + reporter.error_report(e) raise HTTPException(status_code=500) return response @@ -123,6 +127,7 @@ async def change_password_handler( except BugoutResponseException as e: raise HTTPException(status_code=e.status_code, detail=e.detail) except Exception as e: + reporter.error_report(e) raise HTTPException(status_code=500) return user @@ -138,6 +143,7 @@ async def delete_user_handler( except BugoutResponseException as e: raise HTTPException(status_code=e.status_code, detail=e.detail) except Exception as e: + reporter.error_report(e) raise HTTPException(status_code=500) return user @@ -157,6 +163,7 @@ async def login_handler( status_code=e.status_code, detail=f"Error from Brood API: {e.detail}" ) except Exception as e: + reporter.error_report(e) raise HTTPException(status_code=500) return token @@ -169,5 +176,6 @@ async def logout_handler(request: Request) -> uuid.UUID: except BugoutResponseException as e: raise HTTPException(status_code=e.status_code, detail=e.detail) except Exception as e: + reporter.error_report(e) raise HTTPException(status_code=500) return token_id diff --git a/backend/moonstream/settings.py b/backend/moonstream/settings.py index 6408b1d8..b38ea7db 100644 --- a/backend/moonstream/settings.py +++ b/backend/moonstream/settings.py @@ -9,6 +9,8 @@ bugout_client = Bugout(brood_api_url=BUGOUT_BROOD_URL, spire_api_url=BUGOUT_SPIR BUGOUT_REQUEST_TIMEOUT_SECONDS = 5 +BUGOUT_HUMBUG_TOKEN = os.environ.get("BUGOUT_HUMBUG_TOKEN") + # Default value is "" instead of None so that mypy understands that MOONSTREAM_APPLICATION_ID is a string MOONSTREAM_APPLICATION_ID = os.environ.get("MOONSTREAM_APPLICATION_ID", "") if MOONSTREAM_APPLICATION_ID == "": diff --git a/backend/requirements.txt b/backend/requirements.txt index 44e49539..6ebf10e0 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -11,6 +11,7 @@ fastapi==0.66.0 h11==0.12.0 idna==3.2 jmespath==0.10.0 +humbug==0.2.7 -e git+https://git@github.com/bugout-dev/moonstream.git@94135b054cabb9dc11b0a2406431619279979469#egg=moonstreamdb&subdirectory=db mypy==0.910 mypy-extensions==0.4.3 diff --git a/backend/sample.env b/backend/sample.env index 054e040d..bc95e6a6 100644 --- a/backend/sample.env +++ b/backend/sample.env @@ -5,6 +5,7 @@ export MOONSTREAM_DATA_JOURNAL_ID="" export MOONSTREAM_DB_URI="postgresql://:@:/" export MOONSTREAM_POOL_SIZE=0 export MOONSTREAM_ADMIN_ACCESS_TOKEN="" -export AWS_S3_SMARTCONTRACT_BUCKET="" +export AWS_S3_SMARTCONTRACT_BUCKET="" export BUGOUT_BROOD_URL="https://auth.bugout.dev" export BUGOUT_SPIRE_URL="https://spire.bugout.dev" +export BUGOUT_HUMBUG_TOKEN="" diff --git a/backend/setup.py b/backend/setup.py index 0cd9b6e5..7cc23eee 100644 --- a/backend/setup.py +++ b/backend/setup.py @@ -10,7 +10,15 @@ setup( name="moonstream", version=MOONSTREAM_VERSION, packages=find_packages(), - install_requires=["boto3", "bugout >= 0.1.17", "fastapi", "python-dateutil", "uvicorn", "types-python-dateutil"], + install_requires=[ + "boto3", + "bugout >= 0.1.17", + "fastapi", + "humbug>=0.2.7", + "python-dateutil", + "uvicorn", + "types-python-dateutil", + ], extras_require={ "dev": ["black", "mypy"], "distribute": ["setuptools", "twine", "wheel"], From 44fb2bc8c70f3450cd870da291c0bd0202f7f17b Mon Sep 17 00:00:00 2001 From: kompotkot Date: Wed, 1 Sep 2021 13:41:31 +0000 Subject: [PATCH 16/32] Humbug reporter for moonstream python crawlers --- crawlers/mooncrawl/mooncrawl/__init__.py | 7 +++++++ crawlers/mooncrawl/mooncrawl/ethcrawler.py | 2 +- crawlers/mooncrawl/mooncrawl/reporter.py | 18 ++++++++++++++++++ crawlers/mooncrawl/mooncrawl/settings.py | 6 +++++- crawlers/mooncrawl/sample.env | 1 + crawlers/mooncrawl/setup.py | 1 + 6 files changed, 33 insertions(+), 2 deletions(-) create mode 100644 crawlers/mooncrawl/mooncrawl/reporter.py diff --git a/crawlers/mooncrawl/mooncrawl/__init__.py b/crawlers/mooncrawl/mooncrawl/__init__.py index e69de29b..9548e297 100644 --- a/crawlers/mooncrawl/mooncrawl/__init__.py +++ b/crawlers/mooncrawl/mooncrawl/__init__.py @@ -0,0 +1,7 @@ +from .reporter import reporter +from .version import MOONCRAWL_VERSION + +# Reporting +reporter.tags.append(f"version:{MOONCRAWL_VERSION}") +reporter.system_report(publish=True) +reporter.setup_excepthook(publish=True) diff --git a/crawlers/mooncrawl/mooncrawl/ethcrawler.py b/crawlers/mooncrawl/mooncrawl/ethcrawler.py index 793ab3ea..5e9275c4 100644 --- a/crawlers/mooncrawl/mooncrawl/ethcrawler.py +++ b/crawlers/mooncrawl/mooncrawl/ethcrawler.py @@ -48,7 +48,7 @@ def yield_blocks_numbers_lists( print( "Wrong format provided, expected {bottom_block}-{top_block}, as ex. 105-340" ) - return + raise Exception starting_block = max(input_start_block, input_end_block) ending_block = min(input_start_block, input_end_block) diff --git a/crawlers/mooncrawl/mooncrawl/reporter.py b/crawlers/mooncrawl/mooncrawl/reporter.py new file mode 100644 index 00000000..96f3de4a --- /dev/null +++ b/crawlers/mooncrawl/mooncrawl/reporter.py @@ -0,0 +1,18 @@ +import uuid + +from humbug.consent import HumbugConsent +from humbug.report import HumbugReporter + +from .settings import HUMBUG_REPORTER_CRAWLERS_TOKEN + +session_id = str(uuid.uuid4()) +client_id = "moonstream-crawlers" + +reporter = HumbugReporter( + name="moonstream", + consent=HumbugConsent(True), + client_id=client_id, + session_id=session_id, + bugout_token=HUMBUG_REPORTER_CRAWLERS_TOKEN, + tags=[], +) diff --git a/crawlers/mooncrawl/mooncrawl/settings.py b/crawlers/mooncrawl/mooncrawl/settings.py index 82b13772..307f2e8a 100644 --- a/crawlers/mooncrawl/mooncrawl/settings.py +++ b/crawlers/mooncrawl/mooncrawl/settings.py @@ -1,5 +1,9 @@ import os +# Bugout +HUMBUG_REPORTER_CRAWLERS_TOKEN = os.environ.get("HUMBUG_REPORTER_CRAWLERS_TOKEN") + +# Geth MOONSTREAM_IPC_PATH = os.environ.get("MOONSTREAM_IPC_PATH", None) MOONSTREAM_CRAWL_WORKERS = 4 @@ -12,5 +16,5 @@ except: f"Could not parse MOONSTREAM_CRAWL_WORKERS as int: {MOONSTREAM_CRAWL_WORKERS_RAW}" ) - +# Etherscan MOONSTREAM_ETHERSCAN_TOKEN = os.environ.get("MOONSTREAM_ETHERSCAN_TOKEN") diff --git a/crawlers/mooncrawl/sample.env b/crawlers/mooncrawl/sample.env index 5ebad6e0..5d4ac548 100644 --- a/crawlers/mooncrawl/sample.env +++ b/crawlers/mooncrawl/sample.env @@ -6,3 +6,4 @@ export MOONSTREAM_ETHERSCAN_TOKEN="" export AWS_S3_SMARTCONTRACT_BUCKET="" export MOONSTREAM_HUMBUG_TOKEN="" export COINMARKETCAP_API_KEY="" +export HUMBUG_REPORTER_CRAWLERS_TOKEN="" diff --git a/crawlers/mooncrawl/setup.py b/crawlers/mooncrawl/setup.py index 1e893bd4..46b397a3 100644 --- a/crawlers/mooncrawl/setup.py +++ b/crawlers/mooncrawl/setup.py @@ -34,6 +34,7 @@ setup( zip_safe=False, install_requires=[ "moonstreamdb @ git+https://git@github.com/bugout-dev/moonstream.git@39d2b8e36a49958a9ae085ec2cc1be3fc732b9d0#egg=moonstreamdb&subdirectory=db", + "humbug", "python-dateutil", "requests", "tqdm", From c34f9245c087cf8bb1c56530e5ac00e52d975d2d Mon Sep 17 00:00:00 2001 From: kompotkot Date: Wed, 1 Sep 2021 13:43:44 +0000 Subject: [PATCH 17/32] Renamed env variable for crash reports token --- backend/moonstream/reporter.py | 4 ++-- backend/moonstream/settings.py | 2 +- backend/sample.env | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/backend/moonstream/reporter.py b/backend/moonstream/reporter.py index 0943c327..1ba0997b 100644 --- a/backend/moonstream/reporter.py +++ b/backend/moonstream/reporter.py @@ -3,7 +3,7 @@ import uuid from humbug.consent import HumbugConsent from humbug.report import HumbugReporter -from .settings import BUGOUT_HUMBUG_TOKEN +from .settings import HUMBUG_REPORTER_BACKEND_TOKEN session_id = str(uuid.uuid4()) client_id = "moonstream-backend" @@ -13,6 +13,6 @@ reporter = HumbugReporter( consent=HumbugConsent(True), client_id=client_id, session_id=session_id, - bugout_token=BUGOUT_HUMBUG_TOKEN, + bugout_token=HUMBUG_REPORTER_BACKEND_TOKEN, tags=[], ) diff --git a/backend/moonstream/settings.py b/backend/moonstream/settings.py index b38ea7db..0a90d751 100644 --- a/backend/moonstream/settings.py +++ b/backend/moonstream/settings.py @@ -9,7 +9,7 @@ bugout_client = Bugout(brood_api_url=BUGOUT_BROOD_URL, spire_api_url=BUGOUT_SPIR BUGOUT_REQUEST_TIMEOUT_SECONDS = 5 -BUGOUT_HUMBUG_TOKEN = os.environ.get("BUGOUT_HUMBUG_TOKEN") +HUMBUG_REPORTER_BACKEND_TOKEN = os.environ.get("HUMBUG_REPORTER_BACKEND_TOKEN") # Default value is "" instead of None so that mypy understands that MOONSTREAM_APPLICATION_ID is a string MOONSTREAM_APPLICATION_ID = os.environ.get("MOONSTREAM_APPLICATION_ID", "") diff --git a/backend/sample.env b/backend/sample.env index bc95e6a6..779bbb0c 100644 --- a/backend/sample.env +++ b/backend/sample.env @@ -8,4 +8,4 @@ export MOONSTREAM_ADMIN_ACCESS_TOKEN="" export AWS_S3_SMARTCONTRACT_BUCKET="" export BUGOUT_BROOD_URL="https://auth.bugout.dev" export BUGOUT_SPIRE_URL="https://spire.bugout.dev" -export BUGOUT_HUMBUG_TOKEN="" +export HUMBUG_REPORTER_BACKEND_TOKEN="" From ec19d9e739dfa26f841fe43e7f5180fdff644212 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Wed, 1 Sep 2021 13:56:11 +0000 Subject: [PATCH 18/32] Specified a name for report for crawlers --- crawlers/mooncrawl/mooncrawl/reporter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crawlers/mooncrawl/mooncrawl/reporter.py b/crawlers/mooncrawl/mooncrawl/reporter.py index 96f3de4a..0cf170ac 100644 --- a/crawlers/mooncrawl/mooncrawl/reporter.py +++ b/crawlers/mooncrawl/mooncrawl/reporter.py @@ -9,7 +9,7 @@ session_id = str(uuid.uuid4()) client_id = "moonstream-crawlers" reporter = HumbugReporter( - name="moonstream", + name="moonstream-crawlers", consent=HumbugConsent(True), client_id=client_id, session_id=session_id, From 8bbfd1e911d9621c3839fc06fb89fd7130fd8126 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Wed, 1 Sep 2021 14:45:05 +0000 Subject: [PATCH 19/32] Humbug crash reporter for moonstream crawlers --- crawlers/ethtxpool/main.go | 28 +++++++++++++++++++++------- crawlers/ethtxpool/sample.env | 1 + 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/crawlers/ethtxpool/main.go b/crawlers/ethtxpool/main.go index 224ae7dc..217ec8fa 100644 --- a/crawlers/ethtxpool/main.go +++ b/crawlers/ethtxpool/main.go @@ -21,12 +21,8 @@ import ( "github.com/google/uuid" ) -// Generate humbug client to be able write data in Bugout journal. -func humbugClientFromEnv() (*humbug.HumbugReporter, error) { - clientID := os.Getenv("ETHTXPOOL_HUMBUG_CLIENT_ID") - humbugToken := os.Getenv("ETHTXPOOL_HUMBUG_TOKEN") - sessionID := uuid.New().String() - +// Generate humbug client +func humbugClient(sessionID string, clientID string, humbugToken string) (*humbug.HumbugReporter, error) { consent := humbug.CreateHumbugConsent(humbug.True) reporter, err := humbug.CreateHumbugReporter(consent, clientID, sessionID, humbugToken) return reporter, err @@ -188,6 +184,23 @@ func main() { flag.IntVar(&intervalSeconds, "interval", 1, "Number of seconds to wait between RPC calls to query the transaction pool (default: 1)") flag.Parse() + sessionID := uuid.New().String() + + // Humbug crash client to collect errors + crashReporter, err := humbugClient(sessionID, "moonstream-crawlers", os.Getenv("HUMBUG_REPORTER_CRAWLERS_TOKEN")) + if err != nil { + panic(fmt.Sprintf("Invalid Humbug Crash configuration: %s", err.Error())) + } + crashReporter.Publish(humbug.SystemReport()) + + defer func() { + message := recover() + if message != nil { + fmt.Printf("Error: %s\n", message) + crashReporter.Publish(humbug.PanicReport(message)) + } + }() + // Set connection with Ethereum blockchain via geth gethClient, err := rpc.Dial(gethConnectionString) if err != nil { @@ -195,7 +208,8 @@ func main() { } defer gethClient.Close() - reporter, err := humbugClientFromEnv() + // Humbug client to be able write data in Bugout journal + reporter, err := humbugClient(sessionID, os.Getenv("ETHTXPOOL_HUMBUG_CLIENT_ID"), os.Getenv("ETHTXPOOL_HUMBUG_TOKEN")) if err != nil { panic(fmt.Sprintf("Invalid Humbug configuration: %s", err.Error())) } diff --git a/crawlers/ethtxpool/sample.env b/crawlers/ethtxpool/sample.env index abd4bc8e..809cc325 100644 --- a/crawlers/ethtxpool/sample.env +++ b/crawlers/ethtxpool/sample.env @@ -1,2 +1,3 @@ export ETHTXPOOL_HUMBUG_CLIENT_ID="" export ETHTXPOOL_HUMBUG_TOKEN="" +export HUMBUG_REPORTER_CRAWLERS_TOKEN="" From 78141cf626d429f4250e54cc3e0f4d5eef41c343 Mon Sep 17 00:00:00 2001 From: Andrey Dolgolev Date: Wed, 1 Sep 2021 20:29:40 +0300 Subject: [PATCH 20/32] Add migration file. --- ..._add_opensea_state_table_and_add_index_.py | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 db/alembic/versions/ecb7817db377_add_opensea_state_table_and_add_index_.py diff --git a/db/alembic/versions/ecb7817db377_add_opensea_state_table_and_add_index_.py b/db/alembic/versions/ecb7817db377_add_opensea_state_table_and_add_index_.py new file mode 100644 index 00000000..47f3572a --- /dev/null +++ b/db/alembic/versions/ecb7817db377_add_opensea_state_table_and_add_index_.py @@ -0,0 +1,49 @@ +"""Add opensea state table and add index by label_data ->> name + +Revision ID: ecb7817db377 +Revises: ea8185bd24c7 +Create Date: 2021-08-31 17:44:24.139028 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = "ecb7817db377" +down_revision = "ea8185bd24c7" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "opensea_crawler_state", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("query", sa.Text(), nullable=False), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.text("TIMEZONE('utc', statement_timestamp())"), + nullable=False, + ), + sa.Column("total_count", sa.Integer(), nullable=False), + sa.PrimaryKeyConstraint("id", name=op.f("pk_opensea_crawler_state")), + sa.UniqueConstraint("id", name=op.f("uq_opensea_crawler_state_id")), + ) + op.drop_constraint("uq_ethereum_labels_label", "ethereum_labels", type_="unique") + op.create_index( + op.f("idx_label_name"), "ethereum_labels", [sa.text("(label_data->'name')")] + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_unique_constraint( + "uq_ethereum_labels_label", "ethereum_labels", ["label", "address_id"] + ) + op.drop_table("opensea_crawler_state") + op.drop_index("idx_label_name") + # ### end Alembic commands ### From 7bc722661216fa638529cae614ba60683c18be4a Mon Sep 17 00:00:00 2001 From: Andrey Dolgolev Date: Wed, 1 Sep 2021 20:32:11 +0300 Subject: [PATCH 21/32] Remove inrequred index. --- db/moonstreamdb/models.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/db/moonstreamdb/models.py b/db/moonstreamdb/models.py index f917db55..391724bc 100644 --- a/db/moonstreamdb/models.py +++ b/db/moonstreamdb/models.py @@ -11,8 +11,6 @@ from sqlalchemy import ( Numeric, Text, VARCHAR, - UniqueConstraint, - Index, ) from sqlalchemy.dialects.postgresql import JSONB, UUID from sqlalchemy.sql import expression, text @@ -156,7 +154,6 @@ class EthereumLabel(Base): # type: ignore created_at = Column( DateTime(timezone=True), server_default=utcnow(), nullable=False ) - __table_args__ = (Index("ix_label_name", text("(label_data->>'name')")),) class EthereumPendingTransaction(Base): # type: ignore From 6388716156ff9fb4ade5392584c0d7c85d65de91 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Wed, 1 Sep 2021 17:34:02 +0000 Subject: [PATCH 22/32] Hardcoded version in setup.py --- crawlers/mooncrawl/setup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crawlers/mooncrawl/setup.py b/crawlers/mooncrawl/setup.py index 46b397a3..f26b82d5 100644 --- a/crawlers/mooncrawl/setup.py +++ b/crawlers/mooncrawl/setup.py @@ -1,6 +1,5 @@ from setuptools import find_packages, setup -from mooncrawl.version import MOONCRAWL_VERSION long_description = "" with open("README.md") as ifp: @@ -8,7 +7,7 @@ with open("README.md") as ifp: setup( name="mooncrawl", - version=MOONCRAWL_VERSION, + version="0.0.3", author="Bugout.dev", author_email="engineers@bugout.dev", license="Apache License 2.0", From 5ca122612a8e7ccfdeb56e6e52b215f1cc474ddc Mon Sep 17 00:00:00 2001 From: Tim Pechersky Date: Wed, 1 Sep 2021 19:45:32 +0200 Subject: [PATCH 23/32] remove analytics from hubspot analytics page --- frontend/pages/analytics.js | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/frontend/pages/analytics.js b/frontend/pages/analytics.js index a1666c37..fb320883 100644 --- a/frontend/pages/analytics.js +++ b/frontend/pages/analytics.js @@ -3,9 +3,6 @@ import HubspotForm from "react-hubspot-form"; import { getLayout } from "../src/layouts/AppLayout"; import { Spinner, Flex, Heading } from "@chakra-ui/react"; import Scrollable from "../src/components/Scrollable"; -import mixpanel from "mixpanel-browser"; -import MIXPANEL_EVENTS from "../src/core/providers/AnalyticsProvider/constants"; -import { useUser } from "../src/core/hooks"; const Analytics = () => { useEffect(() => { @@ -14,8 +11,6 @@ const Analytics = () => { } }, []); - const { user } = useUser(); - return ( { portalId="8018701" formId="39bc0fbe-41c4-430a-b885-46eba66c59c2" loading={} - onSubmit={() => - mixpanel.track(MIXPANEL_EVENTS.FORM_SUBMITTED, { - formType: "Hubspot analytics", - user: user, - }) - } /> From 6ac9df74aa3a7a21d80276472f8a1040e439a1d2 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Thu, 2 Sep 2021 10:49:23 +0000 Subject: [PATCH 24/32] Extended HTTPException and Response with error handling --- backend/moonstream/middleware.py | 48 ++++++++++++++++++++-- backend/moonstream/routes/address_info.py | 14 +++---- backend/moonstream/routes/streams.py | 29 +++++-------- backend/moonstream/routes/subscriptions.py | 30 +++++++------- backend/moonstream/routes/users.py | 45 ++++++++------------ backend/requirements.txt | 1 + backend/setup.py | 1 + 7 files changed, 94 insertions(+), 74 deletions(-) diff --git a/backend/moonstream/middleware.py b/backend/moonstream/middleware.py index a872095a..7bb94a84 100644 --- a/backend/moonstream/middleware.py +++ b/backend/moonstream/middleware.py @@ -1,10 +1,11 @@ import logging -from typing import Awaitable, Callable, Dict, Optional +from typing import Any, Awaitable, Callable, Dict, Optional from bugout.data import BugoutUser from bugout.exceptions import BugoutResponseException +from fastapi import HTTPException, Request, Response +from starlette.background import BackgroundTask from starlette.middleware.base import BaseHTTPMiddleware -from fastapi import Request, Response from .reporter import reporter from .settings import MOONSTREAM_APPLICATION_ID, bugout_client as bc @@ -12,6 +13,44 @@ from .settings import MOONSTREAM_APPLICATION_ID, bugout_client as bc logger = logging.getLogger(__name__) +class MoonstreamResponse(Response): + """ + Extended Response to handle 500 Internal server errors + and send crash reports. + """ + + def __init__( + self, + content: Any = None, + status_code: int = 200, + headers: dict = None, + media_type: str = None, + background: BackgroundTask = None, + internal_error: Exception = None, + ): + super().__init__(content, status_code, headers, media_type, background) + if internal_error is not None: + reporter.error_report(internal_error) + + +class MoonstreamHTTPException(HTTPException): + """ + Extended HTTPException to handle 500 Internal server errors + and send crash reports. + """ + + def __init__( + self, + status_code: int, + detail: Any = None, + headers: Optional[Dict[str, Any]] = None, + internal_error: Exception = None, + ): + super().__init__(status_code, detail, headers) + if internal_error is not None: + reporter.error_report(internal_error) + + class BroodAuthMiddleware(BaseHTTPMiddleware): """ Checks the authorization header on the request. If it represents a verified Brood user, @@ -62,8 +101,9 @@ class BroodAuthMiddleware(BaseHTTPMiddleware): return Response(status_code=e.status_code, content=e.detail) except Exception as e: logger.error(f"Error processing Brood response: {str(e)}") - reporter.error_report(e) - return Response(status_code=500, content="Internal server error") + return MoonstreamResponse( + status_code=500, content="Internal server error", internal_error=e + ) request.state.user = user request.state.token = user_token diff --git a/backend/moonstream/routes/address_info.py b/backend/moonstream/routes/address_info.py index b9794937..c985c683 100644 --- a/backend/moonstream/routes/address_info.py +++ b/backend/moonstream/routes/address_info.py @@ -3,15 +3,14 @@ from typing import Dict, List, Optional from sqlalchemy.sql.expression import true -from fastapi import FastAPI, Depends, Query, HTTPException +from fastapi import FastAPI, Depends, Query from fastapi.middleware.cors import CORSMiddleware from moonstreamdb.db import yield_db_session from sqlalchemy.orm import Session from .. import actions from .. import data -from ..middleware import BroodAuthMiddleware -from ..reporter import reporter +from ..middleware import BroodAuthMiddleware, MoonstreamHTTPException from ..settings import DOCS_TARGET_PATH, ORIGINS, DOCS_PATHS from ..version import MOONSTREAM_VERSION @@ -74,16 +73,15 @@ async def addresses_labels_bulk_handler( about known addresses. """ if limit > 100: - raise HTTPException( + raise MoonstreamHTTPException( status_code=406, detail="The limit cannot exceed 100 addresses" ) try: addresses_response = actions.get_address_labels( db_session=db_session, start=start, limit=limit, addresses=addresses ) - except Exception as err: - logger.error(f"Unable to get info about Ethereum addresses {err}") - reporter.error_report(err) - raise HTTPException(status_code=500) + except Exception as e: + logger.error(f"Unable to get info about Ethereum addresses {e}") + raise MoonstreamHTTPException(status_code=500, internal_error=e) return addresses_response diff --git a/backend/moonstream/routes/streams.py b/backend/moonstream/routes/streams.py index 35bfe0a2..10d0426c 100644 --- a/backend/moonstream/routes/streams.py +++ b/backend/moonstream/routes/streams.py @@ -5,14 +5,14 @@ import logging from typing import Dict, List, Optional from bugout.data import BugoutResource -from fastapi import FastAPI, HTTPException, Request, Query, Depends +from fastapi import FastAPI, Request, Query, Depends from fastapi.middleware.cors import CORSMiddleware from moonstreamdb import db from sqlalchemy.orm import Session from .. import data -from ..middleware import BroodAuthMiddleware +from ..middleware import BroodAuthMiddleware, MoonstreamHTTPException from ..providers import ( ReceivingEventsException, event_providers, @@ -21,7 +21,6 @@ from ..providers import ( next_event, previous_event, ) -from ..reporter import reporter from ..settings import ( DOCS_TARGET_PATH, MOONSTREAM_ADMIN_ACCESS_TOKEN, @@ -137,12 +136,10 @@ async def stream_handler( ) except ReceivingEventsException as e: logger.error("Error receiving events from provider") - reporter.error_report(e) - raise HTTPException(status_code=500) + raise MoonstreamHTTPException(status_code=500, internal_error=e) except Exception as e: logger.error("Unable to get events") - reporter.error_report(e) - raise HTTPException(status_code=500) + raise MoonstreamHTTPException(status_code=500, internal_error=e) response = data.GetEventsResponse(stream_boundary=stream_boundary, events=events) return response @@ -182,12 +179,10 @@ async def latest_events_handler( ) except ReceivingEventsException as e: logger.error("Error receiving events from provider") - reporter.error_report(e) - raise HTTPException(status_code=500) + raise MoonstreamHTTPException(status_code=500, internal_error=e) except Exception as e: logger.error("Unable to get latest events") - reporter.error_report(e) - raise HTTPException(status_code=500) + raise MoonstreamHTTPException(status_code=500, internal_error=e) return events @@ -239,12 +234,10 @@ async def next_event_handler( ) except ReceivingEventsException as e: logger.error("Error receiving events from provider") - reporter.error_report(e) - raise HTTPException(status_code=500) + raise MoonstreamHTTPException(status_code=500, internal_error=e) except Exception as e: logger.error("Unable to get next events") - reporter.error_report(e) - raise HTTPException(status_code=500) + raise MoonstreamHTTPException(status_code=500, internal_error=e) return event @@ -296,11 +289,9 @@ async def previous_event_handler( ) except ReceivingEventsException as e: logger.error("Error receiving events from provider") - reporter.error_report(e) - raise HTTPException(status_code=500) + raise MoonstreamHTTPException(status_code=500, internal_error=e) except Exception as e: logger.error("Unable to get previous events") - reporter.error_report(e) - raise HTTPException(status_code=500) + raise MoonstreamHTTPException(status_code=500, internal_error=e) return event diff --git a/backend/moonstream/routes/subscriptions.py b/backend/moonstream/routes/subscriptions.py index f05eea0e..f86f4686 100644 --- a/backend/moonstream/routes/subscriptions.py +++ b/backend/moonstream/routes/subscriptions.py @@ -6,12 +6,12 @@ from typing import Dict, List, Optional from bugout.data import BugoutResource, BugoutResources from bugout.exceptions import BugoutResponseException -from fastapi import FastAPI, HTTPException, Request, Form +from fastapi import FastAPI, Request, Form from fastapi.middleware.cors import CORSMiddleware from ..admin import subscription_types from .. import data -from ..middleware import BroodAuthMiddleware +from ..middleware import BroodAuthMiddleware, MoonstreamHTTPException from ..reporter import reporter from ..settings import ( DOCS_TARGET_PATH, @@ -78,7 +78,7 @@ async def add_subscription_handler( ] if subscription_type_id not in available_subscription_type_ids: - raise HTTPException( + raise MoonstreamHTTPException( status_code=404, detail=f"Invalid subscription type: {subscription_type_id}.", ) @@ -100,10 +100,11 @@ async def add_subscription_handler( application_id=MOONSTREAM_APPLICATION_ID, resource_data=resource_data, ) + except BugoutResponseException as e: + raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) except Exception as e: logger.error(f"Error creating subscription resource: {str(e)}") - reporter.error_report(e) - raise HTTPException(status_code=500) + raise MoonstreamHTTPException(status_code=500, internal_error=e) return data.SubscriptionResourceData( id=str(resource.id), @@ -128,11 +129,10 @@ async def delete_subscription_handler(request: Request, subscription_id: str): try: deleted_resource = bc.delete_resource(token=token, resource_id=subscription_id) except BugoutResponseException as e: - raise HTTPException(status_code=e.status_code, detail=e.detail) + raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) except Exception as e: logger.error(f"Error deleting subscription: {str(e)}") - reporter.error_report(e) - raise HTTPException(status_code=500) + raise MoonstreamHTTPException(status_code=500, internal_error=e) return data.SubscriptionResourceData( id=str(deleted_resource.id), @@ -156,12 +156,14 @@ async def get_subscriptions_handler(request: Request) -> data.SubscriptionsListR } try: resources: BugoutResources = bc.list_resources(token=token, params=params) + except BugoutResponseException as e: + raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) except Exception as e: logger.error( f"Error listing subscriptions for user ({request.user.id}) with token ({request.state.token}), error: {str(e)}" ) reporter.error_report(e) - raise HTTPException(status_code=500) + raise MoonstreamHTTPException(status_code=500, internal_error=e) return data.SubscriptionsListResponse( subscriptions=[ @@ -211,11 +213,10 @@ async def update_subscriptions_handler( ).dict(), ) except BugoutResponseException as e: - raise HTTPException(status_code=e.status_code, detail=e.detail) + raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) except Exception as e: logger.error(f"Error getting user subscriptions: {str(e)}") - reporter.error_report(e) - raise HTTPException(status_code=500) + raise MoonstreamHTTPException(status_code=500, internal_error=e) return data.SubscriptionResourceData( id=str(resource.id), @@ -241,9 +242,10 @@ async def list_subscription_types() -> data.SubscriptionTypesListResponse: data.SubscriptionTypeResourceData.validate(resource.resource_data) for resource in response.resources ] + except BugoutResponseException as e: + raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) except Exception as e: logger.error(f"Error reading subscription types from Brood API: {str(e)}") - reporter.error_report(e) - raise HTTPException(status_code=500) + raise MoonstreamHTTPException(status_code=500, internal_error=e) return data.SubscriptionTypesListResponse(subscription_types=results) diff --git a/backend/moonstream/routes/users.py b/backend/moonstream/routes/users.py index 60090740..4407a67e 100644 --- a/backend/moonstream/routes/users.py +++ b/backend/moonstream/routes/users.py @@ -7,16 +7,10 @@ import uuid from bugout.data import BugoutToken, BugoutUser from bugout.exceptions import BugoutResponseException -from fastapi import ( - FastAPI, - Form, - HTTPException, - Request, -) +from fastapi import FastAPI, Form, Request from fastapi.middleware.cors import CORSMiddleware -from ..middleware import BroodAuthMiddleware -from ..reporter import reporter +from ..middleware import BroodAuthMiddleware, MoonstreamHTTPException from ..settings import ( MOONSTREAM_APPLICATION_ID, DOCS_TARGET_PATH, @@ -76,10 +70,9 @@ async def create_user_handler( application_id=MOONSTREAM_APPLICATION_ID, ) except BugoutResponseException as e: - raise HTTPException(status_code=e.status_code, detail=e.detail) + raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) except Exception as e: - reporter.error_report(e) - raise HTTPException(status_code=500) + raise MoonstreamHTTPException(status_code=500, internal_error=e) return user @@ -94,10 +87,9 @@ async def restore_password_handler(email: str = Form(...)) -> Dict[str, Any]: try: response = bc.restore_password(email=email) except BugoutResponseException as e: - raise HTTPException(status_code=e.status_code, detail=e.detail) + raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) except Exception as e: - reporter.error_report(e) - raise HTTPException(status_code=500) + raise MoonstreamHTTPException(status_code=500, internal_error=e) return response @@ -108,10 +100,9 @@ async def reset_password_handler( try: response = bc.reset_password(reset_id=reset_id, new_password=new_password) except BugoutResponseException as e: - raise HTTPException(status_code=e.status_code, detail=e.detail) + raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) except Exception as e: - reporter.error_report(e) - raise HTTPException(status_code=500) + raise MoonstreamHTTPException(status_code=500, internal_error=e) return response @@ -125,10 +116,9 @@ async def change_password_handler( token=token, current_password=current_password, new_password=new_password ) except BugoutResponseException as e: - raise HTTPException(status_code=e.status_code, detail=e.detail) + raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) except Exception as e: - reporter.error_report(e) - raise HTTPException(status_code=500) + raise MoonstreamHTTPException(status_code=500, internal_error=e) return user @@ -141,10 +131,9 @@ async def delete_user_handler( try: user = bc.delete_user(token=token, user_id=user.id, password=password) except BugoutResponseException as e: - raise HTTPException(status_code=e.status_code, detail=e.detail) + raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) except Exception as e: - reporter.error_report(e) - raise HTTPException(status_code=500) + raise MoonstreamHTTPException(status_code=500, internal_error=e) return user @@ -159,12 +148,11 @@ async def login_handler( application_id=MOONSTREAM_APPLICATION_ID, ) except BugoutResponseException as e: - raise HTTPException( + raise MoonstreamHTTPException( status_code=e.status_code, detail=f"Error from Brood API: {e.detail}" ) except Exception as e: - reporter.error_report(e) - raise HTTPException(status_code=500) + raise MoonstreamHTTPException(status_code=500, internal_error=e) return token @@ -174,8 +162,7 @@ async def logout_handler(request: Request) -> uuid.UUID: try: token_id: uuid.UUID = bc.revoke_token(token=token) except BugoutResponseException as e: - raise HTTPException(status_code=e.status_code, detail=e.detail) + raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) except Exception as e: - reporter.error_report(e) - raise HTTPException(status_code=500) + raise MoonstreamHTTPException(status_code=500, internal_error=e) return token_id diff --git a/backend/requirements.txt b/backend/requirements.txt index 6ebf10e0..4cc7c386 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -29,5 +29,6 @@ toml==0.10.2 tomli==1.0.4 types-python-dateutil==0.1.6 typing-extensions==3.10.0.0 +types-requests==2.25.6 urllib3==1.26.6 uvicorn==0.14.0 diff --git a/backend/setup.py b/backend/setup.py index 7cc23eee..06a2a807 100644 --- a/backend/setup.py +++ b/backend/setup.py @@ -18,6 +18,7 @@ setup( "python-dateutil", "uvicorn", "types-python-dateutil", + "types-requests", ], extras_require={ "dev": ["black", "mypy"], From e1fe56a49d28e53c9302ab0bfb0972b6733cd677 Mon Sep 17 00:00:00 2001 From: Andrey Dolgolev Date: Thu, 2 Sep 2021 14:46:16 +0300 Subject: [PATCH 25/32] Add fixes. --- .../ecb7817db377_add_opensea_state_table_and_add_index_.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/db/alembic/versions/ecb7817db377_add_opensea_state_table_and_add_index_.py b/db/alembic/versions/ecb7817db377_add_opensea_state_table_and_add_index_.py index 47f3572a..d6176845 100644 --- a/db/alembic/versions/ecb7817db377_add_opensea_state_table_and_add_index_.py +++ b/db/alembic/versions/ecb7817db377_add_opensea_state_table_and_add_index_.py @@ -33,17 +33,14 @@ def upgrade(): sa.UniqueConstraint("id", name=op.f("uq_opensea_crawler_state_id")), ) op.drop_constraint("uq_ethereum_labels_label", "ethereum_labels", type_="unique") - op.create_index( - op.f("idx_label_name"), "ethereum_labels", [sa.text("(label_data->'name')")] + op.execute( + f"CREATE INDEX idx_ethereum_labels_opensea_nft_name ON ethereum_labels((label_data->>'name')) where label='opensea_nft';" ) # ### end Alembic commands ### def downgrade(): # ### commands auto generated by Alembic - please adjust! ### - op.create_unique_constraint( - "uq_ethereum_labels_label", "ethereum_labels", ["label", "address_id"] - ) op.drop_table("opensea_crawler_state") op.drop_index("idx_label_name") # ### end Alembic commands ### From 5290e4bf344746ea45355d88f8f9044e011433ca Mon Sep 17 00:00:00 2001 From: Andrey Dolgolev Date: Thu, 2 Sep 2021 17:13:51 +0300 Subject: [PATCH 26/32] Fix drop condition name. --- .../ecb7817db377_add_opensea_state_table_and_add_index_.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/alembic/versions/ecb7817db377_add_opensea_state_table_and_add_index_.py b/db/alembic/versions/ecb7817db377_add_opensea_state_table_and_add_index_.py index d6176845..13378881 100644 --- a/db/alembic/versions/ecb7817db377_add_opensea_state_table_and_add_index_.py +++ b/db/alembic/versions/ecb7817db377_add_opensea_state_table_and_add_index_.py @@ -42,5 +42,5 @@ def upgrade(): def downgrade(): # ### commands auto generated by Alembic - please adjust! ### op.drop_table("opensea_crawler_state") - op.drop_index("idx_label_name") + op.drop_index("idx_ethereum_labels_opensea_nft_name") # ### end Alembic commands ### From 6836cdb723d2ed37983292dfceb8c88bc92d047f Mon Sep 17 00:00:00 2001 From: Andrey Dolgolev Date: Thu, 2 Sep 2021 17:15:30 +0300 Subject: [PATCH 27/32] Remove import. --- db/moonstreamdb/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/moonstreamdb/models.py b/db/moonstreamdb/models.py index 391724bc..55d5307a 100644 --- a/db/moonstreamdb/models.py +++ b/db/moonstreamdb/models.py @@ -13,7 +13,7 @@ from sqlalchemy import ( VARCHAR, ) from sqlalchemy.dialects.postgresql import JSONB, UUID -from sqlalchemy.sql import expression, text +from sqlalchemy.sql import expression from sqlalchemy.ext.compiler import compiles """ From dd9954094ccaee2dc6c5d51121f84f37e097fff5 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Thu, 2 Sep 2021 15:33:20 +0000 Subject: [PATCH 28/32] Removed Response child --- backend/moonstream/middleware.py | 25 ++----------------------- 1 file changed, 2 insertions(+), 23 deletions(-) diff --git a/backend/moonstream/middleware.py b/backend/moonstream/middleware.py index 7bb94a84..5e089aa6 100644 --- a/backend/moonstream/middleware.py +++ b/backend/moonstream/middleware.py @@ -13,26 +13,6 @@ from .settings import MOONSTREAM_APPLICATION_ID, bugout_client as bc logger = logging.getLogger(__name__) -class MoonstreamResponse(Response): - """ - Extended Response to handle 500 Internal server errors - and send crash reports. - """ - - def __init__( - self, - content: Any = None, - status_code: int = 200, - headers: dict = None, - media_type: str = None, - background: BackgroundTask = None, - internal_error: Exception = None, - ): - super().__init__(content, status_code, headers, media_type, background) - if internal_error is not None: - reporter.error_report(internal_error) - - class MoonstreamHTTPException(HTTPException): """ Extended HTTPException to handle 500 Internal server errors @@ -101,9 +81,8 @@ class BroodAuthMiddleware(BaseHTTPMiddleware): return Response(status_code=e.status_code, content=e.detail) except Exception as e: logger.error(f"Error processing Brood response: {str(e)}") - return MoonstreamResponse( - status_code=500, content="Internal server error", internal_error=e - ) + reporter.error_report(e) + return Response(status_code=500, content="Internal server error") request.state.user = user request.state.token = user_token From 3960133c9e9a643900376eadfe87bf7b723c24f2 Mon Sep 17 00:00:00 2001 From: Andrey Dolgolev Date: Thu, 2 Sep 2021 19:01:57 +0300 Subject: [PATCH 29/32] Change updated_at to crawled_at. --- .../ecb7817db377_add_opensea_state_table_and_add_index_.py | 7 +++++-- db/moonstreamdb/models.py | 7 +++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/db/alembic/versions/ecb7817db377_add_opensea_state_table_and_add_index_.py b/db/alembic/versions/ecb7817db377_add_opensea_state_table_and_add_index_.py index 13378881..98a07eb0 100644 --- a/db/alembic/versions/ecb7817db377_add_opensea_state_table_and_add_index_.py +++ b/db/alembic/versions/ecb7817db377_add_opensea_state_table_and_add_index_.py @@ -23,7 +23,7 @@ def upgrade(): sa.Column("id", sa.Integer(), nullable=False), sa.Column("query", sa.Text(), nullable=False), sa.Column( - "updated_at", + "crawled_at", sa.DateTime(timezone=True), server_default=sa.text("TIMEZONE('utc', statement_timestamp())"), nullable=False, @@ -32,7 +32,10 @@ def upgrade(): sa.PrimaryKeyConstraint("id", name=op.f("pk_opensea_crawler_state")), sa.UniqueConstraint("id", name=op.f("uq_opensea_crawler_state_id")), ) - op.drop_constraint("uq_ethereum_labels_label", "ethereum_labels", type_="unique") + op.execute( + "ALTER TABLE ethereum_labels DROP CONSTRAINT IF EXISTS uq_ethereum_labels_label" + ) + op.execute( f"CREATE INDEX idx_ethereum_labels_opensea_nft_name ON ethereum_labels((label_data->>'name')) where label='opensea_nft';" ) diff --git a/db/moonstreamdb/models.py b/db/moonstreamdb/models.py index 55d5307a..7d7798a4 100644 --- a/db/moonstreamdb/models.py +++ b/db/moonstreamdb/models.py @@ -221,8 +221,11 @@ class OpenSeaCrawlingState(Base): # type: ignore id = Column(Integer, primary_key=True, unique=True, nullable=False) query = Column(Text, nullable=False) - updated_at = Column( - DateTime(timezone=True), server_default=utcnow(), nullable=False + crawled_at = Column( + DateTime(timezone=True), + server_default=utcnow(), + onupdate=utcnow(), + nullable=False, ) total_count = Column(Integer, nullable=False) From c12d4f9a36fefc4ef50418db37f0313357b80271 Mon Sep 17 00:00:00 2001 From: Neeraj Kashyap Date: Fri, 3 Sep 2021 11:55:20 -0700 Subject: [PATCH 30/32] Updated Discord link in footer Previous link was invalid! --- frontend/src/components/Footer.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/components/Footer.js b/frontend/src/components/Footer.js index d212befe..2b95a9f6 100644 --- a/frontend/src/components/Footer.js +++ b/frontend/src/components/Footer.js @@ -6,7 +6,7 @@ import RouterLink from "next/link"; const ICONS = [ { social: "discord", - link: "https://discord.gg/FetK5BxD", + link: "https://discord.gg/K56VNUQGvA", }, { social: "twit", link: "https://twitter.com/moonstreamto" }, From 741383b50373de3b6385ba91ac58fcdd0aad279b Mon Sep 17 00:00:00 2001 From: Neeraj Kashyap Date: Mon, 6 Sep 2021 09:46:50 -0700 Subject: [PATCH 31/32] Fix Etherscan crawler so that we don't manually specify autogenerated id --- crawlers/mooncrawl/mooncrawl/etherscan.py | 46 +++++++++++------------ 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/crawlers/mooncrawl/mooncrawl/etherscan.py b/crawlers/mooncrawl/mooncrawl/etherscan.py index 6a1ead76..3f3bc1a2 100644 --- a/crawlers/mooncrawl/mooncrawl/etherscan.py +++ b/crawlers/mooncrawl/mooncrawl/etherscan.py @@ -66,21 +66,16 @@ def get_address_id(db_session: Session, contract_address: str) -> int: if id is not None: return id[0] - latest_address_id = ( - db_session.query(EthereumAddress.id).order_by(text("id desc")).limit(1).one() - )[0] - - id = latest_address_id + 1 smart_contract = EthereumAddress( - id=id, address=contract_address, ) try: db_session.add(smart_contract) db_session.commit() - except: + return smart_contract.id + except Exception as e: db_session.rollback() - return id + raise e def crawl_step(db_session: Session, contract: VerifiedSmartContract, crawl_url: str): @@ -112,22 +107,27 @@ def crawl_step(db_session: Session, contract: VerifiedSmartContract, crawl_url: object_key = f"/etherscan/v1/{contract.address}.json" push_to_bucket(contract_info, object_key) - eth_address_id = get_address_id(db_session, contract.address) - - eth_label = EthereumLabel( - label=ETHERSCAN_SMARTCONTRACTS_LABEL_NAME, - address_id=eth_address_id, - label_data={ - "object_uri": f"s3://{bucket}/{object_key}", - "name": contract.name, - "tx_hash": contract.tx_hash, - }, - ) try: - db_session.add(eth_label) - db_session.commit() - except: - db_session.rollback() + eth_address_id = get_address_id(db_session, contract.address) + eth_label = EthereumLabel( + label=ETHERSCAN_SMARTCONTRACTS_LABEL_NAME, + address_id=eth_address_id, + label_data={ + "object_uri": f"s3://{bucket}/{object_key}", + "name": contract.name, + "tx_hash": contract.tx_hash, + }, + ) + try: + db_session.add(eth_label) + db_session.commit() + except Exception as e: + db_session.rollback() + raise e + except Exception as e: + logger.error( + f"Failed to add addresss label ${contract.address} to database\n{str(e)}" + ) def crawl( From 0eb757d492f8532816df75ce13aba4c9e341713b Mon Sep 17 00:00:00 2001 From: Neeraj Kashyap Date: Mon, 6 Sep 2021 09:49:38 -0700 Subject: [PATCH 32/32] Fixed imports, added logging --- crawlers/mooncrawl/mooncrawl/etherscan.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/crawlers/mooncrawl/mooncrawl/etherscan.py b/crawlers/mooncrawl/mooncrawl/etherscan.py index 3f3bc1a2..f3f61a83 100644 --- a/crawlers/mooncrawl/mooncrawl/etherscan.py +++ b/crawlers/mooncrawl/mooncrawl/etherscan.py @@ -1,24 +1,26 @@ import argparse +import codecs +import csv +from dataclasses import dataclass +from datetime import datetime +import json +import logging +import os import sys import time -from datetime import datetime from typing import Any, List, Optional, Dict -from dataclasses import dataclass -import csv -import codecs -import json -import os import boto3 # type: ignore from moonstreamdb.db import yield_db_session_ctx from moonstreamdb.models import EthereumAddress, EthereumLabel import requests from sqlalchemy.orm import Session -from sqlalchemy.sql.expression import text from .version import MOONCRAWL_VERSION from .settings import MOONSTREAM_ETHERSCAN_TOKEN +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) if MOONSTREAM_ETHERSCAN_TOKEN is None: raise Exception("MOONSTREAM_ETHERSCAN_TOKEN environment variable must be set")