Merge branch 'main' into extend-subscriptions

backend/moonstream/actions.py

@@ -1,7 +1,9 @@
 import json
 import logging
-from typing import Optional
+
+from typing import Optional, Dict, Any
 from enum import Enum
+import uuid
 
 import boto3  # type: ignore
 from moonstreamdb.models import (
@@ -14,6 +16,12 @@ from sqlalchemy.orm import Session
 from . import data
 from .reporter import reporter
 from .settings import ETHERSCAN_SMARTCONTRACTS_BUCKET
+from bugout.data import BugoutResource
+from .settings import (
+    MOONSTREAM_APPLICATION_ID,
+    bugout_client as bc,
+    BUGOUT_REQUEST_TIMEOUT_SECONDS,
+)
 
 logger = logging.getLogger(__name__)
 ETHERSCAN_SMARTCONTRACT_LABEL_NAME = "etherscan_smartcontract"
@@ -144,3 +152,21 @@ def get_address_labels(
         )
 
     return addresses_response
+
+
+def create_onboarding_resource(
+    token: uuid.UUID,
+    resource_data: Dict[str, Any] = {
+        "type": data.USER_ONBOARDING_STATE,
+        "steps": {"welcome": 0, "subscriptions": 0, "stream": 0,},
+        "is_complete": False,
+    },
+) -> BugoutResource:
+
+    resource = bc.create_resource(
+        token=token,
+        application_id=MOONSTREAM_APPLICATION_ID,
+        resource_data=resource_data,
+        timeout=BUGOUT_REQUEST_TIMEOUT_SECONDS,
+    )
+    return resource

backend/moonstream/data.py

@@ -5,6 +5,8 @@ from typing import List, Optional, Dict, Any
 
 from pydantic import BaseModel, Field
 
+USER_ONBOARDING_STATE = "onboarding_state"
+
 
 class SubscriptionTypeResourceData(BaseModel):
     id: str
@@ -196,3 +198,8 @@ class AddressLabelsResponse(BaseModel):
 
 class AddressListLabelsResponse(BaseModel):
     addresses: List[AddressLabelsResponse] = Field(default_factory=list)
+
+
+class OnboardingState(BaseModel):
+    is_complete: bool
+    steps: Dict[str, int]

backend/moonstream/routes/users.py

@@ -5,20 +5,30 @@ import logging
 from typing import Any, Dict
 import uuid
 
-from bugout.data import BugoutToken, BugoutUser
+from bugout.data import BugoutToken, BugoutUser, BugoutResource
 from bugout.exceptions import BugoutResponseException
-from fastapi import FastAPI, Form, Request
+
+from fastapi import (
+    Body,
+    FastAPI,
+    Form,
+    Request,
+)
 from fastapi.middleware.cors import CORSMiddleware
 
+from .. import data
 from ..middleware import BroodAuthMiddleware, MoonstreamHTTPException
+
 from ..settings import (
     MOONSTREAM_APPLICATION_ID,
     DOCS_TARGET_PATH,
     ORIGINS,
     DOCS_PATHS,
     bugout_client as bc,
+    BUGOUT_REQUEST_TIMEOUT_SECONDS,
 )
 from ..version import MOONSTREAM_VERSION
+from ..actions import create_onboarding_resource
 
 logger = logging.getLogger(__name__)
 
@@ -166,3 +176,121 @@ async def logout_handler(request: Request) -> uuid.UUID:
     except Exception as e:
         raise MoonstreamHTTPException(status_code=500, internal_error=e)
     return token_id
+
+
+@app.post("/onboarding", tags=["users"], response_model=data.OnboardingState)
+async def set_onboarding_state(
+    request: Request, onboarding_data: data.OnboardingState = Body(...),
+) -> data.OnboardingState:
+
+    token = request.state.token
+    try:
+        response = bc.list_resources(
+            token=token,
+            params={"type": data.USER_ONBOARDING_STATE},
+            timeout=BUGOUT_REQUEST_TIMEOUT_SECONDS,
+        )
+        resource_data = {"type": data.USER_ONBOARDING_STATE, **onboarding_data.dict()}
+        if response.resources:
+            resource = bc.update_resource(
+                token=token,
+                resource_id=str(response.resources[0].id),
+                resource_data={"update": resource_data, "drop_keys": []},
+            )
+        else:
+            resource = create_onboarding_resource(
+                token=token, resource_data=resource_data
+            )
+
+    except BugoutResponseException as e:
+        raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail)
+    except Exception as e:
+        raise MoonstreamHTTPException(status_code=500)
+
+    if (
+        resource.resource_data.get("is_complete") is None
+        or resource.resource_data.get("steps") is None
+    ):
+        logger.error(
+            f"Resources did not return correct onboarding object. Resource id:{resource.id}"
+        )
+        raise MoonstreamHTTPException(status_code=500)
+
+    result = data.OnboardingState(
+        is_complete=resource.resource_data.get("is_complete", False),
+        steps=resource.resource_data.get("steps", {}),
+    )
+    return result
+
+
+@app.get("/onboarding", tags=["users"], response_model=data.OnboardingState)
+async def get_onboarding_state(request: Request) -> data.OnboardingState:
+    token = request.state.token
+    try:
+        response = bc.list_resources(
+            token=token,
+            params={"type": data.USER_ONBOARDING_STATE},
+            timeout=BUGOUT_REQUEST_TIMEOUT_SECONDS,
+        )
+
+        if response.resources:
+            resource = response.resources[0]
+        else:
+            resource = create_onboarding_resource(token=token)
+    except BugoutResponseException as e:
+        raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail)
+    except Exception as e:
+
+        raise MoonstreamHTTPException(status_code=500)
+
+    if (
+        resource.resource_data.get("is_complete") is None
+        or resource.resource_data.get("steps") is None
+    ):
+        logger.error(
+            f"Resources did not return correct onboarding object. Resource id:{resource.id}"
+        )
+        raise MoonstreamHTTPException(status_code=500)
+    result = data.OnboardingState(
+        is_complete=resource.resource_data.get("is_complete", False),
+        steps=resource.resource_data.get("steps", {}),
+    )
+    return result
+
+
+@app.delete("/onboarding", tags=["users"], response_model=data.OnboardingState)
+async def delete_onboarding_state(request: Request) -> data.OnboardingState:
+    token = request.state.token
+    try:
+        response = bc.list_resources(
+            token=token,
+            params={"type": data.USER_ONBOARDING_STATE},
+            timeout=BUGOUT_REQUEST_TIMEOUT_SECONDS,
+        )
+        if not response.resources:
+            raise MoonstreamHTTPException(status_code=404, detail="not found")
+        if response.resources:
+            resource: BugoutResource = bc.delete_resource(
+                token=token,
+                resource_id=response.resources[0].id,
+                timeout=BUGOUT_REQUEST_TIMEOUT_SECONDS,
+            )
+
+    except BugoutResponseException as e:
+        raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail)
+    except Exception as e:
+        raise MoonstreamHTTPException(status_code=500)
+
+    if (
+        resource.resource_data.get("is_complete") is None
+        or resource.resource_data.get("steps") is None
+    ):
+        logger.error(
+            f"Resources did not return correct onboarding object. Resource id:{resource.id}"
+        )
+        raise MoonstreamHTTPException(status_code=500)
+    result = data.OnboardingState(
+        is_complete=resource.resource_data.get("is_complete", False),
+        steps=resource.resource_data.get("steps", {}),
+    )
+    return result

crawlers/mooncrawl/mooncrawl/etherscan.py

@@ -1,24 +1,26 @@
 import argparse
+import codecs
+import csv
+from dataclasses import dataclass
+from datetime import datetime
+import json
+import logging
+import os
 import sys
 import time
-from datetime import datetime
 from typing import Any, List, Optional, Dict
-from dataclasses import dataclass
-import csv
-import codecs
-import json
-import os
 
 import boto3  # type: ignore
 from moonstreamdb.db import yield_db_session_ctx
 from moonstreamdb.models import EthereumAddress, EthereumLabel
 import requests
 from sqlalchemy.orm import Session
-from sqlalchemy.sql.expression import text
 
 from .version import MOONCRAWL_VERSION
 from .settings import MOONSTREAM_ETHERSCAN_TOKEN
 
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.INFO)
 if MOONSTREAM_ETHERSCAN_TOKEN is None:
     raise Exception("MOONSTREAM_ETHERSCAN_TOKEN environment variable must be set")
 
@@ -66,21 +68,16 @@ def get_address_id(db_session: Session, contract_address: str) -> int:
     if id is not None:
         return id[0]
 
-    latest_address_id = (
-        db_session.query(EthereumAddress.id).order_by(text("id desc")).limit(1).one()
-    )[0]
-
-    id = latest_address_id + 1
     smart_contract = EthereumAddress(
-        id=id,
         address=contract_address,
     )
     try:
         db_session.add(smart_contract)
         db_session.commit()
-    except:
+        return smart_contract.id
+    except Exception as e:
         db_session.rollback()
-    return id
+        raise e
 
 
 def crawl_step(db_session: Session, contract: VerifiedSmartContract, crawl_url: str):
@@ -112,8 +109,8 @@ def crawl_step(db_session: Session, contract: VerifiedSmartContract, crawl_url:
     object_key = f"/etherscan/v1/{contract.address}.json"
     push_to_bucket(contract_info, object_key)
 
+    try:
         eth_address_id = get_address_id(db_session, contract.address)
-
         eth_label = EthereumLabel(
             label=ETHERSCAN_SMARTCONTRACTS_LABEL_NAME,
             address_id=eth_address_id,
@@ -126,8 +123,13 @@ def crawl_step(db_session: Session, contract: VerifiedSmartContract, crawl_url:
         try:
             db_session.add(eth_label)
             db_session.commit()
-    except:
+        except Exception as e:
             db_session.rollback()
+            raise e
+    except Exception as e:
+        logger.error(
+            f"Failed to add addresss label ${contract.address} to database\n{str(e)}"
+        )
 
 
 def crawl(

frontend/pages/welcome.js

@@ -70,7 +70,7 @@ const Welcome = () => {
       ui.setOnboardingStep(ui.onboardingStep + 1);
       scrollRef?.current?.scrollIntoView();
     } else {
-      ui.setisOnboardingComplete(true);
+      ui.setOnboardingComplete(true);
       router.push("/stream");
     }
   };

frontend/src/components/AppNavbar.js

@@ -115,12 +115,6 @@ const AppNavbar = () => {
                   </RouteButton>
                 ))}
                 {USER_NAV_PATHES.map((item, idx) => {
-                  console.log(
-                    "item.path:",
-                    item.path,
-                    "pathname:",
-                    router.nextRouter.pathname
-                  );
                   return (
                     <RouteButton
                       key={`${idx}-${item.title}-navlink`}

frontend/src/core/providers/UIProvider/index.js

@@ -1,10 +1,26 @@
-import React, { useState, useContext, useEffect } from "react";
+import React, { useState, useContext, useEffect, useCallback } from "react";
 import { useBreakpointValue } from "@chakra-ui/react";
 import { useStorage, useQuery, useRouter } from "../../hooks";
 import UIContext from "./context";
 import UserContext from "../UserProvider/context";
 import ModalContext from "../ModalProvider/context";
 import { v4 as uuid4 } from "uuid";
+import { PreferencesService } from "../../services";
+
+const onboardingSteps = [
+  {
+    step: "welcome",
+    description: "Basics of how Moonstream works",
+  },
+  {
+    step: "subscriptions",
+    description: "Learn how to subscribe to blockchain events",
+  },
+  {
+    step: "stream",
+    description: "Learn how to use your Moonstream to analyze blah blah blah",
+  },
+];
 
 const UIProvider = ({ children }) => {
   const router = useRouter();
@@ -53,14 +69,6 @@ const UIProvider = ({ children }) => {
     }
   }, [isAppView, user, isLoggingOut]);
 
-  useEffect(() => {
-    if (isInit && router.nextRouter.isReady) {
-      setAppReady(true);
-    } else {
-      setAppReady(false);
-    }
-  }, [isInit, router]);
-
   useEffect(() => {
     if (user && user.username) {
       setLoggedIn(true);
@@ -155,90 +163,109 @@ const UIProvider = ({ children }) => {
 
   // *********** Onboarding state **********************
 
-  const onboardingSteps = [
+  const [onboardingState, setOnboardingState] = useState(false);
-    {
+  const [onboardingStep, setOnboardingStep] = useState();
-      step: "welcome",
+  const [onboardingStateInit, setOnboardingStateInit] = useState(false);
-      description: "Basics of how Moonstream works",
-    },
-    {
-      step: "subscriptions",
-      description: "Learn how to subscribe to blockchain events",
-    },
-    {
-      step: "stream",
-      description: "Learn how to use your Moonstream to analyze blah blah blah",
-    },
-  ];
 
-  const [onboardingState, setOnboardingState] = useStorage(
+  const setOnboardingComplete = useCallback(
-    window.localStorage,
+    (newState) => {
-    "onboardingState",
+      setOnboardingState({ ...onboardingState, is_complete: newState });
-    {
+    },
-      welcome: 0,
+    [onboardingState]
-      subscriptions: 0,
-      stream: 0,
-    }
-  );
-
-  const [onboardingStep, setOnboardingStep] = useState(() => {
-    //First find onboarding step that was viewed once (resume where left)
-    // If none - find step that was never viewed
-    // if none - set onboarding to zero
-    let step = onboardingSteps.findIndex(
-      (event) => onboardingState[event.step] === 1
-    );
-    step =
-      step === -1
-        ? onboardingSteps.findIndex(
-            (event) => onboardingState[event.step] === 0
-          )
-        : step;
-    if (step === -1 && isOnboardingComplete) return 0;
-    else if (step === -1 && !isOnboardingComplete) return 0;
-    else return step;
-  });
-
-  const [isOnboardingComplete, setisOnboardingComplete] = useStorage(
-    window.localStorage,
-    "isOnboardingComplete",
-    isLoggedIn ? true : false
   );
 
   useEffect(() => {
-    if (isLoggedIn && !isOnboardingComplete) {
+    //If onboarding state not exists - fetch it from backend
+    //If it exists but init is not set - set init true
+    //If it exists and is init -> post update to backend
+    if (!onboardingState && user) {
+      const currentOnboardingState = async () =>
+        PreferencesService.getOnboardingState().then((response) => {
+          return response.data;
+        });
+
+      currentOnboardingState().then((response) => {
+        setOnboardingState(response);
+      });
+    } else if (user && onboardingState && !onboardingStateInit) {
+      setOnboardingStateInit(true);
+    } else if (onboardingStateInit) {
+      PreferencesService.setOnboardingState(onboardingState);
+    }
+    // eslint-disable-next-line
+  }, [onboardingState, user]);
+
+  useEffect(() => {
+    //This will set step after state is fetched from backend
+    if (!Number.isInteger(onboardingStep) && onboardingState) {
+      const step = onboardingSteps.findIndex(
+        (event) => onboardingState[event.step] === 0
+      );
+      if (step === -1 && onboardingState["is_complete"])
+        setOnboardingStep(onboardingSteps.length - 1);
+      else if (step === -1 && !onboardingState["is_complete"])
+        return setOnboardingStep(0);
+      else setOnboardingStep(step);
+    }
+  }, [onboardingState, onboardingStep]);
+
+  useEffect(() => {
+    //redirect to welcome page if yet not completed onboarding
+    if (
+      isLoggedIn &&
+      isAppReady &&
+      onboardingState &&
+      !onboardingState?.is_complete
+    ) {
       router.replace("/welcome");
     }
     // eslint-disable-next-line
-  }, [isLoggedIn, isOnboardingComplete]);
+  }, [isLoggedIn, onboardingState?.is_complete, isAppReady]);
 
   useEffect(() => {
+    //This will set up onboarding complete once user finishes each view at least once
+    if (onboardingState?.steps && user && isAppReady) {
       if (
         onboardingSteps.findIndex(
-        (event) => onboardingState[event.step] === 0
+          (event) => onboardingState.steps[event.step] === 0
         ) === -1
       ) {
-      setisOnboardingComplete(true);
+        !onboardingState.is_complete && setOnboardingComplete(true);
       }
-    //eslint-disable-next-line
+    }
-  }, [onboardingState]);
+  }, [onboardingState, user, isAppReady, setOnboardingComplete]);
 
   useEffect(() => {
-    if (router.nextRouter.pathname === "/welcome") {
+    //This will update onboardingState when step changes
-      const newOnboardingState = {
+    if (
+      router.nextRouter.pathname === "/welcome" &&
+      isAppReady &&
+      user &&
+      Number.isInteger(onboardingStep) &&
+      onboardingState
+    ) {
+      setOnboardingState({
         ...onboardingState,
+        steps: {
+          ...onboardingState.steps,
           [`${onboardingSteps[onboardingStep].step}`]:
-          onboardingState[onboardingSteps[onboardingStep].step] + 1,
+            onboardingState.steps[onboardingSteps[onboardingStep].step] + 1,
-      };
+        },
-
+      });
-      setOnboardingState(newOnboardingState);
     }
     // eslint-disable-next-line
-  }, [onboardingStep, router.nextRouter.pathname]);
+  }, [onboardingStep, router.nextRouter.pathname, user, isAppReady]);
-
-  // const ONBOARDING_STEP_NUM = steps.length;
 
   // ********************************************************
 
+  useEffect(() => {
+    if (isInit && router.nextRouter.isReady && onboardingState) {
+      setAppReady(true);
+    } else {
+      setAppReady(false);
+    }
+  }, [isInit, router, onboardingState]);
+
   return (
     <UIContext.Provider
       value={{
@@ -268,8 +295,7 @@ const UIProvider = ({ children }) => {
         isEntryDetailView,
         onboardingStep,
         setOnboardingStep,
-        isOnboardingComplete,
+        setOnboardingComplete,
-        setisOnboardingComplete,
         onboardingSteps,
         setOnboardingState,
         onboardingState,

frontend/src/core/services/preferences.service.js

@@ -1,23 +1,18 @@
 import { http } from "../utils";
 
-const API = process.env.NEXT_PUBLIC_SIMIOTICS_JOURNALS_URL;
+const API_URL = process.env.NEXT_PUBLIC_MOONSTREAM_API_URL;
-const PREFERENCES_API = `${API}/preferences`;
+export const PREFERENCES_URL = `${API_URL}/users`;
 
-export const getDefaultJournal = () =>
+export const getOnboardingState = () =>
   http({
     method: "GET",
-    url: `${PREFERENCES_API}/default_journal`,
+    url: `${PREFERENCES_URL}/onboarding`,
   });
 
-export const setDefaultJournal = (journalId) =>
+export const setOnboardingState = (data) => {
-  http({
+  return http({
     method: "POST",
-    url: `${PREFERENCES_API}/default_journal`,
+    url: `${PREFERENCES_URL}/onboarding`,
-    data: { id: journalId },
+    data,
-  });
-
-export const unsetDefaultJournal = () =>
-  http({
-    method: "DELETE",
-    url: `${PREFERENCES_API}/default_journal`,
   });
+};