Merge branch 'smart-contract-crawlers' of github.com:bugout-dev/moonstream into smart-contract-crawlers

backend/deploy/deploy.bash

@@ -3,20 +3,25 @@
 # Deployment script - intended to run on Moonstream servers
 
 # Main
-APP_DIR="${APP_DIR:-/home/ubuntu/app}"
+APP_DIR="${APP_DIR:-/home/ubuntu/moonstream}"
 AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION:-us-east-1}"
-PYTHON_ENV_DIR="${PYTHON_ENV_DIR:-/home/ubuntu/app-env}"
+PYTHON_ENV_DIR="${PYTHON_ENV_DIR:-/home/ubuntu/moonstream-env}"
 PYTHON="${PYTHON_ENV_DIR}/bin/python"
 PIP="${PYTHON_ENV_DIR}/bin/pip"
 SCRIPT_DIR="$(realpath $(dirname $0))"
 PARAMETERS_SCRIPT="${SCRIPT_DIR}/parameters.py"
-SECRETS_DIR="${SECRETS_DIR:-/home/ubuntu/app-secrets}"
+SECRETS_DIR="${SECRETS_DIR:-/home/ubuntu/moonstream-secrets}"
 PARAMETERS_ENV_PATH="${SECRETS_DIR}/app.env"
 AWS_SSM_PARAMETER_PATH="${AWS_SSM_PARAMETER_PATH:-/moonstream/prod}"
 SERVICE_FILE="${SCRIPT_DIR}/moonstream.service"
 
 set -eu
 
+echo
+echo
+echo "Updating pip and setuptools"
+"${PIP}" install -U pip setuptools
+
 echo
 echo
 echo "Updating Python dependencies"

backend/deploy/moonstream.service

@@ -5,9 +5,9 @@ After=network.target
 [Service]
 User=ubuntu
 Group=www-data
-WorkingDirectory=/home/ubuntu/app
+WorkingDirectory=/home/ubuntu/moonstream
-EnvironmentFile=/home/ubuntu/secrets/app.env
+EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
-ExecStart=/home/ubuntu/server-env/bin/uvicorn --host 0.0.0.0 --port 7481 --workers 8 moonstream.api:app
+ExecStart=/home/ubuntu/moonstream-env/bin/uvicorn --host 0.0.0.0 --port 7481 --workers 8 moonstream.api:app
 SyslogIdentifier=moonstream
 
 [Install]

backend/moonstream/data.py

@@ -1,11 +1,41 @@
 """
 Pydantic schemas for the Moonstream HTTP API
 """
+from enum import Enum
 from typing import List, Optional
 
+
 from pydantic import BaseModel, Field
 
 
+class SubscriptionTypeResourceData(BaseModel):
+    id: str
+    name: str
+    description: str
+    subscription_plan_id: Optional[str] = None
+    active: bool = False
+
+
+class SubscriptionTypesListResponce(BaseModel):
+    subscriptions: List[SubscriptionTypeResourceData] = Field(default_factory=list)
+
+
+class SubscriptionResourceData(BaseModel):
+    id: str
+    address: str
+    color: str
+    label: str
+    user_id: str
+    subscription_type_id: str
+
+
+class CreateSubscriptionRequest(BaseModel):
+    address: str
+    color: str
+    label: str
+    subscription_type_id: str
+
+
 class PingResponse(BaseModel):
     """
     Schema for ping response
@@ -56,4 +86,4 @@ class EVMEventSignature(BaseModel):
 
 class ContractABI(BaseModel):
     functions: List[EVMFunctionSignature]
-    events: List[EVMEventSignature]
+    events: List[EVMEventSignature]

backend/moonstream/routes/subscriptions.py

@@ -2,11 +2,11 @@
 The Moonstream subscriptions HTTP API
 """
 import logging
-from typing import Dict
+from typing import Dict, List
 
 from bugout.data import BugoutResource, BugoutResources
 from bugout.exceptions import BugoutResponseException
-from fastapi import Body, FastAPI, HTTPException, Request
+from fastapi import Body, FastAPI, HTTPException, Request, Form
 from fastapi.middleware.cors import CORSMiddleware
 
 from .. import data
@@ -49,18 +49,57 @@ whitelist_paths.update(DOCS_PATHS)
 app.add_middleware(BroodAuthMiddleware, whitelist=whitelist_paths)
 
 
-@app.post("/", tags=["subscriptions"], response_model=data.SubscriptionResponse)
+@app.post("/", tags=["subscriptions"], response_model=data.SubscriptionResourceData)
 async def add_subscription_handler(
-    request: Request, subscription_data: data.SubscriptionRequest = Body(...)
+    request: Request,  # subscription_data: data.CreateSubscriptionRequest = Body(...)
-) -> data.SubscriptionResponse:
+    address: str = Form(...),
+    color: str = Form(...),
+    label: str = Form(...),
+    subscription_type_id: str = Form(...),
+) -> data.SubscriptionResourceData:
     """
     Add subscription to blockchain stream data for user.
     """
+    subscription_data = data.CreateSubscriptionRequest(
+        address=address,
+        color=color,
+        label=label,
+        subscription_type_id=subscription_type_id,
+    )
+
     token = request.state.token
+
+    params = {"type": "subscription_type"}
+
+    # request availble subscriptions
+    try:
+        subscription_resources: BugoutResources = bc.list_resources(
+            token=token, params=params
+        )
+    except BugoutResponseException as e:
+        raise HTTPException(status_code=e.status_code, detail=e.detail)
+    except Exception as e:
+        raise HTTPException(status_code=500)
+
+    # allowed subscriptions
+    subscription_ids_list = [
+        resource.resource_data["id"] for resource in subscription_resources.resources
+    ]
+
+    if subscription_data.subscription_type_id not in subscription_ids_list:
+        raise HTTPException(
+            status_code=403, detail="Subscription type is not avilable."
+        )
+
     user = request.state.user
+
+    # chek if that contract not already setted up
+
     resource_data = {"user_id": str(user.id)}
     resource_data.update(subscription_data)
+
     try:
+
         resource: BugoutResource = bc.create_resource(
             token=token,
             application_id=MOONSTREAM_APPLICATION_ID,
@@ -70,9 +109,42 @@ async def add_subscription_handler(
         raise HTTPException(status_code=e.status_code, detail=e.detail)
     except Exception as e:
         raise HTTPException(status_code=500)
-    return data.SubscriptionResponse(
+
+    return data.SubscriptionResourceData(
+        id=str(resource.id),
         user_id=resource.resource_data["user_id"],
-        blockchain=resource.resource_data["blockchain"],
+        address=resource.resource_data["address"],
+        color=resource.resource_data["color"],
+        label=resource.resource_data["label"],
+        subscription_type_id=resource.resource_data["subscription_type_id"],
+    )
+
+
+@app.delete(
+    "/{subscription_id}",
+    tags=["subscriptions"],
+    response_model=data.SubscriptionResourceData,
+)
+async def delete_subscription_handler(request: Request, subscription_id: str):
+    """
+    Delete subscriptions.
+    """
+
+    token = request.state.token
+    try:
+        deleted_resource = bc.delete_resource(token=token, resource_id=subscription_id)
+    except BugoutResponseException as e:
+        raise HTTPException(status_code=e.status_code, detail=e.detail)
+    except Exception as e:
+        raise HTTPException(status_code=500)
+
+    return data.SubscriptionResourceData(
+        id=str(deleted_resource.id),
+        user_id=deleted_resource.resource_data["user_id"],
+        address=deleted_resource.resource_data["address"],
+        color=deleted_resource.resource_data["color"],
+        label=deleted_resource.resource_data["label"],
+        subscription_type_id=deleted_resource.resource_data["subscription_type_id"],
     )
 
 
@@ -86,15 +158,48 @@ async def get_subscriptions_handler(request: Request) -> data.SubscriptionsListR
     try:
         resources: BugoutResources = bc.list_resources(token=token, params=params)
     except BugoutResponseException as e:
+        if e.detail == "Resources not found":
+            return data.SubscriptionsListResponse(subscriptions=[])
         raise HTTPException(status_code=e.status_code, detail=e.detail)
     except Exception as e:
         raise HTTPException(status_code=500)
+
     return data.SubscriptionsListResponse(
         subscriptions=[
-            data.SubscriptionResponse(
+            data.SubscriptionResourceData(
+                id=str(resource.id),
                 user_id=resource.resource_data["user_id"],
-                blockchain=resource.resource_data["blockchain"],
+                address=resource.resource_data["address"],
+                color=resource.resource_data["color"],
+                label=resource.resource_data["label"],
+                subscription_type_id=resource.resource_data["subscription_type_id"],
             )
             for resource in resources.resources
         ]
     )
+
+
+@app.get(
+    "/types", tags=["subscriptions"], response_model=data.SubscriptionTypesListResponce
+)
+async def get_available_subscriptions_type(
+    request: Request,
+) -> data.SubscriptionTypesListResponce:
+
+    """
+    Get available's subscriptions types.
+    """
+    token = request.state.token
+    params = {"type": "subscription_type"}
+    try:
+        resources: BugoutResources = bc.list_resources(token=token, params=params)
+    except BugoutResponseException as e:
+        raise HTTPException(status_code=e.status_code, detail=e.detail)
+    except Exception as e:
+        raise HTTPException(status_code=500)
+    return data.SubscriptionTypesListResponce(
+        subscriptions=[
+            data.SubscriptionTypeResourceData.validate(resource.resource_data)
+            for resource in resources.resources
+        ]
+    )

backend/requirements.txt

@@ -8,6 +8,7 @@ certifi==2021.5.30
 charset-normalizer==2.0.3
 click==8.0.1
 fastapi==0.66.0
+-e git+https://git@github.com/bugout-dev/moonstream.git@60e90219a8e24077a1ab046463775f837df5f03e#egg=moonstreamdb&subdirectory=db
 h11==0.12.0
 idna==3.2
 jmespath==0.10.0

backend/setup.py

@@ -29,5 +29,4 @@ setup(
         "Topic :: Software Development :: Libraries",
     ],
     url="https://github.com/bugout-dev/moonstream",
-    entry_points={"console_scripts": ["moonstream=moonstream.cli:main"]},
 )

crawlers/.gitignore

@@ -162,4 +162,6 @@ cython_debug/
 # Custom
 dev.env
 prod.env
+prod.env.ps1
+dev.env.ps1
 .venv

crawlers/moonstreamcrawlers/version.py

@@ -0,0 +1,5 @@
+"""
+Moonstream crawlers version.
+"""
+
+MOONSTREAMCRAWLERS_VERSION = "0.0.1"

crawlers/sample.env

@@ -1,2 +1,3 @@
 export MOONSTREAM_IPC_PATH=null
-export MOONSTREAM_CRAWL_WORKERS_RAW=4
+export MOONSTREAM_CRAWL_WORKERS=4
+export MOONSTREAM_DB_URI="<moonstream_database_uri>"

crawlers/setup.py

@@ -1,12 +1,14 @@
 from setuptools import find_packages, setup
 
+from moonstreamcrawlers.version import MOONSTREAMCRAWLERS_VERSION
+
 long_description = ""
 with open("README.md") as ifp:
     long_description = ifp.read()
 
 setup(
     name="moonstreamcrawlers",
-    version="0.0.1",
+    version=MOONSTREAMCRAWLERS_VERSION,
     author="Bugout.dev",
     author_email="engineers@bugout.dev",
     license="Apache License 2.0",
@@ -28,7 +30,7 @@ setup(
     ],
     python_requires=">=3.6",
     packages=find_packages(),
-    package_data={"bugout": ["py.typed"]},
+    package_data={"moonstreamcrawlers": ["py.typed"]},
     zip_safe=False,
     install_requires=["web3"],
     extras_require={"dev": ["black", "mypy"]},

db/alembic.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env sh
+
+PYTHONPATH=".:$PYTHONPATH" alembic "$@"

db/moonstreamdb/db.py

@@ -1,5 +1,5 @@
 """
-Exploration database connection.
+Moonstream database connection.
 """
 from contextlib import contextmanager
 import os
@@ -7,20 +7,20 @@ import os
 from sqlalchemy import create_engine
 from sqlalchemy.orm import sessionmaker, Session
 
-EXPLORATION_DB_URI = os.environ.get("EXPLORATION_DB_URI")
+MOONSTREAM_DB_URI = os.environ.get("MOONSTREAM_DB_URI")
-if EXPLORATION_DB_URI is None:
+if MOONSTREAM_DB_URI is None:
-    raise ValueError("EXPLORATION_DB_URI environment variable must be set")
+    raise ValueError("MOONSTREAM_DB_URI environment variable must be set")
-EXPLORATION_POOL_SIZE_RAW = os.environ.get("EXPLORATION_POOL_SIZE", 0)
+MOONSTREAM_POOL_SIZE_RAW = os.environ.get("MOONSTREAM_POOL_SIZE", 0)
 try:
-    if EXPLORATION_POOL_SIZE_RAW is not None:
+    if MOONSTREAM_POOL_SIZE_RAW is not None:
-        EXPLORATION_POOL_SIZE = int(EXPLORATION_POOL_SIZE_RAW)
+        MOONSTREAM_POOL_SIZE = int(MOONSTREAM_POOL_SIZE_RAW)
 except:
     raise Exception(
-        f"Could not parse EXPLORATION_POOL_SIZE as int: {EXPLORATION_POOL_SIZE_RAW}"
+        f"Could not parse MOONSTREAM_POOL_SIZE as int: {MOONSTREAM_POOL_SIZE_RAW}"
     )
 
 # https://docs.sqlalchemy.org/en/14/core/pooling.html#sqlalchemy.pool.QueuePool
-engine = create_engine(EXPLORATION_DB_URI, pool_size=EXPLORATION_POOL_SIZE)
+engine = create_engine(MOONSTREAM_DB_URI, pool_size=MOONSTREAM_POOL_SIZE)
 SessionLocal = sessionmaker(bind=engine)
 
 

db/moonstreamdb/version.py

@@ -0,0 +1,5 @@
+"""
+Moonstream database version.
+"""
+
+MOONSTREAMDB_VERSION = "0.0.1"

db/sample.env

@@ -1 +1 @@
-export EXPLORATION_DB_URI="postgresql://<username>:<password>@<db_host>:<db_port>/<db_name>"
+export MOONSTREAM_DB_URI="postgresql://<username>:<password>@<db_host>:<db_port>/<db_name>"

db/setup.py

@@ -1,12 +1,14 @@
 from setuptools import find_packages, setup
 
+from moonstreamdb.version import MOONSTREAMDB_VERSION
+
 long_description = ""
 with open("README.md") as ifp:
     long_description = ifp.read()
 
 setup(
     name="moonstreamdb",
-    version="0.0.1",
+    version=MOONSTREAMDB_VERSION,
     author="Bugout.dev",
     author_email="engineers@bugout.dev",
     license="Apache License 2.0",
@@ -28,7 +30,7 @@ setup(
     ],
     python_requires=">=3.6",
     packages=find_packages(),
-    package_data={"bugout": ["py.typed"]},
+    package_data={"moonstreamdb": ["py.typed"]},
     zip_safe=False,
     install_requires=["alembic", "psycopg2-binary", "sqlalchemy"],
     extras_require={"dev": ["black", "mypy"]},

frontend/pages/subscriptions.js

@@ -100,7 +100,7 @@ const Subscriptions = () => {
                 Add new
               </Button>
             </Flex>
-            <SubscriptionsList />
+            <SubscriptionsList data={subscriptionsCache.data} />
           </Flex>
         </ScaleFade>
       )}

frontend/sample.env

@@ -1,7 +1,9 @@
-export REACT_APP_SIMIOTICS_SEARCH_URL=http://localhost:5000
+export NEXT_PUBLIC_SIMIOTICS_SEARCH_URL=http://localhost:5000
-export REACT_APP_MIXPANEL_TOKEN="<YOUR MIXPANEL TOKEN HERE>"
+export NEXT_PUBLIC_MIXPANEL_TOKEN="<YOUR MIXPANEL TOKEN HERE>"
-export REACT_APP_SIMIOTICS_AUTH_URL=http://localhost:7474
+export NEXT_PUBLIC_SIMIOTICS_AUTH_URL=http://localhost:7474
-export REACT_APP_SIMIOTICS_JOURNALS_URL=http://localhost:7475
+export NEXT_PUBLIC_SIMIOTICS_JOURNALS_URL=http://localhost:7475
-export REACT_APP_BUGOUT_CONTACTUS_TOKEN="<Brood token for contact user>"
+export NEXT_PUBLIC_BUGOUT_CONTACTUS_TOKEN="<Brood token for contact user>"
-export REACT_APP_BUGOUT_CONTACTUS_JOURNAL_ID="<journal ID for contact journal>"
+export NEXT_PUBLIC_BUGOUT_CONTACTUS_JOURNAL_ID="<journal ID for contact journal>"
-export REACT_APP_STRIPE_PUBLISHABLE_KEY="<stripe publishable key>"
+export NEXT_PUBLIC_STRIPE_PUBLISHABLE_KEY="<stripe publishable key>"
+export NEXT_PUBLIC_MOONSTREAM_API_URL=http://localhost:7481
+

frontend/src/components/NewSubscription.js

@@ -44,6 +44,7 @@ const NewSubscription = ({ isFreeOption, onClose }) => {
       type: isFreeOption ? "free" : radioState,
     });
   };
+
   return (
     <form onSubmit={handleSubmit(createSubscriptionWrap)}>
       <ModalHeader>Subscribe to a new address</ModalHeader>
@@ -84,9 +85,9 @@ const NewSubscription = ({ isFreeOption, onClose }) => {
 
           <FormControl isInvalid={errors.type}>
             <HStack {...group} alignItems="stretch">
-              {typesCache.data.map((type) => {
+              {typesCache.data.subscriptions.map((type) => {
                 const radio = getRadioProps({
-                  value: type.subscription_type,
+                  value: type.id,
                   isDisabled:
                     !type.active ||
                     (isFreeOption &&
@@ -94,7 +95,7 @@ const NewSubscription = ({ isFreeOption, onClose }) => {
                 });
                 if (!type.subscription_plan_id) return "";
                 return (
-                  <RadioCard key={`subscription-type-${type.id}`} {...radio}>
+                  <RadioCard key={`subscription_type_${type.id}`} {...radio}>
                     {type.name}
                   </RadioCard>
                 );
@@ -102,6 +103,7 @@ const NewSubscription = ({ isFreeOption, onClose }) => {
             </HStack>
           </FormControl>
         </Stack>
+        <Input placeholder="color" name="color" ref={register()}></Input>
       </ModalBody>
       <ModalFooter>
         <Button

frontend/src/components/SubscriptionsList.js

@@ -18,12 +18,11 @@ import CopyButton from "./CopyButton";
 import { useSubscriptions } from "../core/hooks";
 import ConfirmationRequest from "./ConfirmationRequest";
 
-const List = () => {
+const List = (data) => {
   const { subscriptionsCache, changeNote, deleteSubscription } =
     useSubscriptions();
 
   const updateCallback = ({ id, note }) => {
-    console.log('updateCallback', id)
     changeNote.mutate({ id, note });
   };
 

frontend/src/core/hooks/useSubscriptions.js

@@ -27,7 +27,7 @@ const useSubscriptions = () => {
 
   const getSubscriptions = async () => {
     const response = await SubscriptionsService.getSubscriptions();
-    return response.data.data;
+    return response.data;
   };
 
   const subscriptionsCache = useQuery(["subscriptions"], getSubscriptions, {
@@ -39,7 +39,7 @@ const useSubscriptions = () => {
 
   const getSubscriptionTypes = async () => {
     const response = await SubscriptionsService.getTypes();
-    return response.data.data;
+    return response.data;
   };
 
   const typesCache = useQuery(["subscription_types"], getSubscriptionTypes, {
@@ -73,14 +73,23 @@ const useSubscriptions = () => {
     },
   });
 
-  const deleteSubscription = useMutation(SubscriptionsService.deleteSubscription(), {
+  const deleteSubscription = useMutation(
-    onError: (error) => toast(error, "error"),
+    SubscriptionsService.deleteSubscription(),
-    onSuccess: (response) => {
+    {
-      subscriptionsCache.refetch();
+      onError: (error) => toast(error, "error"),
-    },
+      onSuccess: (response) => {
-  });
+        subscriptionsCache.refetch();
+      },
+    }
+  );
 
-  return { createSubscription, subscriptionsCache, typesCache, changeNote, deleteSubscription };
+  return {
+    createSubscription,
+    subscriptionsCache,
+    typesCache,
+    changeNote,
+    deleteSubscription,
+  };
 };
 
 export default useSubscriptions;

frontend/src/core/services/subscriptions.service.js

@@ -1,7 +1,7 @@
 import { http } from "../utils";
 // import axios from "axios";
 
-const API = process.env.NEXT_PUBLIC_SIMIOTICS_AUTH_URL;
+const API = process.env.NEXT_PUBLIC_MOONSTREAM_API_URL;
 
 export const getStream = ({ searchTerm, limit, offset, isContent }) =>
   http({
@@ -18,13 +18,13 @@ export const getStream = ({ searchTerm, limit, offset, isContent }) =>
 export const getTypes = () =>
   http({
     method: "GET",
-    url: `${API}/subscription_types/`,
+    url: `${API}/subscriptions/types`,
   });
 
 export const getSubscriptions = () =>
   http({
     method: "GET",
-    url: `${API}/subscriptions/`,
+    url: `${API}/subscriptions`,
   });
 
 export const create = ({ address, note, blockchain }) => {
@@ -47,10 +47,11 @@ export const deleteJournal = (id) => () =>
 
 export const createSubscription =
   () =>
-  ({ address, type, label }) => {
+  ({ address, type, label, color }) => {
     const data = new FormData();
     data.append("address", address);
-    data.append("subscription_type", type);
+    data.append("subscription_type_id", type);
+    data.append("color", color);
     data.append("label", label);
     return http({
       method: "POST",
@@ -75,7 +76,7 @@ export const modifySubscription =
 export const deleteSubscription = () => (id) => {
   return http({
     method: "DELETE",
-    url: `${API}/subscription/${id}`,
+    url: `${API}/subscriptions/${id}`,
   });
 };
 

frontend/src/core/utils/mockupRequests.js

@@ -234,28 +234,5 @@ const enableMockupRequests = (axiosInstance) => {
       offset: 0,
     },
   });
-
-  mock.onGet(`${MOCK_API}/subscriptions/`).reply(200, {
-    data: {
-      is_free_subscription_availible: true,
-      subscriptions: MockSubscriptions,
-    },
-  });
-
-  mock.onPost(`${MOCK_API}/subscriptions/`).reply((config) => {
-    const params = config.data; // FormData of {name: ..., file: ...}
-    const id = params.get("id");
-    const label = params.get("label");
-    const address = params.get("address");
-    const subscription_type = params.get("subscription_type");
-
-    return new Promise(function (resolve) {
-      setTimeout(function () {
-        const data = { id, label, address, subscription_type };
-        MockSubscriptions.push({ ...data });
-        resolve([200, { message: "OK", result: true }]);
-      }, 1000);
-    });
-  });
 };
 export default enableMockupRequests;