Merge branch 'main' into add-labels-and-times

.github/workflows/lint.backend.yml

@@ -0,0 +1,25 @@
+name: Lint Moonstream backend
+
+on:
+  push:
+    paths:
+      - "backend/**"
+
+jobs:
+  build:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up python
+        uses: actions/setup-python@v2
+        with:
+          python-version: "3.8"
+      - name: Install test requirements
+        working-directory: ./backend
+        run: pip install -r requirements.txt
+      # - name: Mypy type check
+      #   working-directory: ./backend
+      #   run: mypy moonstream/
+      - name: Black syntax check
+        working-directory: ./backend
+        run: black --check moonstream/

.github/workflows/lint.crawlers.yml

@@ -0,0 +1,25 @@
+name: Lint Moonstream crawlers
+
+on:
+  push:
+    paths:
+      - "crawlers/**"
+
+jobs:
+  build:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up python
+        uses: actions/setup-python@v2
+        with:
+          python-version: "3.8"
+      - name: Install test requirements
+        working-directory: ./crawlers
+        run: pip install -e .[dev]
+      # - name: Mypy type check
+      #   working-directory: ./crawlers
+      #   run: mypy moonstreamcrawlers/
+      - name: Black syntax check
+        working-directory: ./crawlers
+        run: black --check moonstreamcrawlers/

.github/workflows/lint.db.yml

@@ -0,0 +1,25 @@
+name: Lint Moonstream db
+
+on:
+  push:
+    paths:
+      - "db/**"
+
+jobs:
+  build:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up python
+        uses: actions/setup-python@v2
+        with:
+          python-version: "3.8"
+      - name: Install test requirements
+        working-directory: ./db
+        run: pip install -e .[dev]
+      # - name: Mypy type check
+      #   working-directory: ./db
+      #   run: mypy moonstreamdb/
+      - name: Black syntax check
+        working-directory: ./db
+        run: black --check moonstreamdb/

.github/workflows/lint.frontend.yml

@@ -0,0 +1,21 @@
+name: Build Moonstream frontend
+
+on:
+  push:
+    paths:
+      - "frontend/**"
+
+jobs:
+  build:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up node.js
+        uses: actions/setup-node@v1
+        with:
+          node-version: "14.17.4"
+      - name: Check build
+        working-directory: ./frontend
+        run: |
+          yarn
+          yarn build

.github/workflows/locust.yml

@@ -0,0 +1,32 @@
+name: Locust summary
+
+on: [pull_request_target]
+
+jobs:
+  build:
+    runs-on: ubuntu-20.04
+    steps:
+      - name: PR head repo
+        id: head_repo_name
+        run: |
+          HEAD_REPO_NAME=$(jq -r '.pull_request.head.repo.full_name' "$GITHUB_EVENT_PATH")
+          echo "PR head repo: $HEAD_REPO_NAME"
+          echo "::set-output name=repo::$HEAD_REPO_NAME"
+      - name: Checkout git repo
+        uses: actions/checkout@v2
+        with:
+          repository: ${{ steps.head_repo_name.outputs.repo }}
+          fetch-depth: 0
+      - name: Install python
+        uses: actions/setup-python@v2
+        with:
+          python-version: "3.8"
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip setuptools
+          pip install bugout-locust
+      - name: Generate and send Locust summary
+        env:
+          BUGOUT_SECRET: ${{ secrets.BUGOUT_SECRET }}
+        run: |
+          locust.github publish

backend/moonstream/settings.py

@@ -3,7 +3,9 @@ import os
 from bugout.app import Bugout
 
 # Bugout
-bugout_client = Bugout()
+BUGOUT_BROOD_URL = os.environ.get("BUGOUT_BROOD_URL", "https://auth.bugout.dev")
+BUGOUT_SPIRE_URL = os.environ.get("BUGOUT_SPIRE_URL", "https://spire.bugout.dev")
+bugout_client = Bugout(brood_api_url=BUGOUT_BROOD_URL, spire_api_url=BUGOUT_SPIRE_URL)
 
 # Default value is "" instead of None so that mypy understands that MOONSTREAM_APPLICATION_ID is a string
 MOONSTREAM_APPLICATION_ID = os.environ.get("MOONSTREAM_APPLICATION_ID", "")
@@ -14,7 +16,7 @@ MOONSTREAM_DATA_JOURNAL_ID = os.environ.get("MOONSTREAM_DATA_JOURNAL_ID")
 if MOONSTREAM_DATA_JOURNAL_ID is None:
     raise ValueError("MOONSTREAM_DATA_JOURNAL_ID environment variable must be set")
 
-MOONSTREAM_ADMIN_ACCESS_TOKEN = os.environ.get("MOONSTREAM_ADMIN_ACCESS_TOKEN", "")
+MOONSTREAM_ADMIN_ACCESS_TOKEN = os.environ.get("MOONSTREAM_ADMIN_ACCESS_TOKEN")
 if MOONSTREAM_ADMIN_ACCESS_TOKEN is None:
     raise ValueError("MOONSTREAM_ADMIN_ACCESS_TOKEN environment variable must be set")
 

backend/sample.env

@@ -4,5 +4,6 @@ export MOONSTREAM_APPLICATION_ID="<issued_bugout_application_id>"
 export MOONSTREAM_DATA_JOURNAL_ID="<bugout_journal_id_to_store_blockchain_data>"
 export MOONSTREAM_DB_URI="postgresql://<username>:<password>@<db_host>:<db_port>/<db_name>"
 export MOONSTREAM_POOL_SIZE=0
-export MOONSTREAM_AUTO_USER_TOKEN="<Access token to application resources>"
+export MOONSTREAM_ADMIN_ACCESS_TOKEN="<Access token to application resources>"
-
+export BUGOUT_BROOD_URL="https://auth.bugout.dev"
+export BUGOUT_SPIRE_URL="https://spire.bugout.dev"

crawlers/moonstreamcrawlers/cli.py

@@ -2,7 +2,6 @@
 Moonstream crawlers CLI.
 """
 import argparse
-from distutils.util import strtobool
 from enum import Enum
 import json
 import sys
@@ -84,7 +83,7 @@ def ethcrawler_blocks_sync_handler(args: argparse.Namespace) -> None:
     starting_block: int = args.start
     while True:
         bottom_block_number, top_block_number = get_latest_blocks(
-            bool(strtobool(args.transactions))
+            with_transactions=not args.notransactions
         )
         bottom_block_number = max(bottom_block_number + 1, starting_block)
         if bottom_block_number >= top_block_number:
@@ -102,7 +101,7 @@ def ethcrawler_blocks_sync_handler(args: argparse.Namespace) -> None:
             # TODO(kompotkot): Set num_processes argument based on number of blocks to synchronize.
             crawl_blocks_executor(
                 block_numbers_list=blocks_numbers_list,
-                with_transactions=bool(strtobool(args.transactions)),
+                with_transactions=not args.notransactions,
                 num_processes=args.jobs,
             )
         print(f"Synchronized blocks from {bottom_block_number} to {top_block_number}")
@@ -118,7 +117,7 @@ def ethcrawler_blocks_add_handler(args: argparse.Namespace) -> None:
         print(f"Adding blocks {blocks_numbers_list[-1]}-{blocks_numbers_list[0]}")
         crawl_blocks_executor(
             block_numbers_list=blocks_numbers_list,
-            with_transactions=bool(strtobool(args.transactions)),
+            with_transactions=not args.notransactions,
         )
 
     print(f"Required {time.time() - startTime} with {MOONSTREAM_CRAWL_WORKERS} workers")
@@ -134,24 +133,26 @@ def ethcrawler_blocks_missing_handler(args: argparse.Namespace) -> None:
         missing_blocks_numbers = check_missing_blocks(
             blocks_numbers=blocks_numbers_list,
         )
+        if len(missing_blocks_numbers) > 0:
+            print(f"Found {len(missing_blocks_numbers)} missing blocks")
         missing_blocks_numbers_total.extend(missing_blocks_numbers)
-    print(f"Found {len(missing_blocks_numbers_total)} missing blocks")
+    print(f"Found {len(missing_blocks_numbers_total)} missing blocks total")
 
     time.sleep(5)
 
     if (len(missing_blocks_numbers_total)) > 0:
-        if bool(strtobool(args.lazy)):
+        if args.lazy:
             print("Executed lazy block crawler")
             crawl_blocks(
                 missing_blocks_numbers_total,
-                with_transactions=bool(strtobool(args.transactions)),
+                with_transactions=not args.notransactions,
-                verbose=True,
+                verbose=args.verbose,
             )
         else:
             crawl_blocks_executor(
                 missing_blocks_numbers_total,
-                with_transactions=bool(strtobool(args.transactions)),
+                with_transactions=not args.notransactions,
-                verbose=True,
+                verbose=args.verbose,
             )
     print(
         f"Required {time.time() - startTime} with {MOONSTREAM_CRAWL_WORKERS} workers "
@@ -205,11 +206,10 @@ def main() -> None:
         "synchronize", description="Synchronize to latest ethereum block commands"
     )
     parser_ethcrawler_blocks_sync.add_argument(
-        "-t",
+        "-n",
-        "--transactions",
+        "--notransactions",
-        choices=["True", "False"],
+        action="store_true",
-        default="True",
+        help="Skip crawling block transactions",
-        help="Add or not block transactions",
     )
     parser_ethcrawler_blocks_sync.add_argument(
         "-s",
@@ -246,11 +246,10 @@ def main() -> None:
         help="List of blocks range in format {bottom_block}-{top_block}",
     )
     parser_ethcrawler_blocks_add.add_argument(
-        "-t",
+        "-n",
-        "--transactions",
+        "--notransactions",
-        choices=["True", "False"],
+        action="store_true",
-        default="True",
+        help="Skip crawling block transactions",
-        help="Add or not block transactions",
     )
     parser_ethcrawler_blocks_add.set_defaults(func=ethcrawler_blocks_add_handler)
 
@@ -264,19 +263,23 @@ def main() -> None:
         help="List of blocks range in format {bottom_block}-{top_block}",
     )
     parser_ethcrawler_blocks_missing.add_argument(
-        "-t",
+        "-n",
-        "--transactions",
+        "--notransactions",
-        choices=["True", "False"],
+        action="store_true",
-        default="True",
+        help="Skip crawling block transactions",
-        help="Add or not block transactions",
     )
     parser_ethcrawler_blocks_missing.add_argument(
         "-l",
         "--lazy",
-        choices=["True", "False"],
+        action="store_true",
-        default="False",
         help="Lazy block adding one by one",
     )
+    parser_ethcrawler_blocks_missing.add_argument(
+        "-v",
+        "--verbose",
+        action="store_true",
+        help="Print additional information",
+    )
     parser_ethcrawler_blocks_missing.set_defaults(
         func=ethcrawler_blocks_missing_handler
     )

crawlers/moonstreamcrawlers/ethereum.py

@@ -119,16 +119,19 @@ def check_missing_blocks(blocks_numbers: List[int]) -> List[int]:
     Query block from postgres. If block does not presented in database,
     add to missing blocks numbers list.
     """
-    missing_blocks_numbers = []
+    bottom_block = min(blocks_numbers[-1], blocks_numbers[0])
-    for block_number in blocks_numbers:
+    top_block = max(blocks_numbers[-1], blocks_numbers[0])
-        with yield_db_session_ctx() as db_session:
+    with yield_db_session_ctx() as db_session:
-            block_exist = (
+        blocks_exist_raw = (
-                db_session.query(EthereumBlock.block_number)
+            db_session.query(EthereumBlock.block_number)
-                .filter(EthereumBlock.block_number == block_number)
+            .filter(EthereumBlock.block_number >= bottom_block)
-                .one_or_none()
+            .filter(EthereumBlock.block_number <= top_block)
-            )
+            .all()
-            if block_exist is None:
+        )
-                missing_blocks_numbers.append(block_number)
+    blocks_exist = [block[0] for block in blocks_exist_raw]
+    missing_blocks_numbers = [
+        block for block in blocks_numbers if block not in blocks_exist
+    ]
     return missing_blocks_numbers
 
 
@@ -177,6 +180,7 @@ def crawl_blocks_executor(
             for error in errors:
                 print(f"- {error}")
 
+
 def process_contract_deployments() -> List[Tuple[str, str]]:
     """
     Checks for new smart contracts that have been deployed to the blockchain but not registered in

crawlers/setup.py

@@ -33,7 +33,7 @@ setup(
     package_data={"moonstreamcrawlers": ["py.typed"]},
     zip_safe=False,
     install_requires=[
-        "moonstreamdb @ git+https://git@github.com/bugout-dev/moonstream.git@03a929568180d7eb53ea46a11f920db65ea7c772#egg=moonstreamdb&subdirectory=db",
+        "moonstreamdb @ git+https://git@github.com/bugout-dev/moonstream.git@ec3278e192119d1e8a273cfaab6cb53890d2e8e9#egg=moonstreamdb&subdirectory=db",
         "requests",
         "tqdm",
         "web3",

frontend/pages/index.js

@@ -123,7 +123,6 @@ const Homepage = () => {
   }, [isInit, router]);
 
   useLayoutEffect(() => {
-    console.log("rerender check");
     const imageLoader720 = new Image();
     imageLoader720.src = `${AWS_PATH}/background720.png`;
     imageLoader720.onload = () => {

frontend/src/components/EntriesNavigation.js

@@ -31,6 +31,7 @@ import {
   TagLabel,
   TagCloseButton,
   Spacer,
+  useBoolean,
 } from "@chakra-ui/react";
 import { useSubscriptions } from "../core/hooks";
 import StreamEntry from "./StreamEntry";
@@ -38,6 +39,7 @@ import UIContext from "../core/providers/UIProvider/context";
 import { FaFilter } from "react-icons/fa";
 import useStream from "../core/hooks/useStream";
 import { ImCancelCircle } from "react-icons/im";
+import { IoStopCircleOutline, IoPlayCircleOutline } from "react-icons/io5";
 
 const pageSize = 25;
 const FILTER_TYPES = {
@@ -61,6 +63,7 @@ const CONDITION = {
 
 const EntriesNavigation = () => {
   const ui = useContext(UIContext);
+  const [isStreamOn, setStreamState] = useBoolean(true);
   const { isOpen, onOpen, onClose } = useDisclosure();
   const { subscriptionsCache } = useSubscriptions();
   const [newFilterState, setNewFilterState] = useState([
@@ -80,7 +83,7 @@ const EntriesNavigation = () => {
       pageSize,
       refreshRate: 1500,
       searchQuery: ui.searchTerm,
-      enabled: true,
+      enabled: isStreamOn,
       isContent: false,
     });
 
@@ -110,7 +113,7 @@ const EntriesNavigation = () => {
       newFilterState[0].value === null
     ) {
       setFilterProps(0, {
-        value: subscriptionsCache.data.subscriptions[0].address,
+        value: subscriptionsCache?.data?.subscriptions[0]?.address,
       });
     }
   }, [subscriptionsCache, newFilterState, setFilterProps]);
@@ -253,7 +256,7 @@ const EntriesNavigation = () => {
                                 onChange={handleAddressChange(idx)}
                               >
                                 {!subscriptionsCache.isLoading &&
-                                  subscriptionsCache.data.subscriptions.map(
+                                  subscriptionsCache?.data?.subscriptions.map(
                                     (subscription, idx) => {
                                       return (
                                         <option
@@ -318,7 +321,8 @@ const EntriesNavigation = () => {
                             direction: DIRECTIONS.SOURCE,
                             condition: CONDITION.EQUAL,
                             value:
-                              subscriptionsCache.data.subscriptions[0].address,
+                              subscriptionsCache?.data?.subscriptions[0]
+                                ?.address,
                           },
                         ])
                       }
@@ -334,7 +338,8 @@ const EntriesNavigation = () => {
                             direction: DIRECTIONS.DESTINATION,
                             condition: CONDITION.EQUAL,
                             value:
-                              subscriptionsCache.data.subscriptions[0].address,
+                              subscriptionsCache?.data?.subscriptions[0]
+                                ?.address,
                           },
                         ])
                       }
@@ -356,8 +361,22 @@ const EntriesNavigation = () => {
               </DrawerFooter>
             </DrawerContent>
           </Drawer>
-          <Flex h="3rem" w="100%" bgColor="gray.200" alignItems="center">
+          <Flex h="3rem" w="100%" bgColor="gray.100" alignItems="center">
             <Flex maxW="90%">
+              <Flex direction="column">
+                <IconButton
+                  size="sm"
+                  onClick={() => setStreamState.toggle()}
+                  icon={
+                    isStreamOn ? (
+                      <IoStopCircleOutline size="32px" />
+                    ) : (
+                      <IoPlayCircleOutline size="32px" />
+                    )
+                  }
+                  colorScheme={isStreamOn ? "unsafe" : "suggested"}
+                />
+              </Flex>
               {filterState.map((filter, idx) => {
                 if (filter.type === FILTER_TYPES.DISABLED) return "";
                 return (
@@ -407,7 +426,6 @@ const EntriesNavigation = () => {
               id="StreamEntry"
               overflowY="scroll"
               direction="column"
-
               w="100%"
               onScroll={(e) => handleScroll(e)}
             >

frontend/src/components/Footer.js

@@ -8,11 +8,7 @@ const ICONS = [
     link: "https://discord.gg/FetK5BxD",
   },
 
-  { social: "twit", link: "https://twitter.com/Bugout_dev" },
+  { social: "twit", link: "https://twitter.com/moonstreamto" },
-  {
-    social: "slack",
-    link: "https://join.slack.com/t/bugout-dev/shared_invite/zt-fhepyt87-5XcJLy0iu702SO_hMFKNhQ",
-  },
 ];
 
 const SITEMAP_FLEX_PROPS = {
@@ -98,7 +94,7 @@ const Footer = () => (
         fontSize="xl"
         fontWeight="500"
       >
-        Stay in touch
+        Stay in touch{` `}
         <span role="img" aria-label="heart">
           💙
         </span>

frontend/src/components/LandingNavbar.js

@@ -87,10 +87,21 @@ const LandingNavbar = () => {
                   Log in
                 </Button>
               )}
-              {ui.isLoggedIn && <ChakraAccountIconButton />}
+              {ui.isLoggedIn && (
+                <ChakraAccountIconButton
+                  variant="link"
+                  colorScheme="secondary"
+                />
+              )}
             </ButtonGroup>
           </>
         )}
+        {ui.isLoggedIn && ui.isMobileView && (
+          <>
+            <Spacer />
+            <ChakraAccountIconButton variant="link" colorScheme="secondary" />
+          </>
+        )}
       </>
     </>
   );

frontend/src/components/Scrollable.js

@@ -19,7 +19,6 @@ const Scrollable = (props) => {
   const handleScroll = (e) => {
     const currentScroll = Math.ceil(getScrollPrecent(e) / 10);
     if (currentScroll > scrollDepth) {
-      // withTracking(
       setScrollDepth(currentScroll);
       isLoaded &&
         mixpanel.people.increment({

frontend/src/components/SignIn.js

@@ -38,7 +38,7 @@ const SignIn = ({ toggleModal }) => {
         Login now
       </Heading>
       <Text color="gray.1200" fontSize="md">
-        To your Bugout account
+        To your Moonstream account
       </Text>
       <form onSubmit={handleSubmit(login)}>
         <Stack width="100%" pt={4} spacing={3}>
@@ -49,7 +49,7 @@ const SignIn = ({ toggleModal }) => {
                 autoComplete="username"
                 variant="filled"
                 colorScheme="primary"
-                placeholder="Your Bugout username"
+                placeholder="Your Moonstream username"
                 name="username"
                 {...register("username", { required: true })}
                 ref={register({ required: "Username is required!" })}
@@ -64,7 +64,7 @@ const SignIn = ({ toggleModal }) => {
           </FormControl>
           <FormControl isInvalid={errors.password}>
             <PasswordInput
-              placeholder="Your Bugout password"
+              placeholder="Your Moonstream password"
               name="password"
               ref={register({ required: "Password is required!" })}
             />

frontend/src/components/StreamEntry.js

@@ -56,7 +56,6 @@ const StreamEntry = ({ entry, filterCallback, filterConstants }) => {
       transition="0.1s"
       _hover={{ bg: "secondary.200" }}
       flexBasis="50px"
-      flexGrow={1}
       direction="row"
       justifySelf="center"
       justifyContent="normal"