diff --git a/.github/workflows/lint.backend.yml b/.github/workflows/lint.backend.yml new file mode 100644 index 00000000..05d468c4 --- /dev/null +++ b/.github/workflows/lint.backend.yml @@ -0,0 +1,25 @@ +name: Lint Moonstream backend + +on: + push: + paths: + - "backend/**" + +jobs: + build: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + - name: Set up python + uses: actions/setup-python@v2 + with: + python-version: "3.8" + - name: Install test requirements + working-directory: ./backend + run: pip install -r requirements.txt + # - name: Mypy type check + # working-directory: ./backend + # run: mypy moonstream/ + - name: Black syntax check + working-directory: ./backend + run: black --check moonstream/ diff --git a/.github/workflows/lint.crawlers.yml b/.github/workflows/lint.crawlers.yml new file mode 100644 index 00000000..44c48bd3 --- /dev/null +++ b/.github/workflows/lint.crawlers.yml @@ -0,0 +1,25 @@ +name: Lint Moonstream crawlers + +on: + push: + paths: + - "crawlers/**" + +jobs: + build: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + - name: Set up python + uses: actions/setup-python@v2 + with: + python-version: "3.8" + - name: Install test requirements + working-directory: ./crawlers + run: pip install -e .[dev] + # - name: Mypy type check + # working-directory: ./crawlers + # run: mypy moonstreamcrawlers/ + - name: Black syntax check + working-directory: ./crawlers + run: black --check moonstreamcrawlers/ diff --git a/.github/workflows/lint.db.yml b/.github/workflows/lint.db.yml new file mode 100644 index 00000000..76042606 --- /dev/null +++ b/.github/workflows/lint.db.yml @@ -0,0 +1,25 @@ +name: Lint Moonstream db + +on: + push: + paths: + - "db/**" + +jobs: + build: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + - name: Set up python + uses: actions/setup-python@v2 + with: + python-version: "3.8" + - name: Install test requirements + working-directory: ./db + run: pip install -e .[dev] + # - name: Mypy type check + # working-directory: ./db + # run: mypy moonstreamdb/ + - name: Black syntax check + working-directory: ./db + run: black --check moonstreamdb/ diff --git a/.github/workflows/lint.frontend.yml b/.github/workflows/lint.frontend.yml new file mode 100644 index 00000000..07d1c3d0 --- /dev/null +++ b/.github/workflows/lint.frontend.yml @@ -0,0 +1,21 @@ +name: Build Moonstream frontend + +on: + push: + paths: + - "frontend/**" + +jobs: + build: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + - name: Set up node.js + uses: actions/setup-node@v1 + with: + node-version: "14.17.4" + - name: Check build + working-directory: ./frontend + run: | + yarn + yarn build diff --git a/.github/workflows/locust.yml b/.github/workflows/locust.yml new file mode 100644 index 00000000..175bf0ec --- /dev/null +++ b/.github/workflows/locust.yml @@ -0,0 +1,32 @@ +name: Locust summary + +on: [pull_request_target] + +jobs: + build: + runs-on: ubuntu-20.04 + steps: + - name: PR head repo + id: head_repo_name + run: | + HEAD_REPO_NAME=$(jq -r '.pull_request.head.repo.full_name' "$GITHUB_EVENT_PATH") + echo "PR head repo: $HEAD_REPO_NAME" + echo "::set-output name=repo::$HEAD_REPO_NAME" + - name: Checkout git repo + uses: actions/checkout@v2 + with: + repository: ${{ steps.head_repo_name.outputs.repo }} + fetch-depth: 0 + - name: Install python + uses: actions/setup-python@v2 + with: + python-version: "3.8" + - name: Install dependencies + run: | + python -m pip install --upgrade pip setuptools + pip install bugout-locust + - name: Generate and send Locust summary + env: + BUGOUT_SECRET: ${{ secrets.BUGOUT_SECRET }} + run: | + locust.github publish diff --git a/backend/moonstream/settings.py b/backend/moonstream/settings.py index 8bb5fa9e..7ad5bdc0 100644 --- a/backend/moonstream/settings.py +++ b/backend/moonstream/settings.py @@ -3,7 +3,9 @@ import os from bugout.app import Bugout # Bugout -bugout_client = Bugout() +BUGOUT_BROOD_URL = os.environ.get("BUGOUT_BROOD_URL", "https://auth.bugout.dev") +BUGOUT_SPIRE_URL = os.environ.get("BUGOUT_SPIRE_URL", "https://spire.bugout.dev") +bugout_client = Bugout(brood_api_url=BUGOUT_BROOD_URL, spire_api_url=BUGOUT_SPIRE_URL) # Default value is "" instead of None so that mypy understands that MOONSTREAM_APPLICATION_ID is a string MOONSTREAM_APPLICATION_ID = os.environ.get("MOONSTREAM_APPLICATION_ID", "") @@ -14,7 +16,7 @@ MOONSTREAM_DATA_JOURNAL_ID = os.environ.get("MOONSTREAM_DATA_JOURNAL_ID") if MOONSTREAM_DATA_JOURNAL_ID is None: raise ValueError("MOONSTREAM_DATA_JOURNAL_ID environment variable must be set") -MOONSTREAM_ADMIN_ACCESS_TOKEN = os.environ.get("MOONSTREAM_ADMIN_ACCESS_TOKEN", "") +MOONSTREAM_ADMIN_ACCESS_TOKEN = os.environ.get("MOONSTREAM_ADMIN_ACCESS_TOKEN") if MOONSTREAM_ADMIN_ACCESS_TOKEN is None: raise ValueError("MOONSTREAM_ADMIN_ACCESS_TOKEN environment variable must be set") diff --git a/backend/sample.env b/backend/sample.env index 6f5e80b5..32438dc7 100644 --- a/backend/sample.env +++ b/backend/sample.env @@ -4,5 +4,6 @@ export MOONSTREAM_APPLICATION_ID="" export MOONSTREAM_DATA_JOURNAL_ID="" export MOONSTREAM_DB_URI="postgresql://:@:/" export MOONSTREAM_POOL_SIZE=0 -export MOONSTREAM_AUTO_USER_TOKEN="" - +export MOONSTREAM_ADMIN_ACCESS_TOKEN="" +export BUGOUT_BROOD_URL="https://auth.bugout.dev" +export BUGOUT_SPIRE_URL="https://spire.bugout.dev" diff --git a/crawlers/moonstreamcrawlers/cli.py b/crawlers/moonstreamcrawlers/cli.py index 2b72051a..7bd6df5f 100644 --- a/crawlers/moonstreamcrawlers/cli.py +++ b/crawlers/moonstreamcrawlers/cli.py @@ -2,7 +2,6 @@ Moonstream crawlers CLI. """ import argparse -from distutils.util import strtobool from enum import Enum import json import sys @@ -84,7 +83,7 @@ def ethcrawler_blocks_sync_handler(args: argparse.Namespace) -> None: starting_block: int = args.start while True: bottom_block_number, top_block_number = get_latest_blocks( - bool(strtobool(args.transactions)) + with_transactions=not args.notransactions ) bottom_block_number = max(bottom_block_number + 1, starting_block) if bottom_block_number >= top_block_number: @@ -102,7 +101,7 @@ def ethcrawler_blocks_sync_handler(args: argparse.Namespace) -> None: # TODO(kompotkot): Set num_processes argument based on number of blocks to synchronize. crawl_blocks_executor( block_numbers_list=blocks_numbers_list, - with_transactions=bool(strtobool(args.transactions)), + with_transactions=not args.notransactions, num_processes=args.jobs, ) print(f"Synchronized blocks from {bottom_block_number} to {top_block_number}") @@ -118,7 +117,7 @@ def ethcrawler_blocks_add_handler(args: argparse.Namespace) -> None: print(f"Adding blocks {blocks_numbers_list[-1]}-{blocks_numbers_list[0]}") crawl_blocks_executor( block_numbers_list=blocks_numbers_list, - with_transactions=bool(strtobool(args.transactions)), + with_transactions=not args.notransactions, ) print(f"Required {time.time() - startTime} with {MOONSTREAM_CRAWL_WORKERS} workers") @@ -134,24 +133,26 @@ def ethcrawler_blocks_missing_handler(args: argparse.Namespace) -> None: missing_blocks_numbers = check_missing_blocks( blocks_numbers=blocks_numbers_list, ) + if len(missing_blocks_numbers) > 0: + print(f"Found {len(missing_blocks_numbers)} missing blocks") missing_blocks_numbers_total.extend(missing_blocks_numbers) - print(f"Found {len(missing_blocks_numbers_total)} missing blocks") + print(f"Found {len(missing_blocks_numbers_total)} missing blocks total") time.sleep(5) if (len(missing_blocks_numbers_total)) > 0: - if bool(strtobool(args.lazy)): + if args.lazy: print("Executed lazy block crawler") crawl_blocks( missing_blocks_numbers_total, - with_transactions=bool(strtobool(args.transactions)), - verbose=True, + with_transactions=not args.notransactions, + verbose=args.verbose, ) else: crawl_blocks_executor( missing_blocks_numbers_total, - with_transactions=bool(strtobool(args.transactions)), - verbose=True, + with_transactions=not args.notransactions, + verbose=args.verbose, ) print( f"Required {time.time() - startTime} with {MOONSTREAM_CRAWL_WORKERS} workers " @@ -205,11 +206,10 @@ def main() -> None: "synchronize", description="Synchronize to latest ethereum block commands" ) parser_ethcrawler_blocks_sync.add_argument( - "-t", - "--transactions", - choices=["True", "False"], - default="True", - help="Add or not block transactions", + "-n", + "--notransactions", + action="store_true", + help="Skip crawling block transactions", ) parser_ethcrawler_blocks_sync.add_argument( "-s", @@ -246,11 +246,10 @@ def main() -> None: help="List of blocks range in format {bottom_block}-{top_block}", ) parser_ethcrawler_blocks_add.add_argument( - "-t", - "--transactions", - choices=["True", "False"], - default="True", - help="Add or not block transactions", + "-n", + "--notransactions", + action="store_true", + help="Skip crawling block transactions", ) parser_ethcrawler_blocks_add.set_defaults(func=ethcrawler_blocks_add_handler) @@ -264,19 +263,23 @@ def main() -> None: help="List of blocks range in format {bottom_block}-{top_block}", ) parser_ethcrawler_blocks_missing.add_argument( - "-t", - "--transactions", - choices=["True", "False"], - default="True", - help="Add or not block transactions", + "-n", + "--notransactions", + action="store_true", + help="Skip crawling block transactions", ) parser_ethcrawler_blocks_missing.add_argument( "-l", "--lazy", - choices=["True", "False"], - default="False", + action="store_true", help="Lazy block adding one by one", ) + parser_ethcrawler_blocks_missing.add_argument( + "-v", + "--verbose", + action="store_true", + help="Print additional information", + ) parser_ethcrawler_blocks_missing.set_defaults( func=ethcrawler_blocks_missing_handler ) diff --git a/crawlers/moonstreamcrawlers/ethereum.py b/crawlers/moonstreamcrawlers/ethereum.py index 58c4b30f..f385fda5 100644 --- a/crawlers/moonstreamcrawlers/ethereum.py +++ b/crawlers/moonstreamcrawlers/ethereum.py @@ -119,16 +119,19 @@ def check_missing_blocks(blocks_numbers: List[int]) -> List[int]: Query block from postgres. If block does not presented in database, add to missing blocks numbers list. """ - missing_blocks_numbers = [] - for block_number in blocks_numbers: - with yield_db_session_ctx() as db_session: - block_exist = ( - db_session.query(EthereumBlock.block_number) - .filter(EthereumBlock.block_number == block_number) - .one_or_none() - ) - if block_exist is None: - missing_blocks_numbers.append(block_number) + bottom_block = min(blocks_numbers[-1], blocks_numbers[0]) + top_block = max(blocks_numbers[-1], blocks_numbers[0]) + with yield_db_session_ctx() as db_session: + blocks_exist_raw = ( + db_session.query(EthereumBlock.block_number) + .filter(EthereumBlock.block_number >= bottom_block) + .filter(EthereumBlock.block_number <= top_block) + .all() + ) + blocks_exist = [block[0] for block in blocks_exist_raw] + missing_blocks_numbers = [ + block for block in blocks_numbers if block not in blocks_exist + ] return missing_blocks_numbers @@ -177,6 +180,7 @@ def crawl_blocks_executor( for error in errors: print(f"- {error}") + def process_contract_deployments() -> List[Tuple[str, str]]: """ Checks for new smart contracts that have been deployed to the blockchain but not registered in diff --git a/crawlers/setup.py b/crawlers/setup.py index e64e9b30..07c2277a 100644 --- a/crawlers/setup.py +++ b/crawlers/setup.py @@ -33,7 +33,7 @@ setup( package_data={"moonstreamcrawlers": ["py.typed"]}, zip_safe=False, install_requires=[ - "moonstreamdb @ git+https://git@github.com/bugout-dev/moonstream.git@03a929568180d7eb53ea46a11f920db65ea7c772#egg=moonstreamdb&subdirectory=db", + "moonstreamdb @ git+https://git@github.com/bugout-dev/moonstream.git@ec3278e192119d1e8a273cfaab6cb53890d2e8e9#egg=moonstreamdb&subdirectory=db", "requests", "tqdm", "web3", diff --git a/db/requirements.txt b/db/requirements.txt deleted file mode 100644 index 3440a5b4..00000000 Binary files a/db/requirements.txt and /dev/null differ diff --git a/frontend/pages/index.js b/frontend/pages/index.js index f7c87572..c567c6df 100644 --- a/frontend/pages/index.js +++ b/frontend/pages/index.js @@ -123,7 +123,6 @@ const Homepage = () => { }, [isInit, router]); useLayoutEffect(() => { - console.log("rerender check"); const imageLoader720 = new Image(); imageLoader720.src = `${AWS_PATH}/background720.png`; imageLoader720.onload = () => { diff --git a/frontend/src/components/EntriesNavigation.js b/frontend/src/components/EntriesNavigation.js index 2b6a7409..90277207 100644 --- a/frontend/src/components/EntriesNavigation.js +++ b/frontend/src/components/EntriesNavigation.js @@ -31,6 +31,7 @@ import { TagLabel, TagCloseButton, Spacer, + useBoolean, } from "@chakra-ui/react"; import { useSubscriptions } from "../core/hooks"; import StreamEntry from "./StreamEntry"; @@ -38,6 +39,7 @@ import UIContext from "../core/providers/UIProvider/context"; import { FaFilter } from "react-icons/fa"; import useStream from "../core/hooks/useStream"; import { ImCancelCircle } from "react-icons/im"; +import { IoStopCircleOutline, IoPlayCircleOutline } from "react-icons/io5"; const pageSize = 25; const FILTER_TYPES = { @@ -61,6 +63,7 @@ const CONDITION = { const EntriesNavigation = () => { const ui = useContext(UIContext); + const [isStreamOn, setStreamState] = useBoolean(true); const { isOpen, onOpen, onClose } = useDisclosure(); const { subscriptionsCache } = useSubscriptions(); const [newFilterState, setNewFilterState] = useState([ @@ -80,7 +83,7 @@ const EntriesNavigation = () => { pageSize, refreshRate: 1500, searchQuery: ui.searchTerm, - enabled: true, + enabled: isStreamOn, isContent: false, }); @@ -110,7 +113,7 @@ const EntriesNavigation = () => { newFilterState[0].value === null ) { setFilterProps(0, { - value: subscriptionsCache.data.subscriptions[0].address, + value: subscriptionsCache?.data?.subscriptions[0]?.address, }); } }, [subscriptionsCache, newFilterState, setFilterProps]); @@ -253,7 +256,7 @@ const EntriesNavigation = () => { onChange={handleAddressChange(idx)} > {!subscriptionsCache.isLoading && - subscriptionsCache.data.subscriptions.map( + subscriptionsCache?.data?.subscriptions.map( (subscription, idx) => { return (