mirror
/
moonstream
kopia lustrzana https://github.com/bugout-dev/moonstream
1
0
Forkuj 0
Kod Zgłoszenia Projekty Wydania Wiki Aktywność

Merge branch 'main' into add-labels-and-times

pull/54/head
Andrey Dolgolev 2021-08-03 18:04:19 +03:00
rodzic 164ebfe537 b52827cdf3
commit d0a898d836
18 zmienionych plików z 222 dodań i 62 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

25
.github/workflows/lint.backend.yml vendored 100644
Wyświetl plik

@ -0,0 +1,25 @@
name: Lint Moonstream backend
on:
push:
paths:
- "backend/**"
jobs:
build:
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v2
- name: Set up python
uses: actions/setup-python@v2
with:
python-version: "3.8"
- name: Install test requirements
working-directory: ./backend
run: pip install -r requirements.txt
# - name: Mypy type check
# working-directory: ./backend
# run: mypy moonstream/
- name: Black syntax check
working-directory: ./backend
run: black --check moonstream/

25
.github/workflows/lint.crawlers.yml vendored 100644
Wyświetl plik

@ -0,0 +1,25 @@
name: Lint Moonstream crawlers
on:
push:
paths:
- "crawlers/**"
jobs:
build:
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v2
- name: Set up python
uses: actions/setup-python@v2
with:
python-version: "3.8"
- name: Install test requirements
working-directory: ./crawlers
run: pip install -e .[dev]
# - name: Mypy type check
# working-directory: ./crawlers
# run: mypy moonstreamcrawlers/
- name: Black syntax check
working-directory: ./crawlers
run: black --check moonstreamcrawlers/

25
.github/workflows/lint.db.yml vendored 100644
Wyświetl plik

@ -0,0 +1,25 @@
name: Lint Moonstream db
on:
push:
paths:
- "db/**"
jobs:
build:
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v2
- name: Set up python
uses: actions/setup-python@v2
with:
python-version: "3.8"
- name: Install test requirements
working-directory: ./db
run: pip install -e .[dev]
# - name: Mypy type check
# working-directory: ./db
# run: mypy moonstreamdb/
- name: Black syntax check
working-directory: ./db
run: black --check moonstreamdb/

21
.github/workflows/lint.frontend.yml vendored 100644
Wyświetl plik

@ -0,0 +1,21 @@
name: Build Moonstream frontend
on:
push:
paths:
- "frontend/**"
jobs:
build:
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v2
- name: Set up node.js
uses: actions/setup-node@v1
with:
node-version: "14.17.4"
- name: Check build
working-directory: ./frontend
run: |
yarn
yarn build

32
.github/workflows/locust.yml vendored 100644
Wyświetl plik

@ -0,0 +1,32 @@
name: Locust summary
on: [pull_request_target]
jobs:
build:
runs-on: ubuntu-20.04
steps:
- name: PR head repo
id: head_repo_name
run: |
HEAD_REPO_NAME=$(jq -r '.pull_request.head.repo.full_name' "$GITHUB_EVENT_PATH")
echo "PR head repo: $HEAD_REPO_NAME"
echo "::set-output name=repo::$HEAD_REPO_NAME"
- name: Checkout git repo
uses: actions/checkout@v2
with:
repository: ${{ steps.head_repo_name.outputs.repo }}
fetch-depth: 0
- name: Install python
uses: actions/setup-python@v2
with:
python-version: "3.8"
- name: Install dependencies
run: |
python -m pip install --upgrade pip setuptools
pip install bugout-locust
- name: Generate and send Locust summary
env:
BUGOUT_SECRET: ${{ secrets.BUGOUT_SECRET }}
run: |
locust.github publish

6
backend/moonstream/settings.py
Wyświetl plik

@ -3,7 +3,9 @@ import os
from bugout.app import Bugout
# Bugout
bugout_client = Bugout()
BUGOUT_BROOD_URL = os.environ.get("BUGOUT_BROOD_URL", "https://auth.bugout.dev")
BUGOUT_SPIRE_URL = os.environ.get("BUGOUT_SPIRE_URL", "https://spire.bugout.dev")
bugout_client = Bugout(brood_api_url=BUGOUT_BROOD_URL, spire_api_url=BUGOUT_SPIRE_URL)
# Default value is "" instead of None so that mypy understands that MOONSTREAM_APPLICATION_ID is a string
MOONSTREAM_APPLICATION_ID = os.environ.get("MOONSTREAM_APPLICATION_ID", "")
@ -14,7 +16,7 @@ MOONSTREAM_DATA_JOURNAL_ID = os.environ.get("MOONSTREAM_DATA_JOURNAL_ID")
if MOONSTREAM_DATA_JOURNAL_ID is None:
raise ValueError("MOONSTREAM_DATA_JOURNAL_ID environment variable must be set")
MOONSTREAM_ADMIN_ACCESS_TOKEN = os.environ.get("MOONSTREAM_ADMIN_ACCESS_TOKEN", "")
MOONSTREAM_ADMIN_ACCESS_TOKEN = os.environ.get("MOONSTREAM_ADMIN_ACCESS_TOKEN")
if MOONSTREAM_ADMIN_ACCESS_TOKEN is None:
raise ValueError("MOONSTREAM_ADMIN_ACCESS_TOKEN environment variable must be set")

5
backend/sample.env
Wyświetl plik

@ -4,5 +4,6 @@ export MOONSTREAM_APPLICATION_ID="<issued_bugout_application_id>"
export MOONSTREAM_DATA_JOURNAL_ID="<bugout_journal_id_to_store_blockchain_data>"
export MOONSTREAM_DB_URI="postgresql://<username>:<password>@<db_host>:<db_port>/<db_name>"
export MOONSTREAM_POOL_SIZE=0
export MOONSTREAM_AUTO_USER_TOKEN="<Access token to application resources>"
export MOONSTREAM_ADMIN_ACCESS_TOKEN="<Access token to application resources>"
export BUGOUT_BROOD_URL="https://auth.bugout.dev"
export BUGOUT_SPIRE_URL="https://spire.bugout.dev"

57
crawlers/moonstreamcrawlers/cli.py
Wyświetl plik

@ -2,7 +2,6 @@
Moonstream crawlers CLI.
"""
import argparse
from distutils.util import strtobool
from enum import Enum
import json
import sys
@ -84,7 +83,7 @@ def ethcrawler_blocks_sync_handler(args: argparse.Namespace) -> None:
starting_block: int = args.start
while True:
bottom_block_number, top_block_number = get_latest_blocks(
bool(strtobool(args.transactions))
with_transactions=not args.notransactions
)
bottom_block_number = max(bottom_block_number + 1, starting_block)
if bottom_block_number >= top_block_number:
@ -102,7 +101,7 @@ def ethcrawler_blocks_sync_handler(args: argparse.Namespace) -> None:
# TODO(kompotkot): Set num_processes argument based on number of blocks to synchronize.
crawl_blocks_executor(
block_numbers_list=blocks_numbers_list,
with_transactions=bool(strtobool(args.transactions)),
with_transactions=not args.notransactions,
num_processes=args.jobs,
)
print(f"Synchronized blocks from {bottom_block_number} to {top_block_number}")
@ -118,7 +117,7 @@ def ethcrawler_blocks_add_handler(args: argparse.Namespace) -> None:
print(f"Adding blocks {blocks_numbers_list[-1]}-{blocks_numbers_list[0]}")
crawl_blocks_executor(
block_numbers_list=blocks_numbers_list,
with_transactions=bool(strtobool(args.transactions)),
with_transactions=not args.notransactions,
)
print(f"Required {time.time() - startTime} with {MOONSTREAM_CRAWL_WORKERS} workers")
@ -134,24 +133,26 @@ def ethcrawler_blocks_missing_handler(args: argparse.Namespace) -> None:
missing_blocks_numbers = check_missing_blocks(
blocks_numbers=blocks_numbers_list,
)
if len(missing_blocks_numbers) > 0:
print(f"Found {len(missing_blocks_numbers)} missing blocks")
missing_blocks_numbers_total.extend(missing_blocks_numbers)
print(f"Found {len(missing_blocks_numbers_total)} missing blocks")
print(f"Found {len(missing_blocks_numbers_total)} missing blocks total")
time.sleep(5)
if (len(missing_blocks_numbers_total)) > 0:
if bool(strtobool(args.lazy)):
if args.lazy:
print("Executed lazy block crawler")
crawl_blocks(
missing_blocks_numbers_total,
with_transactions=bool(strtobool(args.transactions)),
verbose=True,
with_transactions=not args.notransactions,
verbose=args.verbose,
)
else:
crawl_blocks_executor(
missing_blocks_numbers_total,
with_transactions=bool(strtobool(args.transactions)),
verbose=True,
with_transactions=not args.notransactions,
verbose=args.verbose,
)
print(
f"Required {time.time() - startTime} with {MOONSTREAM_CRAWL_WORKERS} workers "
@ -205,11 +206,10 @@ def main() -> None:
"synchronize", description="Synchronize to latest ethereum block commands"
)
parser_ethcrawler_blocks_sync.add_argument(
"-t",
"--transactions",
choices=["True", "False"],
default="True",
help="Add or not block transactions",
"-n",
"--notransactions",
action="store_true",
help="Skip crawling block transactions",
)
parser_ethcrawler_blocks_sync.add_argument(
"-s",
@ -246,11 +246,10 @@ def main() -> None:
help="List of blocks range in format {bottom_block}-{top_block}",
)
parser_ethcrawler_blocks_add.add_argument(
"-t",
"--transactions",
choices=["True", "False"],
default="True",
help="Add or not block transactions",
"-n",
"--notransactions",
action="store_true",
help="Skip crawling block transactions",
)
parser_ethcrawler_blocks_add.set_defaults(func=ethcrawler_blocks_add_handler)
@ -264,19 +263,23 @@ def main() -> None:
help="List of blocks range in format {bottom_block}-{top_block}",
)
parser_ethcrawler_blocks_missing.add_argument(
"-t",
"--transactions",
choices=["True", "False"],
default="True",
help="Add or not block transactions",
"-n",
"--notransactions",
action="store_true",
help="Skip crawling block transactions",
)
parser_ethcrawler_blocks_missing.add_argument(
"-l",
"--lazy",
choices=["True", "False"],
default="False",
action="store_true",
help="Lazy block adding one by one",
)
parser_ethcrawler_blocks_missing.add_argument(
"-v",
"--verbose",
action="store_true",
help="Print additional information",
)
parser_ethcrawler_blocks_missing.set_defaults(
func=ethcrawler_blocks_missing_handler
)

24
crawlers/moonstreamcrawlers/ethereum.py
Wyświetl plik

@ -119,16 +119,19 @@ def check_missing_blocks(blocks_numbers: List[int]) -> List[int]:
Query block from postgres. If block does not presented in database,
add to missing blocks numbers list.
"""
missing_blocks_numbers = []
for block_number in blocks_numbers:
with yield_db_session_ctx() as db_session:
block_exist = (
db_session.query(EthereumBlock.block_number)
.filter(EthereumBlock.block_number == block_number)
.one_or_none()
)
if block_exist is None:
missing_blocks_numbers.append(block_number)
bottom_block = min(blocks_numbers[-1], blocks_numbers[0])
top_block = max(blocks_numbers[-1], blocks_numbers[0])
with yield_db_session_ctx() as db_session:
blocks_exist_raw = (
db_session.query(EthereumBlock.block_number)
.filter(EthereumBlock.block_number >= bottom_block)
.filter(EthereumBlock.block_number <= top_block)
.all()
)
blocks_exist = [block[0] for block in blocks_exist_raw]
missing_blocks_numbers = [
block for block in blocks_numbers if block not in blocks_exist
]
return missing_blocks_numbers
@ -177,6 +180,7 @@ def crawl_blocks_executor(
for error in errors:
print(f"- {error}")
def process_contract_deployments() -> List[Tuple[str, str]]:
"""
Checks for new smart contracts that have been deployed to the blockchain but not registered in

2
crawlers/setup.py
Wyświetl plik

@ -33,7 +33,7 @@ setup(
package_data={"moonstreamcrawlers": ["py.typed"]},
zip_safe=False,
install_requires=[
"moonstreamdb @ git+https://git@github.com/bugout-dev/moonstream.git@03a929568180d7eb53ea46a11f920db65ea7c772#egg=moonstreamdb&subdirectory=db",
"moonstreamdb @ git+https://git@github.com/bugout-dev/moonstream.git@ec3278e192119d1e8a273cfaab6cb53890d2e8e9#egg=moonstreamdb&subdirectory=db",
"requests",
"tqdm",
"web3",

BIN
db/requirements.txt
Wyświetl plik

Plik binarny nie jest wyświetlany.

1
frontend/pages/index.js
Wyświetl plik

@ -123,7 +123,6 @@ const Homepage = () => {
}, [isInit, router]);
useLayoutEffect(() => {
console.log("rerender check");
const imageLoader720 = new Image();
imageLoader720.src = `${AWS_PATH}/background720.png`;
imageLoader720.onload = () => {

32
frontend/src/components/EntriesNavigation.js
Wyświetl plik

@ -31,6 +31,7 @@ import {
TagLabel,
TagCloseButton,
Spacer,
useBoolean,
} from "@chakra-ui/react";
import { useSubscriptions } from "../core/hooks";
import StreamEntry from "./StreamEntry";
@ -38,6 +39,7 @@ import UIContext from "../core/providers/UIProvider/context";
import { FaFilter } from "react-icons/fa";
import useStream from "../core/hooks/useStream";
import { ImCancelCircle } from "react-icons/im";
import { IoStopCircleOutline, IoPlayCircleOutline } from "react-icons/io5";
const pageSize = 25;
const FILTER_TYPES = {
@ -61,6 +63,7 @@ const CONDITION = {
const EntriesNavigation = () => {
const ui = useContext(UIContext);
const [isStreamOn, setStreamState] = useBoolean(true);
const { isOpen, onOpen, onClose } = useDisclosure();
const { subscriptionsCache } = useSubscriptions();
const [newFilterState, setNewFilterState] = useState([
@ -80,7 +83,7 @@ const EntriesNavigation = () => {
pageSize,
refreshRate: 1500,
searchQuery: ui.searchTerm,
enabled: true,
enabled: isStreamOn,
isContent: false,
});
@ -110,7 +113,7 @@ const EntriesNavigation = () => {
newFilterState[0].value === null
) {
setFilterProps(0, {
value: subscriptionsCache.data.subscriptions[0].address,
value: subscriptionsCache?.data?.subscriptions[0]?.address,
});
}
}, [subscriptionsCache, newFilterState, setFilterProps]);
@ -253,7 +256,7 @@ const EntriesNavigation = () => {
onChange={handleAddressChange(idx)}
>
{!subscriptionsCache.isLoading &&
subscriptionsCache.data.subscriptions.map(
subscriptionsCache?.data?.subscriptions.map(
(subscription, idx) => {
return (
<option
@ -318,7 +321,8 @@ const EntriesNavigation = () => {
direction: DIRECTIONS.SOURCE,
condition: CONDITION.EQUAL,
value:
subscriptionsCache.data.subscriptions[0].address,
subscriptionsCache?.data?.subscriptions[0]
?.address,
},
])
}
@ -334,7 +338,8 @@ const EntriesNavigation = () => {
direction: DIRECTIONS.DESTINATION,
condition: CONDITION.EQUAL,
value:
subscriptionsCache.data.subscriptions[0].address,
subscriptionsCache?.data?.subscriptions[0]
?.address,
},
])
}
@ -356,8 +361,22 @@ const EntriesNavigation = () => {
</DrawerFooter>
</DrawerContent>
</Drawer>
<Flex h="3rem" w="100%" bgColor="gray.200" alignItems="center">
<Flex h="3rem" w="100%" bgColor="gray.100" alignItems="center">
<Flex maxW="90%">
<Flex direction="column">
<IconButton
size="sm"
onClick={() => setStreamState.toggle()}
icon={
isStreamOn ? (
<IoStopCircleOutline size="32px" />
) : (
<IoPlayCircleOutline size="32px" />
)
}
colorScheme={isStreamOn ? "unsafe" : "suggested"}
/>
</Flex>
{filterState.map((filter, idx) => {
if (filter.type === FILTER_TYPES.DISABLED) return "";
return (
@ -407,7 +426,6 @@ const EntriesNavigation = () => {
id="StreamEntry"
overflowY="scroll"
direction="column"
w="100%"
onScroll={(e) => handleScroll(e)}
>

8
frontend/src/components/Footer.js
Wyświetl plik

@ -8,11 +8,7 @@ const ICONS = [
link: "https://discord.gg/FetK5BxD",
},
{ social: "twit", link: "https://twitter.com/Bugout_dev" },
{
social: "slack",
link: "https://join.slack.com/t/bugout-dev/shared_invite/zt-fhepyt87-5XcJLy0iu702SO_hMFKNhQ",
},
{ social: "twit", link: "https://twitter.com/moonstreamto" },
];
const SITEMAP_FLEX_PROPS = {
@ -98,7 +94,7 @@ const Footer = () => (
fontSize="xl"
fontWeight="500"
>
Stay in touch
Stay in touch{` `}
<span role="img" aria-label="heart">
💙
</span>

13
frontend/src/components/LandingNavbar.js
Wyświetl plik

@ -87,10 +87,21 @@ const LandingNavbar = () => {
Log in
</Button>
)}
{ui.isLoggedIn && <ChakraAccountIconButton />}
{ui.isLoggedIn && (
<ChakraAccountIconButton
variant="link"
colorScheme="secondary"
/>
)}
</ButtonGroup>
</>
)}
{ui.isLoggedIn && ui.isMobileView && (
<>
<Spacer />
<ChakraAccountIconButton variant="link" colorScheme="secondary" />
</>
)}
</>
</>
);

1
frontend/src/components/Scrollable.js
Wyświetl plik

@ -19,7 +19,6 @@ const Scrollable = (props) => {
const handleScroll = (e) => {
const currentScroll = Math.ceil(getScrollPrecent(e) / 10);
if (currentScroll > scrollDepth) {
// withTracking(
setScrollDepth(currentScroll);
isLoaded &&
mixpanel.people.increment({

6
frontend/src/components/SignIn.js
Wyświetl plik

@ -38,7 +38,7 @@ const SignIn = ({ toggleModal }) => {
Login now
</Heading>
<Text color="gray.1200" fontSize="md">
To your Bugout account
To your Moonstream account
</Text>
<form onSubmit={handleSubmit(login)}>
<Stack width="100%" pt={4} spacing={3}>
@ -49,7 +49,7 @@ const SignIn = ({ toggleModal }) => {
autoComplete="username"
variant="filled"
colorScheme="primary"
placeholder="Your Bugout username"
placeholder="Your Moonstream username"
name="username"
{...register("username", { required: true })}
ref={register({ required: "Username is required!" })}
@ -64,7 +64,7 @@ const SignIn = ({ toggleModal }) => {
</FormControl>
<FormControl isInvalid={errors.password}>
<PasswordInput
placeholder="Your Bugout password"
placeholder="Your Moonstream password"
name="password"
ref={register({ required: "Password is required!" })}
/>

1
frontend/src/components/StreamEntry.js
Wyświetl plik

@ -56,7 +56,6 @@ const StreamEntry = ({ entry, filterCallback, filterConstants }) => {
transition="0.1s"
_hover={{ bg: "secondary.200" }}
flexBasis="50px"
flexGrow={1}
direction="row"
justifySelf="center"
justifyContent="normal"