mirror
/
moonstream
kopia lustrzana https://github.com/bugout-dev/moonstream
1
0
Forkuj 0
Kod Zgłoszenia Projekty Wydania Wiki Aktywność

Merge branch 'main' into cu-state-crawler

pull/659/head
Andrey 2022-08-30 17:38:47 +03:00
rodzic 54870630c4 e1477643b9
commit 7fe587fdbc
20 zmienionych plików z 111 dodań i 22335 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

6
crawlers/mooncrawl/mooncrawl/moonworm_crawler/crawler.py
Wyświetl plik

@ -16,6 +16,7 @@ from web3.main import Web3
from ..blockchain import connect
from ..reporter import reporter
from ..settings import (
BUGOUT_REQUEST_TIMEOUT_SECONDS,
MOONSTREAM_ADMIN_ACCESS_TOKEN,
MOONSTREAM_MOONWORM_TASKS_JOURNAL,
bugout_client,
@ -186,6 +187,7 @@ def get_crawl_job_entries(
query=query,
offset=current_offset,
limit=limit,
timeout=BUGOUT_REQUEST_TIMEOUT_SECONDS,
)
entries.extend(search_result.results)
@ -337,6 +339,7 @@ def _get_heartbeat_entry_id(
journal_id=MOONSTREAM_MOONWORM_TASKS_JOURNAL,
query=f"#{crawler_type} #heartbeat #{blockchain_type.value} !#dead",
limit=1,
timeout=BUGOUT_REQUEST_TIMEOUT_SECONDS,
)
if entries.results:
return entries.results[0].entry_url.split("/")[-1]
@ -348,6 +351,7 @@ def _get_heartbeat_entry_id(
title=f"{crawler_type} Heartbeat - {blockchain_type.value}",
tags=[crawler_type, "heartbeat", blockchain_type.value],
content="",
timeout=BUGOUT_REQUEST_TIMEOUT_SECONDS,
)
return str(entry.id)
@ -376,6 +380,7 @@ def heartbeat(
entry_id=heartbeat_entry_id,
title=f"{crawler_type} Heartbeat - {blockchain_type.value}. Status: {crawler_status['status']} - {crawler_status['current_time']}",
content=f"{json.dumps(crawler_status, indent=2)}",
timeout=BUGOUT_REQUEST_TIMEOUT_SECONDS,
)
if is_dead:
bugout_client.update_tags(
@ -383,4 +388,5 @@ def heartbeat(
journal_id=MOONSTREAM_MOONWORM_TASKS_JOURNAL,
entry_id=heartbeat_entry_id,
tags=[crawler_type, "heartbeat", blockchain_type.value, "dead"],
timeout=BUGOUT_REQUEST_TIMEOUT_SECONDS,
)

27
crawlers/mooncrawl/mooncrawl/settings.py
Wyświetl plik

@ -13,6 +13,17 @@ BUGOUT_BROOD_URL = os.environ.get("BUGOUT_BROOD_URL", "https://auth.bugout.dev")
BUGOUT_SPIRE_URL = os.environ.get("BUGOUT_SPIRE_URL", "https://spire.bugout.dev")
bugout_client = Bugout(brood_api_url=BUGOUT_BROOD_URL, spire_api_url=BUGOUT_SPIRE_URL)
BUGOUT_REQUEST_TIMEOUT_SECONDS_RAW = os.environ.get(
"MOONSTREAM_BUGOUT_TIMEOUT_SECONDS", 30
)
try:
BUGOUT_REQUEST_TIMEOUT_SECONDS = int(BUGOUT_REQUEST_TIMEOUT_SECONDS_RAW)
except:
raise Exception(
f"Could not parse MOONSTREAM_BUGOUT_TIMEOUT_SECONDS_RAW as int: {BUGOUT_REQUEST_TIMEOUT_SECONDS_RAW}"
)
HUMBUG_REPORTER_CRAWLERS_TOKEN = os.environ.get("HUMBUG_REPORTER_CRAWLERS_TOKEN")
# Origin
@ -114,6 +125,22 @@ if MOONSTREAM_MOONWORM_TASKS_JOURNAL == "":
"MOONSTREAM_MOONWORM_TASKS_JOURNAL environment variable must be set"
)
# queries
MOONSTREAM_QUERY_API_DB_STATEMENT_TIMEOUT_MILLIS = 30000
MOONSTREAM_QUERY_API_DB_STATEMENT_TIMEOUT_MILLIS_RAW = os.environ.get(
"MOONSTREAM_QUERY_API_DB_STATEMENT_TIMEOUT_MILLIS"
)
try:
if MOONSTREAM_QUERY_API_DB_STATEMENT_TIMEOUT_MILLIS_RAW is not None:
MOONSTREAM_QUERY_API_DB_STATEMENT_TIMEOUT_MILLIS = int(
MOONSTREAM_QUERY_API_DB_STATEMENT_TIMEOUT_MILLIS_RAW
)
except:
raise Exception(
f"Could not parse MOONSTREAM_QUERY_API_DB_STATEMENT_TIMEOUT_MILLIS as int: {MOONSTREAM_CRAWL_WORKERS_RAW}"
)
MOONSTREAM_S3_QUERIES_BUCKET = os.environ.get("MOONSTREAM_S3_QUERIES_BUCKET", "")
if MOONSTREAM_S3_QUERIES_BUCKET == "":

58
crawlers/mooncrawl/mooncrawl/stats_worker/queries.py
Wyświetl plik

@ -6,9 +6,18 @@ from io import StringIO
from typing import Any, Dict, Optional
import boto3 # type: ignore
from moonstreamdb.db import yield_db_read_only_session_ctx
from moonstreamdb.db import (
create_moonstream_engine,
MOONSTREAM_DB_URI_READ_ONLY,
MOONSTREAM_POOL_SIZE,
)
from sqlalchemy.orm import sessionmaker
from ..reporter import reporter
from ..settings import MOONSTREAM_S3_QUERIES_BUCKET_PREFIX
from ..settings import (
MOONSTREAM_S3_QUERIES_BUCKET_PREFIX,
MOONSTREAM_QUERY_API_DB_STATEMENT_TIMEOUT_MILLIS,
)
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
@ -45,6 +54,16 @@ def query_validation(query: str) -> str:
return query
def to_json_types(value):
if isinstance(value, (str, int, tuple, list, dict)):
return value
elif isinstance(value, set):
return list(value)
else:
return str(value)
def data_generate(
bucket: str,
query_id: str,
@ -57,13 +76,17 @@ def data_generate(
"""
s3 = boto3.client("s3")
with yield_db_read_only_session_ctx() as db_session:
try:
db_session.execute("SELECT 1")
except Exception as e:
db_session.rollback()
# Create session
engine = create_moonstream_engine(
MOONSTREAM_DB_URI_READ_ONLY,
pool_pre_ping=True,
pool_size=MOONSTREAM_POOL_SIZE,
statement_timeout=MOONSTREAM_QUERY_API_DB_STATEMENT_TIMEOUT_MILLIS,
)
process_session = sessionmaker(bind=engine)
db_session = process_session()
try:
if file_type == "csv":
csv_buffer = StringIO()
csv_writer = csv.writer(csv_buffer, delimiter=";")
@ -90,12 +113,7 @@ def data_generate(
"block_number": block_number,
"block_timestamp": block_timestamp,
"data": [
{
key: (
value if type(value) is any((int, str)) else str(value)
)
for key, value in dict(row).items()
}
{key: to_json_types(value) for key, value in dict(row).items()}
for row in db_session.execute(query, params)
],
}
@ -106,3 +124,15 @@ def data_generate(
key=f"{MOONSTREAM_S3_QUERIES_BUCKET_PREFIX}/queries/{query_id}/data.{file_type}",
bucket=bucket,
)
except Exception as err:
db_session.rollback()
reporter.error_report(
err,
[
"queries",
"execution",
f"query_id:{query_id}" f"file_type:{file_type}",
],
)
finally:
db_session.close()

2
crawlers/mooncrawl/mooncrawl/version.py
Wyświetl plik

@ -2,4 +2,4 @@
Moonstream crawlers version.
"""
MOONCRAWL_VERSION = "0.2.1"
MOONCRAWL_VERSION = "0.2.2"

1
crawlers/mooncrawl/sample.env
Wyświetl plik

@ -4,6 +4,7 @@ export BUGOUT_SPIRE_URL="https://spire.bugout.dev"
export HUMBUG_REPORTER_CRAWLERS_TOKEN="<Bugout_Humbug_token_for_crash_reports>"
# Moonstream environment variables
export MOONSTREAM_BUGOUT_TIMEOUT_SECONDS=30
export MOONSTREAM_CORS_ALLOWED_ORIGINS="http://localhost:3000,https://moonstream.to,https://www.moonstream.to"
export MOONSTREAM_DB_URI="postgresql://<username>:<password>@<db_host>:<db_port>/<db_name>"
export MOONSTREAM_DB_URI_READ_ONLY="postgresql://<username>:<password>@<db_host>:<db_port>/<db_name>"

2
crawlers/mooncrawl/setup.py
Wyświetl plik

@ -37,7 +37,7 @@ setup(
"bugout>=0.1.19",
"chardet",
"fastapi",
"moonstreamdb>=0.3.1",
"moonstreamdb>=0.3.2",
"moonworm==0.2.4",
"humbug",
"pydantic",

5
db/moonstreamdb/db.py
Wyświetl plik

@ -39,11 +39,14 @@ except:
)
def create_moonstream_engine(url: str, pool_size: int, statement_timeout: int):
def create_moonstream_engine(
url: str, pool_size: int, statement_timeout: int, pool_pre_ping: bool = False
):
# Pooling: https://docs.sqlalchemy.org/en/14/core/pooling.html#sqlalchemy.pool.QueuePool
# Statement timeout: https://stackoverflow.com/a/44936982
return create_engine(
url=url,
pool_pre_ping=pool_pre_ping,
pool_size=pool_size,
connect_args={"options": f"-c statement_timeout={statement_timeout}"},
)

2
db/moonstreamdb/version.py
Wyświetl plik

@ -2,4 +2,4 @@
Moonstream database version.
"""
MOONSTREAMDB_VERSION = "0.3.1"
MOONSTREAMDB_VERSION = "0.3.2"

2
db/scm.nix
Wyświetl plik

@ -14,7 +14,7 @@ let
"."
(getEnv "MDP_GIT")
(fetchGit {
url = "git@gitlab.com:mixrank/mdp.git";
url = "git@scm.mixrank.com:mixrank/mdp.git";
rev = "309e04f3f646847af3b7c084b01cfd72e1db92c8";
})
] ++ scm_repos;

2
frontend/.gitignore vendored
Wyświetl plik

@ -35,6 +35,8 @@ dev.env.local
.env.test.local
.env.production.local
.secrets/
yarn.lock
package-lock.json
# vercel
.vercel

16112
frontend/package-lock.json wygenerowano
Wyświetl plik

Plik diff jest za duży Load Diff

4
frontend/pages/_app.js
Wyświetl plik

@ -19,7 +19,7 @@ const DefaultLayout = dynamic(() => import("../src/layouts"), {
});
import { useRouter } from "next/router";
import NProgress from "nprogress";
import { WHITE_LOGO_W_TEXT_URL } from "../src/core/constants";
import { PRIMARY_MOON_LOGO_URL } from "../src/core/constants";
export default function CachingApp({ Component, pageProps }) {
const [queryClient] = useState(new QueryClient());
@ -58,7 +58,7 @@ export default function CachingApp({ Component, pageProps }) {
Component.getLayout || ((page) => <DefaultLayout>{page}</DefaultLayout>);
const headLinks = [
{ rel: "preload", as: "image", href: WHITE_LOGO_W_TEXT_URL },
{ rel: "preload", as: "image", href: PRIMARY_MOON_LOGO_URL },
];
pageProps.preloads && headLinks.push(...pageProps.preloads);
return (

9
frontend/pages/index.js
Wyświetl plik

@ -239,7 +239,7 @@ const Homepage = () => {
boxSizing="content-box"
>
<GridItem colSpan="12" bgColor={"blue.50"} id="Header grid item">
<chakra.header boxSize="full" minH="100vh" mb={0}>
<chakra.header boxSize="full" minH={["60vh", "100vh"]} mb={0}>
<Box
bgPos="bottom"
bgColor="transparent"
@ -271,7 +271,7 @@ const Homepage = () => {
pb={[8, 8, 12]}
maxW={[null, "90%", "80%", "58%"]}
>
Build, Scale, and Monitor Your Game on the Blockchain
{DEFAULT_METATAGS.title}
</Heading>
<chakra.span
pb={[6, 6, 12]}
@ -280,10 +280,7 @@ const Homepage = () => {
color="white"
maxW={[null, "85%", "75%"]}
>
Moonstream is a no code web3 game engine. Use
Moonstream’s technical blockchain infrastructure to
add on-chain game mechanics. Watch your game’s economy
flourish.
{DEFAULT_METATAGS.description}
</chakra.span>
<Stack
direction={[

BIN
frontend/public/favicon.png
Wyświetl plik

Plik binarny nie jest wyświetlany.
Side by Side Swipe Nakładka

Przed

Szerokość:  |  Wysokość:  |  Rozmiar: 2.7 KiB

Po

Szerokość:  |  Wysokość:  |  Rozmiar: 797 B

4
frontend/src/components/AppNavbar.js
Wyświetl plik

@ -38,7 +38,7 @@ import AddNewIconButton from "./AddNewIconButton";
import {
USER_NAV_PATHES,
SITEMAP,
WHITE_LOGO_W_TEXT_URL,
PRIMARY_MOON_LOGO_URL,
PAGETYPE,
} from "../core/constants";
@ -244,7 +244,7 @@ const AppNavbar = () => {
<Image
alignSelf="center"
maxH="2.5rem"
src={WHITE_LOGO_W_TEXT_URL}
src={PRIMARY_MOON_LOGO_URL}
alt="Go to app root"
/>
</Link>

4
frontend/src/components/Footer.js
Wyświetl plik

@ -12,7 +12,7 @@ import {
chakra,
} from "@chakra-ui/react";
import RouterLink from "next/link";
import { WHITE_LOGO_W_TEXT_URL, SITEMAP } from "../core/constants";
import { PRIMARY_MOON_LOGO_URL, SITEMAP } from "../core/constants";
import { FaGithub, FaTwitter, FaDiscord } from "react-icons/fa";
import moment from "moment";
@ -79,7 +79,7 @@ const Footer = () => (
// to="/"
h="2.5rem"
minW="2.5rem"
src={WHITE_LOGO_W_TEXT_URL}
src={PRIMARY_MOON_LOGO_URL}
alt="Go to app root"
/>
</Link>

9
frontend/src/components/LandingNavbar.js
Wyświetl plik

@ -19,7 +19,7 @@ import useModals from "../core/hooks/useModals";
import UIContext from "../core/providers/UIProvider/context";
import ChakraAccountIconButton from "./AccountIconButton";
import RouteButton from "./RouteButton";
import { PAGETYPE, SITEMAP, WHITE_LOGO_W_TEXT_URL } from "../core/constants";
import { PAGETYPE, SITEMAP, PRIMARY_MOON_LOGO_URL } from "../core/constants";
import router from "next/router";
import { MODAL_TYPES } from "../core/providers/OverlayProvider/constants";
@ -40,21 +40,22 @@ const LandingNavbar = () => {
</>
)}
<Flex
pl={ui.isMobileView ? 2 : 8}
pl={ui.isMobileView ? 2 : 16}
justifySelf="flex-start"
h="48px"
py={1}
flexBasis="200px"
flexGrow={1}
id="Logo Container"
alignItems="center"
>
<RouterLink href="/" passHref>
<Link
as={Image}
w="auto"
h="100%"
h={["70%", "85%"]}
justifyContent="left"
src={WHITE_LOGO_W_TEXT_URL}
src={PRIMARY_MOON_LOGO_URL}
alt="Moonstream logo"
/>
</RouterLink>

10
frontend/src/components/Sidebar.js
Wyświetl plik

@ -25,7 +25,7 @@ import {
LockIcon,
} from "@chakra-ui/icons";
import { MdSettings, MdDashboard, MdTimeline } from "react-icons/md";
import { WHITE_LOGO_W_TEXT_URL, SITEMAP, PAGETYPE } from "../core/constants";
import { PRIMARY_MOON_LOGO_URL, SITEMAP, PAGETYPE } from "../core/constants";
import useDashboard from "../core/hooks/useDashboard";
import { MODAL_TYPES } from "../core/providers/OverlayProvider/constants";
import OverlayContext from "../core/providers/OverlayProvider/context";
@ -68,12 +68,10 @@ const Sidebar = () => {
/>
<RouterLink href="/" passHref>
<Image
// h="full"
// maxH="100%"
maxW="120px"
maxW="155px"
py="0.75rem"
pl={5}
src={WHITE_LOGO_W_TEXT_URL}
pl={1}
src={PRIMARY_MOON_LOGO_URL}
alt="Moonstream To"
/>
</RouterLink>

10
frontend/src/core/constants.js
Wyświetl plik

@ -1,5 +1,8 @@
export const MOONSTREAM_API_URL = process.env.NEXT_PUBLIC_MOONSTREAM_API_URL;
export const AWS_ASSETS_PATH = `https://s3.amazonaws.com/static.simiotics.com/moonstream/assets`;
export const PRIMARY_MOON_LOGO_URL = `${AWS_ASSETS_PATH}/moonstream-full-logo-2022.png`;
export const BUGOUT_ENDPOINTS = {
Usage: "usage",
Web: "parasite",
@ -8,11 +11,11 @@ export const BUGOUT_ENDPOINTS = {
export const DEFAULT_METATAGS = {
title: "Build, Scale, and Monitor Your Game on the Blockchain",
description:
"Moonstream is a no code web3 game engine. Use Moonstream’s technical blockchain infrastructure to add on-chain game mechanics. Watch your game’s economy flourish.",
"Moonstream is a web3 game engine. Use Moonstream’s technical blockchain infrastructure to add on-chain mechanics to your game. Watch your game economy flourish.",
keywords:
"analytics, blockchain analytics, protocol, protocols, blockchain, crypto, data, NFT gaming, smart contracts, web3, smart contract, ethereum, polygon, matic, transactions, defi, finance, decentralized, mempool, NFT, NFTs, DAO, DAOs, cryptocurrency, cryptocurrencies, bitcoin, blockchain economy, blockchain game, marketplace, blockchain security, loyalty program, Ethereum bridge, Ethereum bridges, NFT game, NFT games",
url: "https://www.moonstream.to",
image: `https://s3.amazonaws.com/static.simiotics.com/moonstream/assets/crypto+traders.png`,
image: `${AWS_ASSETS_PATH}/metadata-image.png`,
};
// export const FOOTER_COLUMNS = {
@ -123,9 +126,6 @@ export const USER_NAV_PATHES = [
export const PAGE_SIZE = 20;
export const AWS_ASSETS_PATH = `https://s3.amazonaws.com/static.simiotics.com/moonstream/assets`;
export const WHITE_LOGO_W_TEXT_URL = `https://s3.amazonaws.com/static.simiotics.com/moonstream/assets/moon-logo%2Btext-white.png`;
export const TIME_RANGE_SECONDS = {
day: 86400,
week: 86400 * 7,

6177
frontend/yarn.lock
Wyświetl plik

Plik diff jest za duży Load Diff