mirror
/
moonstream
kopia lustrzana https://github.com/bugout-dev/moonstream
1
0
Forkuj 0
Kod Zgłoszenia Projekty Wydania Wiki Aktywność

Merge branch 'main' into add-events-provider

pull/475/head
Andrey Dolgolev 2021-12-07 18:16:36 +02:00
rodzic a6ad24262f 74ed0ceac2
commit cad752d68e
30 zmienionych plików z 1023 dodań i 123 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

22
README.md
Wyświetl plik

@ -54,6 +54,28 @@ This monorepo contains the following components:
[Bugout](https://bugout.dev). For more information on how that data is processed, check how the API
inserts events from those sources into a stream.
### Installation and setup
#### Run server with Docker Compose
If you want to deploy Moonstream in isolation against live services, then docker compose is your choice!
- Run script `backend/configs/docker_generate_env.bash` which prepare for you:
- `backend/configs/docker.moonstreamapi.env` with environment variables
- Run script `db/configs/docker_generate_env.bash` which prepare for you:
- `db/configs/alembic.moonstreamdb.ini` with postgresql uri
```bash
./backend/configs/docker_generate_env.bash
./db/configs/docker_generate_env.bash
```
- Run local setup
```bash
docker-compose up --build
```
## Contributing
If you would like to contribute to Moonstream, please reach out to @zomglings on the [Moonstream Discord](https://discord.gg/pYE65FuNSz).

220
backend/.dockerignore 100644
Wyświetl plik

@ -0,0 +1,220 @@
# Created by https://www.toptal.com/developers/gitignore/api/vim,visualstudiocode,linux,python
# Edit at https://www.toptal.com/developers/gitignore?templates=vim,visualstudiocode,linux,python
### Linux ###
*~
# temporary files which can be created if a process still has a handle open of a deleted file
.fuse_hidden*
# KDE directory preferences
.directory
# Linux trash folder which might appear on any partition or disk
.Trash-*
# .nfs files are created when an open file is removed but is still being accessed
.nfs*
### Python ###
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
pip-wheel-metadata/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
*.py,cover
.hypothesis/
.pytest_cache/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
db.sqlite3
db.sqlite3-journal
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
target/
# Jupyter Notebook
.ipynb_checkpoints
# IPython
profile_default/
ipython_config.py
# pyenv
.python-version
# pipenv
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
# However, in case of collaboration, if having platform-specific dependencies or dependencies
# having no cross-platform support, pipenv may install dependencies that don't work, or not
# install all needed dependencies.
#Pipfile.lock
# PEP 582; used by e.g. github.com/David-OConnor/pyflow
__pypackages__/
# Celery stuff
celerybeat-schedule
celerybeat.pid
# SageMath parsed files
*.sage.py
# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/
.dmypy.json
dmypy.json
# Pyre type checker
.pyre/
# pytype static type analyzer
.pytype/
### Vim ###
# Swap
[._]*.s[a-v][a-z]
!*.svg # comment out if you don't need vector files
[._]*.sw[a-p]
[._]s[a-rt-v][a-z]
[._]ss[a-gi-z]
[._]sw[a-p]
# Session
Session.vim
Sessionx.vim
# Temporary
.netrwhist
# Auto-generated tag files
tags
# Persistent undo
[._]*.un~
### VisualStudioCode ###
.vscode/*
!.vscode/settings.json
!.vscode/tasks.json
!.vscode/launch.json
!.vscode/extensions.json
*.code-workspace
### Pycharm ###
.idea
### VisualStudioCode Patch ###
# Ignore all local history of files
.history
# End of https://www.toptal.com/developers/gitignore/api/vim,visualstudiocode,linux,python
### VSCode ###
.vscode
# Git
.git
.github
# Envionments
.moonstream
.moonstreamapi
.venv
.env
# Environment variables
test.env
dev.env
prod.env
configs/test.env
configs/dev.env
configs/prod.env
configs/moonstreamapi.env
configs/docker.env
configs/docker.dev.env
configs/docker.test.env
configs/docker.prod.env
.secrets/
Dockerfile
docker-compose.yml

20
backend/.gitignore vendored
Wyświetl plik

@ -160,9 +160,21 @@ cython_debug/
# End of https://www.toptal.com/developers/gitignore/api/python,visualstudiocode
# Custom
# Envionments
.moonstream/
.moonstreamapi/
.venv/
.env/
# Environment variables
dev.env
test.env
prod.env
.moonstream
.venv
.secrets
moonstreamapi.env
docker.env
docker.dev.env
docker.test.dev
docker.prod.env
docker.moonstreamapi.env
.secrets/

20
backend/Dockerfile 100644
Wyświetl plik

@ -0,0 +1,20 @@
FROM python:3.8-slim-buster
# Update packages and
# prepare alembic for docker compose setup
RUN apt-get update && \
apt-get install -y libpq-dev gcc curl && \
rm -rf /var/lib/apt/lists/* && \
pip3 install --no-cache-dir --upgrade pip setuptools && \
pip3 install --no-cache-dir psycopg2-binary alembic
WORKDIR /usr/src/moonstreamapi
COPY . /usr/src/moonstreamapi
# Install Moonstream API package
RUN pip3 install --no-cache-dir -e .
EXPOSE 7481
ENTRYPOINT ["./dev.sh"]

42
backend/README.md
Wyświetl plik

@ -1 +1,43 @@
# moonstream backend
### Installation and setup
To set up Moonstream API for development, do the following:
- Clone the git repository
- Install postgresql (https://www.postgresql.org/download/linux/ubuntu/)
#### Run server with Docker
To be able to run Moonstream API with your existing local or development services as database, you need to build your own setup. **Be aware! The files with environment variables `docker.dev.env` lives inside your docker container!**
- Copy `configs/sample.env` to `configs/docker.dev.env`, or use your local configs from `configs/dev.env` to `configs/docker.dev.env`
- Edit in `docker.dev.env` file `MOONSTREAM_DB_URI` and other variables if required
- Clean environment file from `export ` prefix and quotation marks to be able to use it with Docker
```bash
sed --in-place 's|^export * ||' configs/docker.dev.env
sed --in-place 's|"||g' configs/docker.dev.env
```
Build container on your machine
```bash
docker build -t moonstreamapi-dev .
```
Run `moonstreamapi-dev` container, with following command we specified `--network="host"` setting which allows to Docker container use localhost interface of your machine (https://docs.docker.com/network/host/)
```bash
docker run --name moonstreamapi-dev \
--network="host" \
--env-file="configs/docker.dev.env" \
-p 7481:7481/tcp \
-ti -d moonstreamapi-dev
```
Attach to container to see logs
```bash
docker container attach moonstreamapi-dev
```

46
backend/configs/docker_generate_env.bash 100755
Wyświetl plik

@ -0,0 +1,46 @@
#!/usr/bin/env bash
# Prepare Moonstream API application for docker-compose use
# Print help message
function usage {
echo "Usage: $0 [-h] -d DATABASE_NAME"
echo
echo "CLI to generate environment variables"
echo
echo "Optional arguments:"
echo " -h Show this help message and exit"
echo " -d Database name for postgres in docker-compose setup"
}
FLAG_DATABASE_NAME="moonstream_dev"
while getopts 'd:' flag; do
case "${flag}" in
d) FLAG_DATABASE_NAME="${OPTARG}" ;;
h) usage
exit 1 ;;
*) usage
exit 1 ;;
esac
done
set -e
SCRIPT_DIR="$(realpath $(dirname $0))"
DOCKER_MOONSTREAM_DB_URI="postgresql://postgres:postgres@db/$FLAG_DATABASE_NAME"
DOCKER_MOONSTREAM_ENV_FILE="docker.moonstreamapi.env"
BUGOUT_BROOD_URL="http://brood:7474"
BUGOUT_SPIRE_URL="http://spire:7475"
# Generate environment variables
cp "$SCRIPT_DIR/sample.env" "$SCRIPT_DIR/$DOCKER_MOONSTREAM_ENV_FILE"
# Clean file with variables from export prefix and quotation marks
sed --in-place 's|^export * ||' "$SCRIPT_DIR/$DOCKER_MOONSTREAM_ENV_FILE"
sed --in-place 's|"||g' "$SCRIPT_DIR/$DOCKER_MOONSTREAM_ENV_FILE"
sed -i "s|^MOONSTREAM_DB_URI=.*|MOONSTREAM_DB_URI=$DOCKER_MOONSTREAM_DB_URI|" "$SCRIPT_DIR/$DOCKER_MOONSTREAM_ENV_FILE"
sed -i "s|^BUGOUT_BROOD_URL=.*|BUGOUT_BROOD_URL=$BUGOUT_BROOD_URL|" "$SCRIPT_DIR/$DOCKER_MOONSTREAM_ENV_FILE"
sed -i "s|^BUGOUT_SPIRE_URL=.*|BUGOUT_SPIRE_URL=$BUGOUT_SPIRE_URL|" "$SCRIPT_DIR/$DOCKER_MOONSTREAM_ENV_FILE"

13
backend/configs/docker_wait.sh 100755
Wyświetl plik

@ -0,0 +1,13 @@
#!/usr/bin/env sh
set -e
HOST="$1"
shift
until curl --request GET --url "http://$HOST/ping"; do
>&2 echo "$HOST is unavailable, sleeping"
sleep 1
done
>&2 echo "$HOST is up"

23
backend/configs/sample.env 100644
Wyświetl plik

@ -0,0 +1,23 @@
# Required environment variables
export MOONSTREAM_DB_URI="postgresql://<username>:<password>@<db_host>:<db_port>/<db_name>"
export MOONSTREAM_CORS_ALLOWED_ORIGINS="http://localhost:3000,https://moonstream.to,https://www.moonstream.to"
export BUGOUT_BROOD_URL="https://auth.bugout.dev"
export BUGOUT_SPIRE_URL="https://spire.bugout.dev"
export MOONSTREAM_APPLICATION_ID="<issued_bugout_application_id>"
export MOONSTREAM_ADMIN_ACCESS_TOKEN="<Access_token_to_application_resources>"
export MOONSTREAM_POOL_SIZE=0
# Blockchain, txpool, whalewatch data depends variables
export MOONSTREAM_DATA_JOURNAL_ID="<bugout_journal_id_to_store_blockchain_data>"
export HUMBUG_TXPOOL_CLIENT_ID="<Bugout_Humbug_client_id_for_txpool_transactions_in_journal>"
export MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI="https://<connection_path_uri_to_ethereum_node>"
export MOONSTREAM_NODE_ETHEREUM_IPC_PORT=8545
# Set following parameters if AWS node instance and S3 smartcontracts configured
export MOONSTREAM_INTERNAL_HOSTED_ZONE_ID="<moonstream_internal_hosted_zone_id>"
export MOONSTREAM_S3_SMARTCONTRACTS_BUCKET="<AWS_S3_bucket_to_store_smart_contracts>"
export MOONSTREAM_S3_SMARTCONTRACTS_ABI_BUCKET="<AWS_S3_bucket_to_store_smart_contracts_ABI>"
export MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX="<Previx_for_AWS_S3_bucket_(prod,dev,..)>"
# Set the following variables in the most reasonable manner for your development environment
export HUMBUG_REPORTER_BACKEND_TOKEN="<Bugout_umbug_token_for_crash_reports>"

18
backend/dev.sh
Wyświetl plik

@ -1,9 +1,19 @@
#!/usr/bin/env sh
# Expects access to Python environment with the requirements for this project installed.
# Sets up Moonstream API server
# Expects access to Python environment with the requirements
# for this project installed.
set -e
MOONSTREAM_HOST="${MOONSTREAM_HOST:-0.0.0.0}"
MOONSTREAM_PORT="${MOONSTREAM_PORT:-7481}"
MOONSTREAMAPI_HOST="${MOONSTREAMAPI_HOST:-127.0.0.1}"
MOONSTREAMAPI_PORT="${MOONSTREAMAPI_PORT:-7481}"
MOONSTREAMAPI_APP_DIR="${MOONSTREAMAPI_APP_DIR:-$PWD}"
MOONSTREAMAPI_ASGI_APP="${MOONSTREAMAPI_ASGI_APP:-moonstreamapi.api:app}"
MOONSTREAMAPI_UVICORN_WORKERS="${MOONSTREAMAPI_UVICORN_WORKERS:-2}"
uvicorn --port "$MOONSTREAM_PORT" --host "$MOONSTREAM_HOST" moonstreamapi.api:app --reload
uvicorn --reload \
--port "$MOONSTREAMAPI_PORT" \
--host "$MOONSTREAMAPI_HOST" \
--app-dir "$MOONSTREAMAPI_APP_DIR" \
--workers "$MOONSTREAMAPI_UVICORN_WORKERS" \
"$MOONSTREAMAPI_ASGI_APP"

4
backend/moonstreamapi/providers/bugout.py
Wyświetl plik

@ -16,7 +16,7 @@ from sqlalchemy.orm import Session
from .. import data
from ..stream_queries import StreamQuery
from ..settings import ETHTXPOOL_HUMBUG_CLIENT_ID
from ..settings import HUMBUG_TXPOOL_CLIENT_ID
logger = logging.getLogger(__name__)
logger.setLevel(logging.WARN)
@ -362,5 +362,5 @@ ethereum_txpool_provider = EthereumTXPoolProvider(
description=ethereum_txpool_description,
default_time_interval_seconds=5,
estimated_events_per_time_interval=50,
tags=[f"client:{ETHTXPOOL_HUMBUG_CLIENT_ID}"],
tags=[f"client:{HUMBUG_TXPOOL_CLIENT_ID}"],
)

8
backend/moonstreamapi/settings.py
Wyświetl plik

@ -39,14 +39,14 @@ DOCS_TARGET_PATH = "docs"
DEFAULT_STREAM_TIMEINTERVAL = 5 * 60
ETHTXPOOL_HUMBUG_CLIENT_ID = os.environ.get(
"ETHTXPOOL_HUMBUG_CLIENT_ID", "client:ethereum-txpool-crawler-0"
HUMBUG_TXPOOL_CLIENT_ID = os.environ.get(
"HUMBUG_TXPOOL_CLIENT_ID", "client:ethereum-txpool-crawler-0"
)
# S3 Bucket
ETHERSCAN_SMARTCONTRACTS_BUCKET = os.environ.get("AWS_S3_SMARTCONTRACT_BUCKET")
ETHERSCAN_SMARTCONTRACTS_BUCKET = os.environ.get("MOONSTREAM_S3_SMARTCONTRACTS_BUCKET")
if ETHERSCAN_SMARTCONTRACTS_BUCKET is None:
raise ValueError("AWS_S3_SMARTCONTRACT_BUCKET is not set")
raise ValueError("MOONSTREAM_S3_SMARTCONTRACTS_BUCKET is not set")
MOONSTREAM_INTERNAL_HOSTED_ZONE_ID = os.environ.get(
"MOONSTREAM_INTERNAL_HOSTED_ZONE_ID", ""

2
backend/moonstreamapi/version.py
Wyświetl plik

@ -2,4 +2,4 @@
Moonstream library and API version.
"""
MOONSTREAMAPI_VERSION = "0.1.0"
MOONSTREAMAPI_VERSION = "0.1.1"

2
backend/moonstreamapi/web3_provider.py
Wyświetl plik

@ -32,7 +32,7 @@ def fetch_web3_provider_ip():
return record_value
if not MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI.replace(".", "").isnumeric():
if not MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI.startswith("http"):
web3_provider_ip = fetch_web3_provider_ip()
if web3_provider_ip is None:
raise ValueError("Unable to extract web3 provider IP")

15
backend/sample.env
Wyświetl plik

@ -1,15 +0,0 @@
export MOONSTREAM_CORS_ALLOWED_ORIGINS="http://localhost:3000,https://moonstream.to,https://www.moonstream.to"
export MOONSTREAM_APPLICATION_ID="<issued_bugout_application_id>"
export MOONSTREAM_DATA_JOURNAL_ID="<bugout_journal_id_to_store_blockchain_data>"
export MOONSTREAM_DB_URI="postgresql://<username>:<password>@<db_host>:<db_port>/<db_name>"
export MOONSTREAM_POOL_SIZE=0
export MOONSTREAM_ADMIN_ACCESS_TOKEN="<Access token to application resources>"
export MOONSTREAM_INTERNAL_HOSTED_ZONE_ID="<moonstream_internal_hosted_zone_id>"
export MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI="<connection_path_uri_to_ethereum_node>"
export AWS_S3_SMARTCONTRACT_BUCKET="<AWS S3 bucket to store smart contracts>"
export MOONSTREAM_S3_SMARTCONTRACTS_ABI_BUCKET="<AWS S3 bucket to store smart contracts ABI>"
export MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX="<Previx for AWS S3 bucket (prod,dev,..)>"
export BUGOUT_BROOD_URL="https://auth.bugout.dev"
export BUGOUT_SPIRE_URL="https://spire.bugout.dev"
export HUMBUG_REPORTER_BACKEND_TOKEN="<Bugout Humbug token for crash reports>"
export ETHTXPOOL_HUMBUG_CLIENT_ID="<Bugout Humbug client id for txpool transactions in journal>"

4
crawlers/mooncrawl/mooncrawl/etherscan.py
Wyświetl plik

@ -29,9 +29,9 @@ BASE_API_URL = "https://api.etherscan.io/api?module=contract&action=getsourcecod
ETHERSCAN_SMARTCONTRACTS_LABEL_NAME = "etherscan_smartcontract"
bucket = os.environ.get("AWS_S3_SMARTCONTRACT_BUCKET")
bucket = os.environ.get("MOONSTREAM_S3_SMARTCONTRACTS_BUCKET")
if bucket is None:
raise ValueError("AWS_S3_SMARTCONTRACT_BUCKET must be set")
raise ValueError("MOONSTREAM_S3_SMARTCONTRACTS_BUCKET must be set")
@dataclass

50
crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py
Wyświetl plik

@ -8,7 +8,7 @@ import logging
import time
from datetime import datetime, timedelta
from enum import Enum
from typing import Any, Callable, Dict, List
from typing import Any, Callable, Dict, List, Union
from uuid import UUID
import boto3 # type: ignore
@ -411,6 +411,46 @@ def get_unique_address(
)
def get_blocks_state(
db_session: Session, blockchain_type: AvailableBlockchainType
) -> Dict[str, int]:
"""
Generate meta information about
"""
blocks_state = {
"latest_stored_block": 0,
"latest_labelled_block": 0,
"earliest_labelled_block": 0,
}
label_model = get_label_model(blockchain_type)
transactions_model = get_transaction_model(blockchain_type)
max_transactions_number = db_session.query(
func.max(transactions_model.block_number).label("block_number")
).scalar()
result = (
db_session.query(
func.min(label_model.block_number).label("earliest_labelled_block"),
func.max(label_model.block_number).label("latest_labelled_block"),
max_transactions_number,
).filter(label_model.label == CRAWLER_LABEL)
).one_or_none()
if result:
earliest_labelled_block, latest_labelled_block, latest_stored_block = result
blocks_state = {
"latest_stored_block": latest_stored_block,
"latest_labelled_block": latest_labelled_block,
"earliest_labelled_block": earliest_labelled_block,
}
return blocks_state
def generate_list_of_names(
type: str, subscription_filters: Dict[str, Any], read_abi: bool, abi_json: Any
):
@ -577,7 +617,7 @@ def stats_generate_handler(args: argparse.Namespace):
# Meen it's are different blockchain type
continue
s3_data_object = {}
s3_data_object: Dict[str, Any] = {}
extention_data = []
@ -677,6 +717,10 @@ def stats_generate_handler(args: argparse.Namespace):
}
)
current_blocks_state = get_blocks_state(
db_session=db_session, blockchain_type=blockchain_type
)
for timescale in [timescale.value for timescale in TimeScale]:
start_date = (
@ -685,6 +729,8 @@ def stats_generate_handler(args: argparse.Namespace):
print(f"Timescale: {timescale}")
s3_data_object["blocks_state"] = current_blocks_state
s3_data_object["web3_metric"] = extention_data
functions_calls_data = generate_data(

2
crawlers/mooncrawl/sample.env
Wyświetl plik

@ -8,7 +8,7 @@ export MOONSTREAM_NODE_POLYGON_IPC_PORT="8545"
export MOONSTREAM_CRAWL_WORKERS=4
export MOONSTREAM_DB_URI="postgresql://<username>:<password>@<db_host>:<db_port>/<db_name>"
export MOONSTREAM_ETHERSCAN_TOKEN="<Token for etherscan>"
export AWS_S3_SMARTCONTRACT_BUCKET="<AWS S3 bucket for smart contracts>"
export MOONSTREAM_S3_SMARTCONTRACTS_BUCKET="<AWS S3 bucket for smart contracts>"
export MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX="<Previx for AWS S3 bucket (prod,dev,..)>"
export MOONSTREAM_HUMBUG_TOKEN="<Token for crawlers store data via Humbug>"
export COINMARKETCAP_API_KEY="<API key to parse conmarketcap>"

225
db/.dockerignore 100644
Wyświetl plik

@ -0,0 +1,225 @@
# Created by https://www.toptal.com/developers/gitignore/api/vim,visualstudiocode,linux,python
# Edit at https://www.toptal.com/developers/gitignore?templates=vim,visualstudiocode,linux,python
### Linux ###
*~
# temporary files which can be created if a process still has a handle open of a deleted file
.fuse_hidden*
# KDE directory preferences
.directory
# Linux trash folder which might appear on any partition or disk
.Trash-*
# .nfs files are created when an open file is removed but is still being accessed
.nfs*
### Python ###
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
pip-wheel-metadata/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
*.py,cover
.hypothesis/
.pytest_cache/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
db.sqlite3
db.sqlite3-journal
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
target/
# Jupyter Notebook
.ipynb_checkpoints
# IPython
profile_default/
ipython_config.py
# pyenv
.python-version
# pipenv
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
# However, in case of collaboration, if having platform-specific dependencies or dependencies
# having no cross-platform support, pipenv may install dependencies that don't work, or not
# install all needed dependencies.
#Pipfile.lock
# PEP 582; used by e.g. github.com/David-OConnor/pyflow
__pypackages__/
# Celery stuff
celerybeat-schedule
celerybeat.pid
# SageMath parsed files
*.sage.py
# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/
.dmypy.json
dmypy.json
# Pyre type checker
.pyre/
# pytype static type analyzer
.pytype/
### Vim ###
# Swap
[._]*.s[a-v][a-z]
!*.svg # comment out if you don't need vector files
[._]*.sw[a-p]
[._]s[a-rt-v][a-z]
[._]ss[a-gi-z]
[._]sw[a-p]
# Session
Session.vim
Sessionx.vim
# Temporary
.netrwhist
# Auto-generated tag files
tags
# Persistent undo
[._]*.un~
### VisualStudioCode ###
.vscode/*
!.vscode/settings.json
!.vscode/tasks.json
!.vscode/launch.json
!.vscode/extensions.json
*.code-workspace
### Pycharm ###
.idea
### VisualStudioCode Patch ###
# Ignore all local history of files
.history
# End of https://www.toptal.com/developers/gitignore/api/vim,visualstudiocode,linux,python
### VSCode ###
.vscode
# Git
.git
.github
# Envionments
.moonstream
.moonstreamdb
.venv
.env
# Environment variables
test.env
dev.env
prod.env
configs/test.env
configs/dev.env
configs/prod.env
configs/moonstreamdb.env
configs/docker.env
configs/docker.dev.env
configs/docker.test.env
configs/docker.prod.env
.secrets/
# Alembic migrations
configs/alembic.test.ini
configs/alembic.dev.ini
configs/alembic.prod.ini
Dockerfile
docker-compose.yml

31
db/.gitignore vendored
Wyświetl plik

@ -160,14 +160,29 @@ cython_debug/
# End of https://www.toptal.com/developers/gitignore/api/python,visualstudiocode
# Custom
dev.env
prod.env
dev.env.ps1
prod.env.ps1
alembic.dev.ini
alembic.prod.ini
# Envionments
.moonstream/
.moonstreamdb/
.db/
.venv/
.env/
# Environment variables
dev.env
test.env
prod.env
moonstreamdb.env
docker.env
docker.dev.env
docker.test.dev
docker.prod.env
docker.moonstreamdb.env
.secrets/
.moonstreamdb
# Alembic migrations
alembic.test.ini
alembic.dev.ini
alembic.prod.ini
alembic.moonstreamdb.ini
alembic.docker.ini

18
db/Dockerfile 100644
Wyświetl plik

@ -0,0 +1,18 @@
FROM python:3.8-slim-buster
# Update packages and
# prepare alembic for docker compose setup
RUN apt-get update && \
apt-get install -y libpq-dev gcc && \
rm -rf /var/lib/apt/lists/* && \
pip3 install --no-cache-dir --upgrade pip setuptools && \
pip3 install --no-cache-dir psycopg2-binary alembic
WORKDIR /usr/src/moonstreamdb
COPY . /usr/src/moonstreamdb
# Install Moonstream DB package
RUN pip3 install --no-cache-dir -e .
ENTRYPOINT ["./migrate.sh"]

0
db/alembic.sample.ini → db/configs/alembic.sample.ini
Wyświetl plik

49
db/configs/docker_generate_env.bash 100755
Wyświetl plik

@ -0,0 +1,49 @@
#!/usr/bin/env bash
# Prepare Moonstream DB application for docker-compose use
# Print help message
function usage {
echo "Usage: $0 [-h] -d DATABASE_NAME"
echo
echo "CLI to generate environment variables"
echo
echo "Optional arguments:"
echo " -h Show this help message and exit"
echo " -d Database name for postgres in docker-compose setup"
}
FLAG_DATABASE_NAME="moonstream_dev"
while getopts 'd:' flag; do
case "${flag}" in
d) FLAG_DATABASE_NAME="${OPTARG}" ;;
h) usage
exit 1 ;;
*) usage
exit 1 ;;
esac
done
set -e
SCRIPT_DIR="$(realpath $(dirname $0))"
DOCKER_MOONSTREAMDB_DB_URI="postgresql://postgres:postgres@db/$FLAG_DATABASE_NAME"
DOCKER_MOONSTREAMDB_ENV_FILE="docker.moonstreamdb.env"
DOCKER_MOONSTREAMDB_ALEMBIC_FILE="alembic.moonstreamdb.ini"
# Generate environment variables
cp "$SCRIPT_DIR/sample.env" "$SCRIPT_DIR/$DOCKER_MOONSTREAMDB_ENV_FILE"
# Clean file with variables from export prefix and quotation marks
sed --in-place 's|^export * ||' "$SCRIPT_DIR/$DOCKER_MOONSTREAMDB_ENV_FILE"
sed --in-place 's|"||g' "$SCRIPT_DIR/$DOCKER_MOONSTREAMDB_ENV_FILE"
sed -i "s|^MOONSTREAM_DB_URI=.*|MOONSTREAM_DB_URI=$DOCKER_MOONSTREAMDB_DB_URI|" "$SCRIPT_DIR/$DOCKER_MOONSTREAMDB_ENV_FILE"
# Generate alembic config
cp "$SCRIPT_DIR/alembic.sample.ini" "$SCRIPT_DIR/$DOCKER_MOONSTREAMDB_ALEMBIC_FILE"
sed -i "s|^sqlalchemy.url =.*|sqlalchemy.url = $DOCKER_MOONSTREAMDB_DB_URI|" "$SCRIPT_DIR/$DOCKER_MOONSTREAMDB_ALEMBIC_FILE"

1
db/sample.env → db/configs/sample.env
Wyświetl plik

@ -1,2 +1,3 @@
# Required environment variables to work with database CLI
export MOONSTREAM_DB_URI="postgresql://<username>:<password>@<db_host>:<db_port>/<db_name>"
export MOONSTREAM_POOL_SIZE=0

16
db/migrate.sh 100755
Wyświetl plik

@ -0,0 +1,16 @@
#!/usr/bin/env sh
# Sets up Brood server for docker compose:
# 1. Running alembic migrations to head (using config file specified
# by the ALEMBIC_CONFIG environment variable)
# 2. Running dev.sh (from the directory from which this script was called)
set -e
if [ -z "$ALEMBIC_CONFIG" ]
then
echo "Please explicitly set the ALEMBIC_CONFIG environment variable to point to an alembic configuration file"
exit 1
fi
ALEMBIC_CONFIG="$ALEMBIC_CONFIG" sh alembic.sh upgrade head

2
db/moonstreamdb/version.py
Wyświetl plik

@ -2,4 +2,4 @@
Moonstream database version.
"""
MOONSTREAMDB_VERSION = "0.2.0"
MOONSTREAMDB_VERSION = "0.2.1"

57
docker-compose.yml 100644
Wyświetl plik

@ -0,0 +1,57 @@
# Compose version
version: "3"
services:
# Moonstream API application
moonstreamapi:
build:
context: ./backend/
dockerfile: ./Dockerfile
image: moonstreamapi:latest
ports:
- "127.0.0.1:7481:7481"
# Specify environment file for compose setup
env_file: ./backend/configs/docker.moonstreamapi.env
environment:
MOONSTREAMAPI_HOST: 0.0.0.0
MOONSTREAMAPI_PORT: 7481
MOONSTREAMAPI_UVICORN_WORKERS: 1
healthcheck:
test: ["CMD", "curl", "-f", "http://moonstreamapi:7481/ping"]
interval: 5s
timeout: 1s
retries: 2
start_period: 2s
depends_on:
db:
condition: service_healthy
# Moonstream DB application
moonstreamdb:
build:
context: ./db/
dockerfile: ./Dockerfile
image: moonstreamdb:latest
# Specify environment file for compose setup
env_file: ./db/configs/docker.moonstreamdb.env
environment:
ALEMBIC_CONFIG: ./configs/alembic.moonstreamdb.ini
depends_on:
db:
condition: service_healthy
# DB postgres application
db:
image: postgres:13
ports:
- "127.0.0.1:5432:5432"
environment:
POSTGRES_PASSWORD: postgres
POSTGRES_USER: postgres
POSTGRES_DB: moonstream_dev
healthcheck:
test: ["CMD", "psql", "-U", "postgres", "-c", "SELECT 1;"]
interval: 5s
timeout: 1s
retries: 3
start_period: 2s

175
frontend/pages/status/index.js
Wyświetl plik

@ -1,11 +1,14 @@
import React from "react";
import React, { useContext } from "react";
import { useStatus } from "../../src/core/hooks";
import { Heading, Text, Flex, Spacer, chakra, Spinner } from "@chakra-ui/react";
import { getLayout, getLayoutProps } from "../../src/layouts/InfoPageLayout";
import UserContext from "../../src/core/providers/UserProvider/context";
const Status = () => {
const user = useContext(UserContext);
const healthyStatusText = "Available";
const downStatusText = "Unavailable";
const unauthorizedText = "Please login";
const healthyStatusColor = "green.900";
const downStatusColor = "red.600";
@ -14,14 +17,43 @@ const Status = () => {
};
const {
apiServerStatusCache,
ethereumClusterServerStatusCache,
gethStatusCache,
serverListStatusCache,
crawlersStatusCache,
dbServerStatusCache,
latestBlockDBStatusCache,
} = useStatus();
const moonstreamapiStatus = serverListStatusCache?.data?.filter(
(i) => i.status.name === "moonstreamapi"
)[0];
const moonstreamCrawlersStatus = serverListStatusCache?.data?.filter(
(i) => i.status.name === "moonstream_crawlers"
)[0];
const nodeEthereumAStatus = serverListStatusCache?.data?.filter(
(i) => i.status.name === "node_ethereum_a"
)[0];
const nodeEthereumAGeth = serverListStatusCache?.data?.filter(
(i) => i.status.name === "node_ethereum_a_geth"
)[0];
const nodeEthereumBStatus = serverListStatusCache?.data?.filter(
(i) => i.status.name === "node_ethereum_b"
)[0];
const nodeEthereumBGeth = serverListStatusCache?.data?.filter(
(i) => i.status.name === "node_ethereum_b_geth"
)[0];
const nodePolygonAStatus = serverListStatusCache?.data?.filter(
(i) => i.status.name === "node_polygon_a"
)[0];
const nodePolygonAGeth = serverListStatusCache?.data?.filter(
(i) => i.status.name === "node_polygon_a_geth"
)[0];
const nodePolygonBStatus = serverListStatusCache?.data?.filter(
(i) => i.status.name === "node_polygon_b"
)[0];
const nodePolygonBGeth = serverListStatusCache?.data?.filter(
(i) => i.status.name === "node_polygon_b_geth"
)[0];
const StatusRow = (props) => {
console.log(props.cache.data);
return (
@ -46,50 +78,44 @@ const Status = () => {
{`Status page`}
</Heading>
<chakra.span pl={2} px={12} py={2} width="400px">
<StatusRow title="Backend server" cache={apiServerStatusCache}>
<StatusRow title="Backend server" cache={serverListStatusCache}>
<Text
color={
apiServerStatusCache?.data?.status == "ok"
moonstreamapiStatus?.status.body.status == "ok"
? healthyStatusColor
: downStatusColor
}
>
{apiServerStatusCache?.data?.status == "ok"
{moonstreamapiStatus?.status.body.status == "ok"
? healthyStatusText
: downStatusText}
</Text>
</StatusRow>
<br />
<StatusRow
title="Crawlers server"
cache={ethereumClusterServerStatusCache}
>
<StatusRow title="Crawlers server" cache={crawlersStatusCache}>
<Text
color={
ethereumClusterServerStatusCache?.data?.status == "ok"
moonstreamCrawlersStatus?.status.body.status == "ok"
? healthyStatusColor
: downStatusColor
}
>
{ethereumClusterServerStatusCache?.data
{moonstreamCrawlersStatus?.status.body.status == "ok"
? healthyStatusText
: downStatusText}
</Text>
</StatusRow>
<StatusRow title="Latest block in Geth" cache={gethStatusCache}>
<Text>
{gethStatusCache?.data?.current_block
? gethStatusCache.data.current_block
: 0}
</Text>
</StatusRow>
<StatusRow title="Txpool latest record ts" cache={crawlersStatusCache}>
<Text>
{crawlersStatusCache?.data?.ethereum_txpool_timestamp
? shortTimestamp(
crawlersStatusCache?.data?.ethereum_txpool_timestamp
)
: downStatusText}
{!user
? crawlersStatusCache?.data?.ethereum_txpool_timestamp
? shortTimestamp(
crawlersStatusCache?.data?.ethereum_txpool_timestamp
)
: downStatusText
: unauthorizedText}
</Text>
</StatusRow>
<StatusRow
@ -97,15 +123,104 @@ const Status = () => {
cache={crawlersStatusCache}
>
<Text>
{crawlersStatusCache?.data?.ethereum_trending_timestamp
? shortTimestamp(
crawlersStatusCache?.data?.ethereum_trending_timestamp
)
: downStatusText}
{!user
? crawlersStatusCache?.data?.ethereum_trending_timestamp
? shortTimestamp(
crawlersStatusCache?.data?.ethereum_trending_timestamp
)
: downStatusText
: unauthorizedText}
</Text>
</StatusRow>
<br />
<StatusRow title="Node Ethereum A" cache={serverListStatusCache}>
<Text
color={
nodeEthereumAStatus?.status.body.status == "ok"
? healthyStatusColor
: downStatusColor
}
>
{nodeEthereumAStatus?.status.body.status == "ok"
? healthyStatusText
: downStatusText}
</Text>
</StatusRow>
<StatusRow title="Current block" cache={serverListStatusCache}>
<Text>
{nodeEthereumAGeth?.status.body.current_block
? nodeEthereumAGeth.status.body.current_block
: 0}
</Text>
</StatusRow>
<br />
<StatusRow title="Node Ethereum B" cache={serverListStatusCache}>
<Text
color={
nodeEthereumBStatus?.status.body.status == "ok"
? healthyStatusColor
: downStatusColor
}
>
{nodeEthereumBStatus?.status.body.status == "ok"
? healthyStatusText
: downStatusText}
</Text>
</StatusRow>
<StatusRow title="Current block" cache={serverListStatusCache}>
<Text>
{nodeEthereumBGeth?.status.body.current_block
? nodeEthereumBGeth.status.body.current_block
: 0}
</Text>
</StatusRow>
<br />
<StatusRow title="Node Polygon A" cache={serverListStatusCache}>
<Text
color={
nodePolygonAStatus?.status.body.status == "ok"
? healthyStatusColor
: downStatusColor
}
>
{nodePolygonAStatus?.status.body.status == "ok"
? healthyStatusText
: downStatusText}
</Text>
</StatusRow>
<StatusRow title="Current block" cache={serverListStatusCache}>
<Text>
{nodePolygonAGeth?.status.body.current_block
? nodePolygonAGeth.status.body.current_block
: 0}
</Text>
</StatusRow>
<br />
<StatusRow title="Node Polygon B" cache={serverListStatusCache}>
<Text
color={
nodePolygonBStatus?.status.body.status == "ok"
? healthyStatusColor
: downStatusColor
}
>
{nodePolygonBStatus?.status.body.status == "ok"
? healthyStatusText
: downStatusText}
</Text>
</StatusRow>
<StatusRow title="Current block" cache={serverListStatusCache}>
<Text>
{nodePolygonBGeth?.status.body.current_block
? nodePolygonBGeth.status.body.current_block
: 0}
</Text>
</StatusRow>
<br />
<StatusRow title="Database server" cache={dbServerStatusCache}>
<Text
color={

10
frontend/sample.env
Wyświetl plik

@ -1,10 +1,8 @@
export NEXT_PUBLIC_MIXPANEL_TOKEN="<YOUR MIXPANEL TOKEN HERE>"
export NEXT_PUBLIC_STRIPE_PUBLISHABLE_KEY="<stripe publishable key>"
export NEXT_PUBLIC_MOONSTREAM_API_URL="<moonstream_api_url>"
export NEXT_PUBLIC_MOONSTREAM_ETHEREUM_CLUSTER_URL="<moonstream_crawlers_url>"
export NEXT_PUBLIC_MOONSTREAM_DB_URL="<moonstream_db_url>"
export NEXT_PUBLIC_BUGOUT_STATUS_URL=https://status.moonstream.to
export NEXT_PUBLIC_MOONSTREAM_API_URL=https://api.moonstream.to
export NEXT_PUBLIC_MOONSTREAM_DB_URL=https://pg.moonstream.to
export NEXT_PUBLIC_SIMIOTICS_AUTH_URL=https://auth.bugout.dev
export NEXT_PUBLIC_SIMIOTICS_JOURNALS_URL=https://spire.bugout.dev
export NEXT_PUBLIC_FRONTEND_VERSION="<frontend_version_number>"%
export NEXT_PUBLIC_FRONTEND_VERSION="<frontend_version_number>"

30
frontend/src/core/hooks/useStatus.js
Wyświetl plik

@ -3,16 +3,8 @@ import { queryCacheProps } from "./hookCommon";
import { StatusService } from "../../core/services";
const useStatus = () => {
const getAPIServerStatus = async () => {
const response = await StatusService.apiServerStatus();
return response.data;
};
const getEthereumClusterServerStatus = async () => {
const response = await StatusService.ethereumClusterServerStatus();
return response.data;
};
const getGethStatus = async () => {
const response = await StatusService.gethStatus();
const getServerListStatus = async () => {
const response = await StatusService.serverListStatus();
return response.data;
};
const getCrawlersStatus = async () => {
@ -28,22 +20,14 @@ const useStatus = () => {
return response.data;
};
const apiServerStatusCache = useQuery("apiServer", getAPIServerStatus, {
...queryCacheProps,
retry: 0,
});
const ethereumClusterServerStatusCache = useQuery(
"ethereumClusterServer",
getEthereumClusterServerStatus,
const serverListStatusCache = useQuery(
"serverListStatus",
getServerListStatus,
{
...queryCacheProps,
retry: 0,
}
);
const gethStatusCache = useQuery("geth", getGethStatus, {
...queryCacheProps,
retry: 0,
});
const crawlersStatusCache = useQuery("crawlers", getCrawlersStatus, {
...queryCacheProps,
retry: 0,
@ -62,9 +46,7 @@ const useStatus = () => {
);
return {
apiServerStatusCache,
ethereumClusterServerStatusCache,
gethStatusCache,
serverListStatusCache,
crawlersStatusCache,
dbServerStatusCache,
latestBlockDBStatusCache,

21
frontend/src/core/services/status.service.js
Wyświetl plik

@ -1,28 +1,13 @@
import { http } from "../utils";
const BUGOUT_STATUS_URL = process.env.NEXT_PUBLIC_BUGOUT_STATUS_URL;
const API_URL = process.env.NEXT_PUBLIC_MOONSTREAM_API_URL;
const DB_URL = process.env.NEXT_PUBLIC_MOONSTREAM_DB_URL;
const ETHEREUM_CLUSTER_URL =
process.env.NEXT_PUBLIC_MOONSTREAM_ETHEREUM_CLUSTER_URL;
export const apiServerStatus = () => {
export const serverListStatus = () => {
return http({
method: "GET",
url: `${API_URL}/ping`,
});
};
export const ethereumClusterServerStatus = () => {
return http({
method: "GET",
url: `${ETHEREUM_CLUSTER_URL}/ping`,
});
};
export const gethStatus = () => {
return http({
method: "GET",
url: `${ETHEREUM_CLUSTER_URL}/status`,
url: `${BUGOUT_STATUS_URL}`,
});
};