diff --git a/.github/workflows/release.clients.python.yml b/.github/workflows/release.clients.python.yml
new file mode 100644
index 00000000..5aa3e1b2
--- /dev/null
+++ b/.github/workflows/release.clients.python.yml
@@ -0,0 +1,45 @@
+name: Publish Moonstream Python client library
+
+on:
+ push:
+ tags:
+ - "clients/python/v*"
+
+defaults:
+ run:
+ working-directory: clients/python
+
+jobs:
+ publish:
+ runs-on: ubuntu-20.04
+ steps:
+ - uses: actions/checkout@v2
+ - name: Set up Python
+ uses: actions/setup-python@v2
+ with:
+ python-version: "3.9"
+ - name: Install dependencies
+ run: |
+ python -m pip install --upgrade pip
+ pip install -e .[distribute]
+ - name: Build and publish
+ env:
+ TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
+ TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+ run: |
+ python setup.py sdist bdist_wheel
+ twine upload dist/*
+ create_release:
+ runs-on: ubuntu-20.04
+ steps:
+ - uses: actions/create-release@v1
+ id: create_release
+ env:
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ with:
+ tag_name: ${{ github.ref }}
+ release_name: "Moonstream Python client library - ${{ github.ref }}"
+ body: |
+ Version ${{ github.ref }} of the Moonstream Python client library.
+ draft: true
+ prerelease: false
diff --git a/.github/workflows/test.clients.python.yml b/.github/workflows/test.clients.python.yml
new file mode 100644
index 00000000..7a454b44
--- /dev/null
+++ b/.github/workflows/test.clients.python.yml
@@ -0,0 +1,38 @@
+name: Linting and tests for the Moonstream Python client library
+
+on:
+ pull_request:
+ branches:
+ - "main"
+ paths:
+ - "clients/python/**"
+
+jobs:
+ build:
+ runs-on: ubuntu-20.04
+ steps:
+ - uses: actions/checkout@v2
+ - name: Set up python
+ uses: actions/setup-python@v2
+ with:
+ python-version: "3.9"
+ - name: Install test requirements
+ working-directory: ./clients/python
+ run: pip install -e .[dev]
+ - name: Mypy type check
+ working-directory: ./clients/python
+ run: mypy moonstream/
+ - name: Black syntax check
+ working-directory: ./clients/python
+ run: black --check moonstream/
+ - name: Unit tests
+ working-directory: ./clients/python
+ run: python -m unittest discover -v
+ - name: Check that versions are synchronized
+ working-directory: ./clients/python
+ run: |
+ CLIENT_VERSION=$(python -c "from moonstream.client import CLIENT_VERSION; print(CLIENT_VERSION)")
+ SETUP_PY_VERSION=$(python setup.py --version)
+ echo "Client version: $CLIENT_VERSION"
+ echo "setup.py version: $SETUP_PY_VERSION"
+ test "$CLIENT_VERSION" = "$SETUP_PY_VERSION"
diff --git a/backend/moonstream/actions.py b/backend/moonstream/actions.py
index 112fc36e..72e894f9 100644
--- a/backend/moonstream/actions.py
+++ b/backend/moonstream/actions.py
@@ -5,6 +5,8 @@ from enum import Enum
import uuid
import boto3 # type: ignore
+from bugout.data import BugoutSearchResults
+from bugout.journal import SearchOrder
from moonstreamdb.models import (
EthereumAddress,
EthereumLabel,
@@ -20,12 +22,20 @@ from .settings import (
MOONSTREAM_APPLICATION_ID,
bugout_client as bc,
BUGOUT_REQUEST_TIMEOUT_SECONDS,
+ MOONSTREAM_ADMIN_ACCESS_TOKEN,
+ MOONSTREAM_DATA_JOURNAL_ID,
)
logger = logging.getLogger(__name__)
ETHERSCAN_SMARTCONTRACT_LABEL_NAME = "etherscan_smartcontract"
+class StatusAPIException(Exception):
+ """
+ Raised during checking Moonstream API statuses.
+ """
+
+
def get_contract_source_info(
db_session: Session, contract_address: str
) -> Optional[data.EthereumSmartContractSourceInfo]:
@@ -192,3 +202,29 @@ def create_onboarding_resource(
timeout=BUGOUT_REQUEST_TIMEOUT_SECONDS,
)
return resource
+
+
+def check_api_status():
+ crawl_types_timestamp: Dict[str, Any] = {
+ "ethereum_txpool": None,
+ "ethereum_trending": None,
+ }
+ for crawl_type in crawl_types_timestamp.keys():
+ try:
+ search_results: BugoutSearchResults = bc.search(
+ token=MOONSTREAM_ADMIN_ACCESS_TOKEN,
+ journal_id=MOONSTREAM_DATA_JOURNAL_ID,
+ query=f"tag:crawl_type:{crawl_type}",
+ limit=1,
+ content=False,
+ timeout=10.0,
+ order=SearchOrder.DESCENDING,
+ )
+ if len(search_results.results) == 1:
+ crawl_types_timestamp[crawl_type] = search_results.results[0].created_at
+ except Exception:
+ raise StatusAPIException(
+ f"Unable to get status for crawler with type: {crawl_type}"
+ )
+
+ return crawl_types_timestamp
diff --git a/backend/moonstream/api.py b/backend/moonstream/api.py
index 6e687eb9..122bf309 100644
--- a/backend/moonstream/api.py
+++ b/backend/moonstream/api.py
@@ -7,9 +7,12 @@ import time
from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware
+from . import actions
from . import data
+from .middleware import MoonstreamHTTPException
from .routes.address_info import app as addressinfo_api
from .routes.nft import app as nft_api
+from .routes.whales import app as whales_api
from .routes.subscriptions import app as subscriptions_api
from .routes.streams import app as streams_api
from .routes.txinfo import app as txinfo_api
@@ -46,9 +49,31 @@ async def now_handler() -> data.NowResponse:
return data.NowResponse(epoch_time=time.time())
+@app.get("/status", response_model=data.StatusResponse)
+async def status_handler() -> data.StatusResponse:
+ """
+ Get latest records and their creation timestamp for crawlers:
+ - ethereum_txpool
+ - ethereum_trending
+ """
+ try:
+ crawl_types_timestamp = actions.check_api_status()
+ except actions.StatusAPIException:
+ raise MoonstreamHTTPException(status_code=500)
+ except Exception as e:
+ logger.error(f"Unhandled status exception, error: {e}")
+ raise MoonstreamHTTPException(status_code=500)
+
+ return data.StatusResponse(
+ ethereum_txpool_timestamp=crawl_types_timestamp["ethereum_txpool"],
+ ethereum_trending_timestamp=crawl_types_timestamp["ethereum_trending"],
+ )
+
+
app.mount("/subscriptions", subscriptions_api)
app.mount("/users", users_api)
app.mount("/streams", streams_api)
app.mount("/txinfo", txinfo_api)
app.mount("/address_info", addressinfo_api)
app.mount("/nft", nft_api)
+app.mount("/whales", whales_api)
diff --git a/backend/moonstream/data.py b/backend/moonstream/data.py
index 05e49ea2..7a7089a2 100644
--- a/backend/moonstream/data.py
+++ b/backend/moonstream/data.py
@@ -67,6 +67,11 @@ class NowResponse(BaseModel):
epoch_time: float
+class StatusResponse(BaseModel):
+ ethereum_txpool_timestamp: Optional[datetime] = None
+ ethereum_trending_timestamp: Optional[datetime] = None
+
+
class SubscriptionUpdate(BaseModel):
update: Dict[str, Any]
drop_keys: List[str] = Field(default_factory=list)
diff --git a/backend/moonstream/providers/bugout.py b/backend/moonstream/providers/bugout.py
index ee96698d..f7b29f76 100644
--- a/backend/moonstream/providers/bugout.py
+++ b/backend/moonstream/providers/bugout.py
@@ -16,10 +16,13 @@ from sqlalchemy.orm import Session
from .. import data
from ..stream_queries import StreamQuery
+from ..settings import ETHTXPOOL_HUMBUG_CLIENT_ID
logger = logging.getLogger(__name__)
logger.setLevel(logging.WARN)
+allowed_tags = ["tag:erc721"]
+
class BugoutEventProviderError(Exception):
"""
@@ -314,14 +317,17 @@ class EthereumTXPoolProvider(BugoutEventProvider):
]
subscriptions_filters = []
for address in addresses:
- subscriptions_filters.extend(
- [f"?#from_address:{address}", f"?#to_address:{address}"]
- )
+ if address in allowed_tags:
+ subscriptions_filters.append(address)
+ else:
+ subscriptions_filters.extend(
+ [f"?#from_address:{address}", f"?#to_address:{address}"]
+ )
return subscriptions_filters
-class NftProvider(BugoutEventProvider):
+class PublicDataProvider(BugoutEventProvider):
def __init__(
self,
event_type: str,
@@ -358,7 +364,7 @@ Shows the top 10 addresses active on the Ethereum blockchain over the last hour
4. Amount (in WEI) received
To restrict your queries to this provider, add a filter of \"type:ethereum_whalewatch\" to your query (query parameter: \"q\") on the /streams endpoint."""
-whalewatch_provider = BugoutEventProvider(
+whalewatch_provider = PublicDataProvider(
event_type="ethereum_whalewatch",
description=whalewatch_description,
default_time_interval_seconds=310,
@@ -376,7 +382,7 @@ ethereum_txpool_provider = EthereumTXPoolProvider(
description=ethereum_txpool_description,
default_time_interval_seconds=5,
estimated_events_per_time_interval=50,
- tags=["client:ethereum-txpool-crawler-0"],
+ tags=[f"client:{ETHTXPOOL_HUMBUG_CLIENT_ID}"],
)
nft_summary_description = """Event provider for NFT market summaries.
@@ -388,7 +394,7 @@ Currently, it summarizes the activities on the following NFT markets:
This provider is currently not accessible for subscription. The data from this provider is publicly
available at the /nft endpoint."""
-nft_summary_provider = NftProvider(
+nft_summary_provider = PublicDataProvider(
event_type="nft_summary",
description=nft_summary_description,
# 40 blocks per summary, 15 seconds per block + 2 seconds wiggle room.
diff --git a/backend/moonstream/providers/ethereum_blockchain.py b/backend/moonstream/providers/ethereum_blockchain.py
index 1eb87bde..dc607012 100644
--- a/backend/moonstream/providers/ethereum_blockchain.py
+++ b/backend/moonstream/providers/ethereum_blockchain.py
@@ -8,11 +8,14 @@ from bugout.data import BugoutResource
from moonstreamdb.models import (
EthereumBlock,
EthereumTransaction,
+ EthereumAddress,
+ EthereumLabel,
)
from sqlalchemy import or_, and_, text
from sqlalchemy.orm import Session, Query
from sqlalchemy.sql.functions import user
+
from .. import data
from ..stream_boundaries import validate_stream_boundary
from ..stream_queries import StreamQuery
@@ -23,6 +26,7 @@ logger.setLevel(logging.WARN)
event_type = "ethereum_blockchain"
+allowed_tags = ["tag:erc721"]
description = f"""Event provider for transactions from the Ethereum blockchain.
@@ -79,6 +83,7 @@ class Filters:
from_addresses: List[str] = field(default_factory=list)
to_addresses: List[str] = field(default_factory=list)
+ labels: List[str] = field(default_factory=list)
def default_filters(subscriptions: List[BugoutResource]) -> Filters:
@@ -91,8 +96,11 @@ def default_filters(subscriptions: List[BugoutResource]) -> Filters:
Optional[str], subscription.resource_data.get("address")
)
if subscription_address is not None:
- filters.from_addresses.append(subscription_address)
- filters.to_addresses.append(subscription_address)
+ if subscription_address in allowed_tags:
+ filters.labels.append(subscription_address.split(":")[1])
+ else:
+ filters.from_addresses.append(subscription_address)
+ filters.to_addresses.append(subscription_address)
else:
logger.warn(
f"Could not find subscription address for subscription with resource id: {subscription.id}"
@@ -157,14 +165,20 @@ def parse_filters(
parsed_filters.from_addresses.append(address)
parsed_filters.to_addresses.append(address)
- if not (parsed_filters.from_addresses or parsed_filters.to_addresses):
+ if not (
+ parsed_filters.from_addresses
+ or parsed_filters.to_addresses
+ or parsed_filters.labels
+ ):
return None
return parsed_filters
def query_ethereum_transactions(
- db_session: Session, stream_boundary: data.StreamBoundary, parsed_filters: Filters
+ db_session: Session,
+ stream_boundary: data.StreamBoundary,
+ parsed_filters: Filters,
) -> Query:
"""
Builds a database query for Ethereum transactions that occurred within the window of time that
@@ -198,15 +212,41 @@ def query_ethereum_transactions(
query = query.filter(EthereumBlock.timestamp <= stream_boundary.end_time)
# We want to take a big disjunction (OR) over ALL the filters, be they on "from" address or "to" address
- address_clauses = [
- EthereumTransaction.from_address == address
- for address in parsed_filters.from_addresses
- ] + [
- EthereumTransaction.to_address == address
- for address in parsed_filters.to_addresses
- ]
- if address_clauses:
- query = query.filter(or_(*address_clauses))
+ address_clauses = []
+
+ address_clauses.extend(
+ [
+ EthereumTransaction.from_address == address
+ for address in parsed_filters.from_addresses
+ ]
+ + [
+ EthereumTransaction.to_address == address
+ for address in parsed_filters.to_addresses
+ ]
+ )
+
+ labels_clause = []
+
+ if parsed_filters.labels:
+ label_clause = (
+ db_session.query(EthereumAddress)
+ .join(EthereumLabel, EthereumAddress.id == EthereumLabel.address_id)
+ .filter(
+ or_(
+ *[
+ EthereumLabel.label.contains(label)
+ for label in list(set(parsed_filters.labels))
+ ]
+ )
+ )
+ .exists()
+ )
+ labels_clause.append(label_clause)
+
+ subscriptions_clause = address_clauses + labels_clause
+
+ if subscriptions_clause:
+ query = query.filter(or_(*subscriptions_clause))
return query
@@ -353,8 +393,7 @@ def next_event(
query_ethereum_transactions(db_session, next_stream_boundary, parsed_filters)
.order_by(text("timestamp asc"))
.limit(1)
- .one_or_none()
- )
+ ).one_or_none()
if maybe_ethereum_transaction is None:
return None
@@ -394,9 +433,7 @@ def previous_event(
)
.order_by(text("timestamp desc"))
.limit(1)
- .one_or_none()
- )
-
+ ).one_or_none()
if maybe_ethereum_transaction is None:
return None
return ethereum_transaction_event(maybe_ethereum_transaction)
diff --git a/backend/moonstream/routes/users.py b/backend/moonstream/routes/users.py
index c414fabf..237f031e 100644
--- a/backend/moonstream/routes/users.py
+++ b/backend/moonstream/routes/users.py
@@ -7,7 +7,6 @@ import uuid
from bugout.data import BugoutToken, BugoutUser, BugoutResource, BugoutUserTokens
from bugout.exceptions import BugoutResponseException
-
from fastapi import (
Body,
FastAPI,
diff --git a/backend/moonstream/routes/whales.py b/backend/moonstream/routes/whales.py
new file mode 100644
index 00000000..2c089f4c
--- /dev/null
+++ b/backend/moonstream/routes/whales.py
@@ -0,0 +1,93 @@
+"""
+Moonstream's /whales endpoints.
+
+These endpoints provide public access to whale watch summaries. No authentication required.
+"""
+from datetime import datetime
+import logging
+from typing import Optional
+
+from bugout.data import BugoutResource
+
+from fastapi import Depends, FastAPI, Query
+from moonstreamdb import db
+from fastapi.middleware.cors import CORSMiddleware
+from sqlalchemy.orm import Session
+
+from .. import data
+from ..providers.bugout import whalewatch_provider
+from ..settings import (
+ bugout_client,
+ DOCS_TARGET_PATH,
+ MOONSTREAM_ADMIN_ACCESS_TOKEN,
+ MOONSTREAM_DATA_JOURNAL_ID,
+ ORIGINS,
+)
+from ..stream_queries import StreamQuery
+from ..version import MOONSTREAM_VERSION
+
+logger = logging.getLogger(__name__)
+
+tags_metadata = [
+ {"name": "whales", "description": "Whales summaries"},
+]
+
+app = FastAPI(
+ title=f"Moonstream /whales API",
+ description="User, token and password handlers.",
+ version=MOONSTREAM_VERSION,
+ openapi_tags=tags_metadata,
+ openapi_url="/openapi.json",
+ docs_url=None,
+ redoc_url=f"/{DOCS_TARGET_PATH}",
+)
+
+app.add_middleware(
+ CORSMiddleware,
+ allow_origins=ORIGINS,
+ allow_credentials=True,
+ allow_methods=["*"],
+ allow_headers=["*"],
+)
+
+
+@app.get("/", tags=["whales"], response_model=data.GetEventsResponse)
+async def stream_handler(
+ start_time: int = Query(0),
+ end_time: Optional[int] = Query(None),
+ include_start: bool = Query(False),
+ include_end: bool = Query(False),
+ db_session: Session = Depends(db.yield_db_session),
+) -> data.GetEventsResponse:
+ """
+ Retrieves the list of whales spotted over given stream boundary
+
+ - **start_time**: Timestamp. Must be provided otherwise this request will hang
+ - **end_time**: Timestamp. Optional.
+ - **include_start** (string): is start_time inclusive or not
+ - **include_end** (string): is end_time inclusive or not
+ """
+ stream_boundary = data.StreamBoundary(
+ start_time=start_time,
+ end_time=end_time,
+ include_start=include_start,
+ include_end=include_end,
+ )
+
+ result = whalewatch_provider.get_events(
+ db_session=db_session,
+ bugout_client=bugout_client,
+ data_journal_id=MOONSTREAM_DATA_JOURNAL_ID,
+ data_access_token=MOONSTREAM_ADMIN_ACCESS_TOKEN,
+ stream_boundary=stream_boundary,
+ user_subscriptions={whalewatch_provider.event_type: []},
+ query=StreamQuery(subscription_types=[whalewatch_provider.event_type]),
+ )
+
+ if result is None:
+ return data.GetEventsResponse(stream_boundary=stream_boundary, events=[])
+
+ provider_stream_boundary, events = result
+ return data.GetEventsResponse(
+ stream_boundary=provider_stream_boundary, events=events
+ )
diff --git a/backend/moonstream/settings.py b/backend/moonstream/settings.py
index 0a90d751..d18bb852 100644
--- a/backend/moonstream/settings.py
+++ b/backend/moonstream/settings.py
@@ -46,6 +46,10 @@ for path in MOONSTREAM_OPENAPI_LIST:
DEFAULT_STREAM_TIMEINTERVAL = 5 * 60
+ETHTXPOOL_HUMBUG_CLIENT_ID = os.environ.get(
+ "ETHTXPOOL_HUMBUG_CLIENT_ID", "client:ethereum-txpool-crawler-0"
+)
+
# S3 Bucket
ETHERSCAN_SMARTCONTRACTS_BUCKET = os.environ.get("AWS_S3_SMARTCONTRACT_BUCKET")
if ETHERSCAN_SMARTCONTRACTS_BUCKET is None:
diff --git a/backend/sample.env b/backend/sample.env
index 779bbb0c..6c44a369 100644
--- a/backend/sample.env
+++ b/backend/sample.env
@@ -9,3 +9,4 @@ export AWS_S3_SMARTCONTRACT_BUCKET="<AWS S3 bucket to store smart contracts>"
export BUGOUT_BROOD_URL="https://auth.bugout.dev"
export BUGOUT_SPIRE_URL="https://spire.bugout.dev"
export HUMBUG_REPORTER_BACKEND_TOKEN="<Bugout Humbug token for crash reports>"
+export ETHTXPOOL_HUMBUG_CLIENT_ID="<Bugout Humbug client id for txpool transactions in journal>"
\ No newline at end of file
diff --git a/clients/python/.gitignore b/clients/python/.gitignore
new file mode 100644
index 00000000..04290913
--- /dev/null
+++ b/clients/python/.gitignore
@@ -0,0 +1,147 @@
+# Created by https://www.toptal.com/developers/gitignore/api/python
+# Edit at https://www.toptal.com/developers/gitignore?templates=python
+
+### Python ###
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+# Usually these files are written by a python script from a template
+# before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+# For a library or package, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+# However, in case of collaboration, if having platform-specific dependencies or dependencies
+# having no cross-platform support, pipenv may install dependencies that don't work, or not
+# install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# End of https://www.toptal.com/developers/gitignore/api/python
+
+.moonstream-py/
+.venv/
diff --git a/clients/python/README.md b/clients/python/README.md
new file mode 100644
index 00000000..7d68db78
--- /dev/null
+++ b/clients/python/README.md
@@ -0,0 +1,13 @@
+# Moonstream Python client
+
+This is the Python client library for the Moonstream API.
+
+## Installation
+
+This library assumes you are using Python 3.6 or greater.
+
+Install using `pip`:
+
+```bash
+pip install moonstream
+```
diff --git a/clients/python/moonstream/__init__.py b/clients/python/moonstream/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/clients/python/moonstream/client.py b/clients/python/moonstream/client.py
new file mode 100644
index 00000000..b2d0211f
--- /dev/null
+++ b/clients/python/moonstream/client.py
@@ -0,0 +1,424 @@
+from dataclasses import dataclass, field
+import logging
+import os
+from typing import Any, Dict, List, Optional
+
+import requests
+
+logger = logging.getLogger(__name__)
+log_level = logging.INFO
+if os.environ.get("DEBUG", "").lower() in ["true", "1"]:
+ log_level = logging.DEBUG
+logger.setLevel(log_level)
+
+
+# Keep this synchronized with the version in setup.py
+CLIENT_VERSION = "0.0.2"
+
+ENDPOINT_PING = "/ping"
+ENDPOINT_VERSION = "/version"
+ENDPOINT_NOW = "/now"
+ENDPOINT_TOKEN = "/users/token"
+ENDPOINT_SUBSCRIPTIONS = "/subscriptions/"
+ENDPOINT_SUBSCRIPTION_TYPES = "/subscriptions/types"
+ENDPOINT_STREAMS = "/streams/"
+ENDPOINT_STREAMS_LATEST = "/streams/latest"
+ENDPOINT_STREAMS_NEXT = "/streams/next"
+ENDPOINT_STREAMS_PREVIOUS = "/streams/previous"
+
+ENDPOINTS = [
+ ENDPOINT_PING,
+ ENDPOINT_VERSION,
+ ENDPOINT_NOW,
+ ENDPOINT_TOKEN,
+ ENDPOINT_SUBSCRIPTIONS,
+ ENDPOINT_SUBSCRIPTION_TYPES,
+ ENDPOINT_STREAMS,
+ ENDPOINT_STREAMS_LATEST,
+ ENDPOINT_STREAMS_NEXT,
+ ENDPOINT_STREAMS_PREVIOUS,
+]
+
+
+def moonstream_endpoints(url: str) -> Dict[str, str]:
+ """
+ Creates a dictionary of Moonstream API endpoints at the given Moonstream API URL.
+ """
+ url_with_protocol = url
+ if not (
+ url_with_protocol.startswith("http://")
+ or url_with_protocol.startswith("https://")
+ ):
+ url_with_protocol = f"http://{url_with_protocol}"
+
+ normalized_url = url_with_protocol.rstrip("/")
+
+ return {endpoint: f"{normalized_url}{endpoint}" for endpoint in ENDPOINTS}
+
+
+class UnexpectedResponse(Exception):
+ """
+ Raised when a server response cannot be parsed into the appropriate/expected Python structure.
+ """
+
+
+class Unauthenticated(Exception):
+ """
+ Raised when a user tries to make a request that needs to be authenticated by they are not authenticated.
+ """
+
+
+@dataclass(frozen=True)
+class APISpec:
+ url: str
+ endpoints: Dict[str, str]
+
+
+class Moonstream:
+ """
+ A Moonstream client configured to communicate with a given Moonstream API server.
+ """
+
+ def __init__(
+ self,
+ url: str = "https://api.moonstream.to",
+ timeout: Optional[float] = None,
+ ):
+ """
+ Initializes a Moonstream API client.
+
+ Arguments:
+ url - Moonstream API URL. By default this points to the production Moonstream API at https://api.moonstream.to,
+ but you can replace it with the URL of any other Moonstream API instance.
+ timeout - Timeout (in seconds) for Moonstream API requests. Default is None, which means that
+ Moonstream API requests will never time out.
+
+ Returns: A Moonstream client.
+ """
+ endpoints = moonstream_endpoints(url)
+ self.api = APISpec(url=url, endpoints=endpoints)
+ self.timeout = timeout
+ self._session = requests.Session()
+ self._session.headers.update(
+ {"User-Agent": f"Moonstream Python client (version {CLIENT_VERSION})"}
+ )
+
+ def ping(self) -> Dict[str, Any]:
+ """
+ Checks that you have a connection to the Moonstream API.
+ """
+ r = self._session.get(self.api.endpoints[ENDPOINT_PING])
+ r.raise_for_status()
+ return r.json()
+
+ def version(self) -> Dict[str, Any]:
+ """
+ Gets the Moonstream API version information from the server.
+ """
+ r = self._session.get(self.api.endpoints[ENDPOINT_VERSION])
+ r.raise_for_status()
+ return r.json()
+
+ def server_time(self) -> float:
+ """
+ Gets the current time (as microseconds since the Unix epoch) on the server.
+ """
+ r = self._session.get(self.api.endpoints[ENDPOINT_NOW])
+ r.raise_for_status()
+ result = r.json()
+ raw_epoch_time = result.get("epoch_time")
+ if raw_epoch_time is None:
+ raise UnexpectedResponse(
+ f'Server response does not contain "epoch_time": {result}'
+ )
+
+ try:
+ epoch_time = float(raw_epoch_time)
+ except:
+ raise UnexpectedResponse(
+ f"Could not process epoch time as a float: {raw_epoch_time}"
+ )
+
+ return epoch_time
+
+ def authorize(self, access_token: str) -> None:
+ if not access_token:
+ logger.warning("Setting authorization header to empty token.")
+ self._session.headers.update({"Authorization": f"Bearer {access_token}"})
+
+ def requires_authorization(self):
+ if self._session.headers.get("Authorization") is None:
+ raise Unauthenticated(
+ 'This method requires that you authenticate to the API, either by calling the "authorize" method with an API token or by calling the "login" method.'
+ )
+
+ def login(self, username: str, password: Optional[str] = None) -> str:
+ """
+ Authorizes this client to act as the given user when communicating with the Moonstream API.
+
+ To register an account on the production Moonstream API, go to https://moonstream.to.
+
+ Arguments:
+ username - Username of the user to authenticate as.
+ password - Optional password for the user. If this is not provided, you will be prompted for
+ the password.
+ """
+ if password is None:
+ password = input(f"Moonstream password for {username}: ")
+
+ r = self._session.post(
+ self.api.endpoints[ENDPOINT_TOKEN],
+ data={"username": username, "password": password},
+ )
+ r.raise_for_status()
+
+ token = r.json()
+ self.authorize(token["id"])
+ return token
+
+ def logout(self) -> None:
+ """
+ Logs the current user out of the Moonstream client.
+ """
+ self._session.delete(self.api.endpoints[ENDPOINT_TOKEN])
+ self._session.headers.pop("Authorization")
+
+ def subscription_types(self) -> Dict[str, Any]:
+ """
+ Gets the currently available subscription types on the Moonstream API.
+ """
+ r = self._session.get(self.api.endpoints[ENDPOINT_SUBSCRIPTION_TYPES])
+ r.raise_for_status()
+ return r.json()
+
+ def list_subscriptions(self) -> Dict[str, Any]:
+ """
+ Gets the currently authorized user's subscriptions from the API server.
+ """
+ self.requires_authorization()
+ r = self._session.get(self.api.endpoints[ENDPOINT_SUBSCRIPTIONS])
+ r.raise_for_status()
+ return r.json()
+
+ def create_subscription(
+ self, subscription_type: str, label: str, color: str, specifier: str = ""
+ ) -> Dict[str, Any]:
+ """
+ Creates a subscription.
+
+ Arguments:
+ subscription_type - The type of subscription you would like to create. To see the available subscription
+ types, call the "subscription_types" method on this Moonstream client. This argument must be
+ the "id" if the subscription type you want.
+ label - A label for the subscription. This will identify the subscription to you in your stream.
+ color - A hexadecimal color to associate with the subscription.
+ specifier - A specifier for the subscription, which must correspond to one of the choices in the
+ subscription type. This is optional because some subscription types do not require a specifier.
+
+ Returns: The subscription resource that was created on the backend.
+ """
+ self.requires_authorization()
+ r = self._session.post(
+ self.api.endpoints[ENDPOINT_SUBSCRIPTIONS],
+ data={
+ "subscription_type_id": subscription_type,
+ "label": label,
+ "color": color,
+ "address": specifier,
+ },
+ )
+ r.raise_for_status()
+ return r.json()
+
+ def delete_subscription(self, id: str) -> Dict[str, Any]:
+ """
+ Delete a subscription by ID.
+
+ Arguments:
+ id - ID of the subscription to delete.
+
+ Returns: The subscription resource that was deleted.
+ """
+ self.requires_authorization()
+ r = self._session.delete(f"{self.api.endpoints[ENDPOINT_SUBSCRIPTIONS]}{id}")
+ r.raise_for_status()
+ return r.json()
+
+ def update_subscription(
+ self, id: str, label: Optional[str] = None, color: Optional[str] = None
+ ) -> Dict[str, Any]:
+ """
+ Update a subscription label or color.
+
+ Arguments:
+ label - New label for subscription (optional).
+ color - New color for subscription (optional).
+
+ Returns - If neither label or color are specified, raises a ValueError. Otherwise PUTs the updated
+ information to the server and returns the updated subscription resource.
+ """
+ if label is None and color is None:
+ raise ValueError(
+ "At least one of the arguments to this method should not be None."
+ )
+ self.requires_authorization()
+ data = {}
+ if label is not None:
+ data["label"] = label
+ if color is not None:
+ data["color"] = color
+
+ r = self._session.put(
+ f"{self.api.endpoints[ENDPOINT_SUBSCRIPTIONS]}{id}", data=data
+ )
+ r.raise_for_status()
+ return r.json()
+
+ def latest_events(self, q: str = "") -> List[Dict[str, Any]]:
+ """
+ Returns the latest events in your stream. You can optionally provide a query parameter to
+ constrain the query to specific subscription types or to specific subscriptions.
+
+ Arguments:
+ - q - Optional query (default is the empty string). The syntax to constrain to a particular
+ type of subscription is "type:<subscription_type>". For example, to get the latest event from
+ your Ethereum transaction pool subscriptions, you would use "type:ethereum_txpool".
+
+ Returns: A list of the latest events in your stream.
+ """
+ self.requires_authorization()
+ query_params: Dict[str, str] = {}
+ if q:
+ query_params["q"] = q
+ r = self._session.get(
+ self.api.endpoints[ENDPOINT_STREAMS_LATEST], params=query_params
+ )
+ r.raise_for_status()
+ return r.json()
+
+ def next_event(
+ self, end_time: int, include_end: bool = True, q: str = ""
+ ) -> Optional[Dict[str, Any]]:
+ """
+ Return the earliest event in your stream that occurred after the given end_time.
+
+ Arguments:
+ - end_time - Time after which you want to retrieve the earliest event from your stream.
+ - include_end - If True, the result is the first event that occurred in your stream strictly
+ *after* the end time. If False, then you will get the first event that occurred in your
+ stream *on* or *after* the end time.
+ - q - Optional query to filter over your available subscriptions and subscription types.
+
+ Returns: None if no event has occurred after the given end time, else returns a dictionary
+ representing that event.
+ """
+ self.requires_authorization()
+ query_params: Dict[str, Any] = {
+ "end_time": end_time,
+ "include_end": include_end,
+ }
+ if q:
+ query_params["q"] = q
+ r = self._session.get(
+ self.api.endpoints[ENDPOINT_STREAMS_NEXT], params=query_params
+ )
+ r.raise_for_status()
+ return r.json()
+
+ def previous_event(
+ self, start_time: int, include_start: bool = True, q: str = ""
+ ) -> Optional[Dict[str, Any]]:
+ """
+ Return the latest event in your stream that occurred before the given start_time.
+
+ Arguments:
+ - start_time - Time before which you want to retrieve the latest event from your stream.
+ - include_start - If True, the result is the last event that occurred in your stream strictly
+ *before* the start time. If False, then you will get the last event that occurred in your
+ stream *on* or *before* the start time.
+ - q - Optional query to filter over your available subscriptions and subscription types.
+
+ Returns: None if no event has occurred before the given start time, else returns a dictionary
+ representing that event.
+ """
+ self.requires_authorization()
+ query_params: Dict[str, Any] = {
+ "start_time": start_time,
+ "include_start": include_start,
+ }
+ if q:
+ query_params["q"] = q
+ r = self._session.get(
+ self.api.endpoints[ENDPOINT_STREAMS_PREVIOUS], params=query_params
+ )
+ r.raise_for_status()
+ return r.json()
+
+ def events(
+ self,
+ start_time: int,
+ end_time: int,
+ include_start: bool = False,
+ include_end: bool = False,
+ q: str = "",
+ ) -> Dict[str, Any]:
+ """
+ Return all events in your stream that occurred between the given start and end times.
+
+ Arguments:
+ - start_time - Time after which you want to query your stream.
+ - include_start - Whether or not events that occurred exactly at the start_time should be included in the results.
+ - end_time - Time before which you want to query your stream.
+ - include_end - Whether or not events that occurred exactly at the end_time should be included in the results.
+ - q - Optional query to filter over your available subscriptions and subscription types.
+
+ Returns: A dictionary representing the results of your query.
+ """
+ self.requires_authorization()
+ query_params: Dict[str, Any] = {
+ "start_time": start_time,
+ "include_start": include_start,
+ "end_time": end_time,
+ "include_end": include_end,
+ }
+ if q:
+ query_params["q"] = q
+
+ r = self._session.get(self.api.endpoints[ENDPOINT_STREAMS], params=query_params)
+ r.raise_for_status()
+ return r.json()
+
+
+def client_from_env() -> Moonstream:
+ """
+ Produces a Moonstream client instantiated using the following environment variables:
+ - MOONSTREAM_API_URL: Specifies the url parameter on the Moonstream client
+ - MOONSTREAM_TIMEOUT_SECONDS: Specifies the request timeout
+ - MOONSTREAM_ACCESS_TOKEN: If this environment variable is defined, the client sets this token as
+ the authorization header for all Moonstream API requests.
+ """
+ kwargs: Dict[str, Any] = {}
+
+ url = os.environ.get("MOONSTREAM_API_URL")
+ if url is not None:
+ kwargs["url"] = url
+
+ raw_timeout = os.environ.get("MOONSTREAM_TIMEOUT_SECONDS")
+ timeout: Optional[float] = None
+ if raw_timeout is not None:
+ try:
+ timeout = float(raw_timeout)
+ except:
+ raise ValueError(
+ f"Could not convert MOONSTREAM_TIMEOUT_SECONDS ({raw_timeout}) to float."
+ )
+
+ kwargs["timeout"] = timeout
+
+ moonstream_client = Moonstream(**kwargs)
+
+ access_token = os.environ.get("MOONSTREAM_ACCESS_TOKEN")
+ if access_token is not None:
+ moonstream_client.authorize(access_token)
+
+ return moonstream_client
diff --git a/clients/python/moonstream/test_client.py b/clients/python/moonstream/test_client.py
new file mode 100644
index 00000000..e0c00cf6
--- /dev/null
+++ b/clients/python/moonstream/test_client.py
@@ -0,0 +1,138 @@
+from dataclasses import FrozenInstanceError
+import os
+import unittest
+
+from . import client
+
+
+class TestMoonstreamClient(unittest.TestCase):
+ def test_client_init(self):
+ m = client.Moonstream()
+ self.assertEqual(m.api.url, "https://api.moonstream.to")
+ self.assertIsNone(m.timeout)
+ self.assertGreater(len(m.api.endpoints), 0)
+
+ def test_client_init_with_timeout(self):
+ timeout = 7
+ m = client.Moonstream(timeout=timeout)
+ self.assertEqual(m.api.url, "https://api.moonstream.to")
+ self.assertEqual(m.timeout, timeout)
+ self.assertGreater(len(m.api.endpoints), 0)
+
+ def test_client_with_custom_url_and_timeout(self):
+ timeout = 9
+ url = "https://my.custom.api.url"
+ m = client.Moonstream(url=url, timeout=timeout)
+ self.assertEqual(m.api.url, url)
+ self.assertEqual(m.timeout, timeout)
+ self.assertGreater(len(m.api.endpoints), 0)
+
+ def test_client_with_custom_messy_url_and_timeout(self):
+ timeout = 3.5
+ url = "https://my.custom.api.url/"
+ m = client.Moonstream(url=url, timeout=timeout)
+ self.assertEqual(m.api.url, url)
+ self.assertEqual(m.timeout, timeout)
+ self.assertGreater(len(m.api.endpoints), 0)
+
+ def test_client_with_custom_messy_url_no_protocol_and_timeout(self):
+ timeout = 5.5
+ url = "my.custom.api.url/"
+ m = client.Moonstream(url=url, timeout=timeout)
+ self.assertEqual(m.api.url, url)
+ self.assertEqual(m.timeout, timeout)
+ self.assertGreater(len(m.api.endpoints), 0)
+
+ def test_immutable_api_url(self):
+ m = client.Moonstream()
+ with self.assertRaises(FrozenInstanceError):
+ m.api.url = "lol"
+
+ def test_immutable_api_endpoints(self):
+ m = client.Moonstream()
+ with self.assertRaises(FrozenInstanceError):
+ m.api.endpoints = {}
+
+ def test_mutable_timeout(self):
+ original_timeout = 5.0
+ updated_timeout = 10.5
+ m = client.Moonstream(timeout=original_timeout)
+ self.assertEqual(m.timeout, original_timeout)
+ m.timeout = updated_timeout
+ self.assertEqual(m.timeout, updated_timeout)
+
+
+class TestMoonstreamClientFromEnv(unittest.TestCase):
+ def setUp(self):
+ self.old_moonstream_api_url = os.environ.get("MOONSTREAM_API_URL")
+ self.old_moonstream_timeout_seconds = os.environ.get(
+ "MOONSTREAM_TIMEOUT_SECONDS"
+ )
+ self.old_moonstream_access_token = os.environ.get("MOONSTREAM_ACCESS_TOKEN")
+
+ self.moonstream_api_url = "https://custom.example.com"
+ self.moonstream_timeout_seconds = 15.333333
+ self.moonstream_access_token = "1d431ca4-af9b-4c3a-b7b9-3cc79f3b0900"
+
+ os.environ["MOONSTREAM_API_URL"] = self.moonstream_api_url
+ os.environ["MOONSTREAM_TIMEOUT_SECONDS"] = str(self.moonstream_timeout_seconds)
+ os.environ["MOONSTREAM_ACCESS_TOKEN"] = self.moonstream_access_token
+
+ def tearDown(self) -> None:
+ del os.environ["MOONSTREAM_API_URL"]
+ del os.environ["MOONSTREAM_TIMEOUT_SECONDS"]
+ del os.environ["MOONSTREAM_ACCESS_TOKEN"]
+
+ if self.old_moonstream_api_url is not None:
+ os.environ["MOONSTREAM_API_URL"] = self.old_moonstream_api_url
+ if self.old_moonstream_timeout_seconds is not None:
+ os.environ[
+ "MOONSTREAM_TIMEOUT_SECONDS"
+ ] = self.old_moonstream_timeout_seconds
+ if self.old_moonstream_access_token is not None:
+ os.environ["MOONSTREAM_ACCESS_TOKEN"] = self.old_moonstream_access_token
+
+ def test_client_from_env(self):
+ m = client.client_from_env()
+ self.assertEqual(m.api.url, self.moonstream_api_url)
+ self.assertEqual(m.timeout, self.moonstream_timeout_seconds)
+ self.assertIsNone(m.requires_authorization())
+
+ authorization_header = m._session.headers["Authorization"]
+ self.assertEqual(authorization_header, f"Bearer {self.moonstream_access_token}")
+
+
+class TestMoonstreamEndpoints(unittest.TestCase):
+ def setUp(self):
+ self.url = "https://api.moonstream.to"
+ self.normalized_url = "https://api.moonstream.to"
+
+ def test_moonstream_endpoints(self):
+ endpoints = client.moonstream_endpoints(self.url)
+ self.assertDictEqual(
+ endpoints,
+ {
+ client.ENDPOINT_PING: f"{self.normalized_url}{client.ENDPOINT_PING}",
+ client.ENDPOINT_VERSION: f"{self.normalized_url}{client.ENDPOINT_VERSION}",
+ client.ENDPOINT_NOW: f"{self.normalized_url}{client.ENDPOINT_NOW}",
+ client.ENDPOINT_TOKEN: f"{self.normalized_url}{client.ENDPOINT_TOKEN}",
+ client.ENDPOINT_SUBSCRIPTION_TYPES: f"{self.normalized_url}{client.ENDPOINT_SUBSCRIPTION_TYPES}",
+ client.ENDPOINT_SUBSCRIPTIONS: f"{self.normalized_url}{client.ENDPOINT_SUBSCRIPTIONS}",
+ client.ENDPOINT_STREAMS: f"{self.normalized_url}{client.ENDPOINT_STREAMS}",
+ client.ENDPOINT_STREAMS_LATEST: f"{self.normalized_url}{client.ENDPOINT_STREAMS_LATEST}",
+ client.ENDPOINT_STREAMS_NEXT: f"{self.normalized_url}{client.ENDPOINT_STREAMS_NEXT}",
+ client.ENDPOINT_STREAMS_PREVIOUS: f"{self.normalized_url}{client.ENDPOINT_STREAMS_PREVIOUS}",
+ },
+ )
+
+
+class TestMoonstreamEndpointsMessyURL(TestMoonstreamEndpoints):
+ def setUp(self):
+ self.url = "https://api.moonstream.to/"
+ self.normalized_url = "https://api.moonstream.to"
+
+
+class TestMoonstreamEndpointsMessyURLWithNoProtocol(TestMoonstreamEndpoints):
+ def setUp(self):
+ self.url = "api.moonstream.to/"
+ self.normalized_url = "http://api.moonstream.to"
diff --git a/clients/python/setup.py b/clients/python/setup.py
new file mode 100644
index 00000000..b12b4518
--- /dev/null
+++ b/clients/python/setup.py
@@ -0,0 +1,35 @@
+from setuptools import find_packages, setup
+
+long_description = ""
+with open("README.md") as ifp:
+ long_description = ifp.read()
+
+setup(
+ name="moonstream",
+ version="0.0.2",
+ packages=find_packages(),
+ package_data={"moonstream": ["py.typed"]},
+ install_requires=["requests", "dataclasses; python_version=='3.6'"],
+ extras_require={
+ "dev": [
+ "black",
+ "mypy",
+ "wheel",
+ "types-requests",
+ "types-dataclasses",
+ ],
+ "distribute": ["setuptools", "twine", "wheel"],
+ },
+ description="Moonstream: Open source blockchain analytics",
+ long_description=long_description,
+ long_description_content_type="text/markdown",
+ author="Moonstream",
+ author_email="engineering@moonstream.to",
+ classifiers=[
+ "Development Status :: 3 - Alpha",
+ "Programming Language :: Python",
+ "License :: OSI Approved :: Apache Software License",
+ "Topic :: Software Development :: Libraries",
+ ],
+ url="https://github.com/bugout-dev/moonstream",
+)
diff --git a/clients/python/tag.bash b/clients/python/tag.bash
new file mode 100755
index 00000000..34b87672
--- /dev/null
+++ b/clients/python/tag.bash
@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+set -e
+TAG="clients/python/v$(python setup.py --version)"
+read -r -p "Tag: $TAG -- tag and push (y/n)?" ACCEPT
+if [ "$ACCEPT" = "y" ]
+then
+ echo "Tagging and pushing: $TAG..."
+ git tag "$TAG"
+ git push upstream "$TAG"
+else
+ echo "noop"
+fi
diff --git a/crawlers/deploy/deploy.bash b/crawlers/deploy/deploy.bash
index 3729d184..1f038ce5 100755
--- a/crawlers/deploy/deploy.bash
+++ b/crawlers/deploy/deploy.bash
@@ -19,6 +19,7 @@ ETHEREUM_SYNCHRONIZE_SERVICE="ethereum-synchronize.service"
ETHEREUM_TRENDING_SERVICE="ethereum-trending.service"
ETHEREUM_TRENDING_TIMER="ethereum-trending.service"
ETHEREUM_TXPOOL_SERVICE="ethereum-txpool.service"
+SERVICE_FILE="moonstreamcrawlers.service"
set -eu
@@ -30,6 +31,14 @@ cd "${APP_CRAWLERS_DIR}/ethtxpool"
HOME=/root /usr/local/go/bin/go build -o "${APP_CRAWLERS_DIR}/ethtxpool/ethtxpool" "${APP_CRAWLERS_DIR}/ethtxpool/main.go"
cd "${EXEC_DIR}"
+echo
+echo
+echo "Building executable server of moonstreamcrawlers with Go"
+EXEC_DIR=$(pwd)
+cd "${APP_CRAWLERS_DIR}/server"
+HOME=/root /usr/local/go/bin/go build -o "${APP_CRAWLERS_DIR}/server/moonstreamcrawlers" "${APP_CRAWLERS_DIR}/server/main.go"
+cd "${EXEC_DIR}"
+
echo
echo
echo "Updating Python dependencies"
@@ -82,3 +91,12 @@ chmod 644 "${SCRIPT_DIR}/${ETHEREUM_TXPOOL_SERVICE}"
cp "${SCRIPT_DIR}/${ETHEREUM_TXPOOL_SERVICE}" "/etc/systemd/system/${ETHEREUM_TXPOOL_SERVICE}"
systemctl daemon-reload
systemctl restart "${ETHEREUM_TXPOOL_SERVICE}"
+
+echo
+echo
+echo "Replacing existing moonstreamcrawlers service definition with ${SERVICE_FILE}"
+chmod 644 "${SCRIPT_DIR}/${SERVICE_FILE}"
+cp "${SCRIPT_DIR}/${SERVICE_FILE}" "/etc/systemd/system/${SERVICE_FILE}"
+systemctl daemon-reload
+systemctl restart "${SERVICE_FILE}"
+systemctl status "${SERVICE_FILE}"
diff --git a/crawlers/deploy/moonstreamcrawlers.service b/crawlers/deploy/moonstreamcrawlers.service
new file mode 100644
index 00000000..3b6e7104
--- /dev/null
+++ b/crawlers/deploy/moonstreamcrawlers.service
@@ -0,0 +1,14 @@
+[Unit]
+Description=moonstreamcrawlers-service
+After=network.target
+
+[Service]
+User=ubuntu
+Group=www-data
+WorkingDirectory=/home/ubuntu/moonstream/crawlers/server
+EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
+ExecStart=/home/ubuntu/moonstream/crawlers/server/moonstreamcrawlers -host 0.0.0.0 -port "${MOONSTREAM_CRAWLERS_SERVER_PORT}"
+SyslogIdentifier=moonstreamcrawlers
+
+[Install]
+WantedBy=multi-user.target
\ No newline at end of file
diff --git a/crawlers/server/dev.sh b/crawlers/server/dev.sh
new file mode 100755
index 00000000..174a6682
--- /dev/null
+++ b/crawlers/server/dev.sh
@@ -0,0 +1,9 @@
+#!/usr/bin/env sh
+
+# Expects access to Python environment with the requirements for this project installed.
+set -e
+
+MOONSTREAM_CRAWLERS_SERVER_HOST="${MOONSTREAM_CRAWLERS_SERVER_HOST:-0.0.0.0}"
+MOONSTREAM_CRAWLERS_SERVER_PORT="${MOONSTREAM_CRAWLERS_SERVER_PORT:-8080}"
+
+go run main.go -host "${MOONSTREAM_CRAWLERS_SERVER_HOST}" -port "${MOONSTREAM_CRAWLERS_SERVER_PORT}"
diff --git a/crawlers/server/go.mod b/crawlers/server/go.mod
new file mode 100644
index 00000000..5a8fa99b
--- /dev/null
+++ b/crawlers/server/go.mod
@@ -0,0 +1,3 @@
+module moonstreamdb
+
+go 1.17
diff --git a/crawlers/server/main.go b/crawlers/server/main.go
new file mode 100644
index 00000000..f0bd58db
--- /dev/null
+++ b/crawlers/server/main.go
@@ -0,0 +1,118 @@
+package main
+
+import (
+ "bytes"
+ "encoding/json"
+ "flag"
+ "io/ioutil"
+ "log"
+ "net/http"
+ "os"
+ "strconv"
+ "strings"
+)
+
+var MOONSTREAM_IPC_PATH = os.Getenv("MOONSTREAM_IPC_PATH")
+var MOONSTREAM_CORS_ALLOWED_ORIGINS = os.Getenv("MOONSTREAM_CORS_ALLOWED_ORIGINS")
+
+type GethResponse struct {
+ Result string `json:"result"`
+}
+
+type PingGethResponse struct {
+ CurrentBlock uint64 `json:"current_block"`
+}
+
+type PingResponse struct {
+ Status string `json:"status"`
+}
+
+// Extends handler with allowed CORS policies
+func setupCorsResponse(w *http.ResponseWriter, req *http.Request) {
+ for _, allowedOrigin := range strings.Split(MOONSTREAM_CORS_ALLOWED_ORIGINS, ",") {
+ for _, reqOrigin := range req.Header["Origin"] {
+ if reqOrigin == allowedOrigin {
+ (*w).Header().Set("Access-Control-Allow-Origin", allowedOrigin)
+ }
+ }
+
+ }
+ (*w).Header().Set("Access-Control-Allow-Methods", "GET,OPTIONS")
+}
+
+func ping(w http.ResponseWriter, req *http.Request) {
+ setupCorsResponse(&w, req)
+ log.Printf("%s, %s, %q", req.RemoteAddr, req.Method, req.URL.String())
+ if (*req).Method == "OPTIONS" {
+ return
+ }
+
+ w.Header().Set("Content-Type", "application/json")
+ response := PingResponse{Status: "ok"}
+ json.NewEncoder(w).Encode(response)
+}
+
+func pingGeth(w http.ResponseWriter, req *http.Request) {
+ setupCorsResponse(&w, req)
+ log.Printf("%s, %s, %q", req.RemoteAddr, req.Method, req.URL.String())
+ if (*req).Method == "OPTIONS" {
+ return
+ }
+
+ postBody, err := json.Marshal(map[string]interface{}{
+ "jsonrpc": "2.0",
+ "method": "eth_blockNumber",
+ "params": []string{},
+ "id": 1,
+ })
+ if err != nil {
+ log.Println(err)
+ http.Error(w, http.StatusText(500), 500)
+ return
+ }
+ gethResponse, err := http.Post(MOONSTREAM_IPC_PATH, "application/json",
+ bytes.NewBuffer(postBody))
+ if err != nil {
+ log.Printf("Unable to request geth, error: %v", err)
+ http.Error(w, http.StatusText(500), 500)
+ return
+ }
+ defer gethResponse.Body.Close()
+
+ gethResponseBody, err := ioutil.ReadAll(gethResponse.Body)
+ if err != nil {
+ log.Printf("Unable to read geth response, error: %v", err)
+ http.Error(w, http.StatusText(500), 500)
+ return
+ }
+ var obj GethResponse
+ _ = json.Unmarshal(gethResponseBody, &obj)
+
+ blockNumberHex := strings.Replace(obj.Result, "0x", "", -1)
+ blockNumberStr, err := strconv.ParseUint(blockNumberHex, 16, 64)
+ if err != nil {
+ log.Printf("Unable to parse block number from hex to string, error: %v", err)
+ http.Error(w, http.StatusText(500), 500)
+ return
+ }
+
+ w.Header().Set("Content-Type", "application/json")
+ response := PingGethResponse{CurrentBlock: blockNumberStr}
+ json.NewEncoder(w).Encode(response)
+}
+
+func main() {
+ var listenAddr string
+ var listenPort string
+ flag.StringVar(&listenAddr, "host", "127.0.0.1", "Server listen address")
+ flag.StringVar(&listenPort, "port", "8080", "Server listen port")
+ flag.Parse()
+
+ address := listenAddr + ":" + listenPort
+ log.Printf("Starting server at %s\n", address)
+
+ http.HandleFunc("/ping", ping)
+ http.HandleFunc("/status", pingGeth)
+
+ http.ListenAndServe(address, nil)
+}
diff --git a/crawlers/server/sample.env b/crawlers/server/sample.env
new file mode 100644
index 00000000..053b87b4
--- /dev/null
+++ b/crawlers/server/sample.env
@@ -0,0 +1,3 @@
+export MOONSTREAM_CRAWLERS_SERVER_PORT="8080"
+export MOONSTREAM_IPC_PATH=null
+export MOONSTREAM_CORS_ALLOWED_ORIGINS="http://localhost:3000,https://moonstream.to,https://www.moonstream.to,https://alpha.moonstream.to"
diff --git a/datasets/nfts/nfts/cli.py b/datasets/nfts/nfts/cli.py
index a5d28efc..71d6dc5d 100644
--- a/datasets/nfts/nfts/cli.py
+++ b/datasets/nfts/nfts/cli.py
@@ -7,10 +7,10 @@ from typing import Optional
from moonstreamdb.db import yield_db_session_ctx
+from .enrich import EthereumBatchloader, enrich
from .data import EventType, event_types, nft_event, BlockBounds
from .datastore import setup_database, import_data
-from .derive import current_owners
-from .enrich import EthereumBatchloader, enrich
+from .derive import current_owners, current_market_values, current_values_distribution
from .materialize import create_dataset
@@ -18,6 +18,13 @@ logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
+derive_functions = {
+ "current_owners": current_owners,
+ "current_market_values": current_market_values,
+ "current_values_distribution": current_values_distribution,
+}
+
+
def handle_initdb(args: argparse.Namespace) -> None:
with contextlib.closing(sqlite3.connect(args.datastore)) as conn:
setup_database(conn)
@@ -80,7 +87,15 @@ def handle_enrich(args: argparse.Namespace) -> None:
def handle_derive(args: argparse.Namespace) -> None:
with contextlib.closing(sqlite3.connect(args.datastore)) as moonstream_datastore:
- results = current_owners(moonstream_datastore)
+ calling_functions = []
+ if not args.derive_functions:
+ calling_functions.extend(derive_functions.keys())
+ else:
+ calling_functions.extend(args.derive_functions)
+
+ for function_name in calling_functions:
+ if function_name in calling_functions:
+ derive_functions[function_name](moonstream_datastore)
logger.info("Done!")
@@ -159,6 +174,13 @@ def main() -> None:
required=True,
help="Path to SQLite database representing the dataset",
)
+ parser_derive.add_argument(
+ "-f",
+ "--derive_functions",
+ required=False,
+ nargs="+",
+ help=f"Functions wich will call from derive module availabel {list(derive_functions.keys())}",
+ )
parser_derive.set_defaults(func=handle_derive)
parser_import_data = subcommands.add_parser(
diff --git a/datasets/nfts/nfts/datastore.py b/datasets/nfts/nfts/datastore.py
index 47929ec3..0c59537d 100644
--- a/datasets/nfts/nfts/datastore.py
+++ b/datasets/nfts/nfts/datastore.py
@@ -70,12 +70,12 @@ def select_events_table_query(event_type: EventType) -> str:
SELECT
event_id,
transaction_hash,
- block_number,
nft_address,
token_id,
from_address,
to_address,
transaction_value,
+ block_number,
timestamp
FROM {event_tables[event_type]};
"""
@@ -376,13 +376,14 @@ def import_data(
if event_type == EventType.ERC721:
batch.append(NFTMetadata(*cast(Tuple[str, str, str], row)))
else:
+ # Order matches select query returned by select_events_table_query
(
event_id,
+ transaction_hash,
nft_address,
token_id,
from_address,
to_address,
- transaction_hash,
value,
block_number,
timestamp,
@@ -401,16 +402,16 @@ def import_data(
row,
)
event = NFTEvent(
- event_id,
- event_type, # Original argument to this function
- nft_address,
- token_id,
- from_address,
- to_address,
- transaction_hash,
- value,
- block_number,
- timestamp,
+ event_id=event_id,
+ event_type=event_type, # Original argument to this function
+ nft_address=nft_address,
+ token_id=token_id,
+ from_address=from_address,
+ to_address=to_address,
+ transaction_hash=transaction_hash,
+ value=value,
+ block_number=block_number,
+ timestamp=timestamp,
)
batch.append(event)
diff --git a/datasets/nfts/nfts/derive.py b/datasets/nfts/nfts/derive.py
index dc13c7c4..0eb46cde 100644
--- a/datasets/nfts/nfts/derive.py
+++ b/datasets/nfts/nfts/derive.py
@@ -30,14 +30,33 @@ class LastValue:
return self.value
+class LastNonzeroValue:
+ """
+ Stores the last non-zero value in a given column. This is meant to be used as an aggregate
+ function. We use it, for example, to get the current market value of an NFT (inside a given
+ window of time).
+ """
+
+ def __init__(self):
+ self.value = 0
+
+ def step(self, value):
+ if value != 0:
+ self.value = value
+
+ def finalize(self):
+ return self.value
+
+
def ensure_custom_aggregate_functions(conn: sqlite3.Connection) -> None:
"""
Loads custom aggregate functions to an active SQLite3 connection.
"""
conn.create_aggregate("last_value", 1, LastValue)
+ conn.create_aggregate("last_nonzero_value", 1, LastNonzeroValue)
-def current_owners(conn: sqlite3.Connection) -> List[Tuple]:
+def current_owners(conn: sqlite3.Connection) -> None:
"""
Requires a connection to a dataset in which the raw data (esp. transfers) has already been
loaded.
@@ -46,7 +65,12 @@ def current_owners(conn: sqlite3.Connection) -> List[Tuple]:
drop_existing_current_owners_query = "DROP TABLE IF EXISTS current_owners;"
current_owners_query = """
CREATE TABLE current_owners AS
- SELECT nft_address, token_id, CAST(last_value(to_address) AS TEXT) AS owner FROM transfers
+ SELECT nft_address, token_id, last_value(to_address) AS owner FROM
+ (
+ SELECT * FROM mints
+ UNION ALL
+ SELECT * FROM transfers
+ )
GROUP BY nft_address, token_id;"""
cur = conn.cursor()
try:
@@ -57,3 +81,53 @@ def current_owners(conn: sqlite3.Connection) -> List[Tuple]:
conn.rollback()
logger.error("Could not create derived dataset: current_owners")
logger.error(e)
+
+
+def current_market_values(conn: sqlite3.Connection) -> None:
+ """
+ Requires a connection to a dataset in which the raw data (esp. transfers) has already been
+ loaded.
+ """
+ ensure_custom_aggregate_functions(conn)
+ drop_existing_current_market_values_query = (
+ "DROP TABLE IF EXISTS current_market_values;"
+ )
+ current_market_values_query = """
+ CREATE TABLE current_market_values AS
+ SELECT nft_address, token_id, last_nonzero_value(transaction_value) AS market_value FROM
+ (
+ SELECT * FROM mints
+ UNION ALL
+ SELECT * FROM transfers
+ )
+ GROUP BY nft_address, token_id;"""
+ cur = conn.cursor()
+ try:
+ cur.execute(drop_existing_current_market_values_query)
+ cur.execute(current_market_values_query)
+ conn.commit()
+ except Exception as e:
+ conn.rollback()
+ logger.error("Could not create derived dataset: current_market_values")
+
+
+def current_values_distribution(conn: sqlite3.Connection) -> List[Tuple]:
+ """
+ Requires a connection to a dataset in which current_market_values has already been loaded.
+ """
+ ensure_custom_aggregate_functions(conn)
+ drop_existing_values_distribution_query = (
+ "DROP TABLE IF EXISTS market_values_distribution;"
+ )
+ current_values_distribution_query = """
+ CREATE TABLE market_values_distribution AS
+ select nft_address as address, market_value as value, CUME_DIST() over (PARTITION BY nft_address ORDER BY market_value) as cumulate_value from current_market_values;"""
+ cur = conn.cursor()
+ try:
+ cur.execute(drop_existing_values_distribution_query)
+ cur.execute(current_values_distribution_query)
+ conn.commit()
+ except Exception as e:
+ conn.rollback()
+ logger.error("Could not create derived dataset: current_values_distribution")
+ logger.error(e)
diff --git a/db/deploy/deploy.bash b/db/deploy/deploy.bash
index b2c30312..a35e87b4 100755
--- a/db/deploy/deploy.bash
+++ b/db/deploy/deploy.bash
@@ -5,15 +5,32 @@
# Main
APP_DIR="${APP_DIR:-/home/ubuntu/moonstream}"
APP_DB_SERVER_DIR="${APP_DIR}/db/server"
+SECRETS_DIR="${SECRETS_DIR:-/home/ubuntu/moonstream-secrets}"
SCRIPT_DIR="$(realpath $(dirname $0))"
SERVICE_FILE="${SCRIPT_DIR}/moonstreamdb.service"
set -eu
+echo
+echo
+echo "Retrieving deployment parameters for GCL Secret Manager"
+mkdir -p "${SECRETS_DIR}"
+echo "" > "${SECRETS_DIR}/app.env"
+SECRET_NAMES=$(gcloud beta secrets list --filter="labels.product=moonstream" --format="get(name)")
+for secret in $SECRET_NAMES
+do
+ secret_key=$(echo "${secret}" | awk -F'/' '{print $NF}')
+ secret_val=$(gcloud secrets versions access latest --secret="${secret_key}")
+ echo "${secret_key}=\"${secret_val}\"" >> "${SECRETS_DIR}/app.env"
+done
+
echo
echo
echo "Building executable database server script with Go"
+EXEC_DIR=$(pwd)
+cd "${APP_DB_SERVER_DIR}"
HOME=/root /usr/local/go/bin/go build -o "${APP_DB_SERVER_DIR}/moonstreamdb" "${APP_DB_SERVER_DIR}/main.go"
+cd "${EXEC_DIR}"
echo
echo
diff --git a/db/deploy/moonstreamdb.service b/db/deploy/moonstreamdb.service
index 70e7dfea..5f0b2b6f 100644
--- a/db/deploy/moonstreamdb.service
+++ b/db/deploy/moonstreamdb.service
@@ -6,7 +6,8 @@ After=network.target
User=ubuntu
Group=www-data
WorkingDirectory=/home/ubuntu/moonstream/db/server
-ExecStart=/home/ubuntu/moonstream/db/server/moonstreamdb
+EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
+ExecStart=/home/ubuntu/moonstream/db/server/moonstreamdb -host 0.0.0.0 -port "${MOONSTREAM_DB_SERVER_PORT}"
SyslogIdentifier=moonstreamdb
[Install]
diff --git a/db/server/dev.sh b/db/server/dev.sh
new file mode 100755
index 00000000..8920f68e
--- /dev/null
+++ b/db/server/dev.sh
@@ -0,0 +1,9 @@
+#!/usr/bin/env sh
+
+# Expects access to Python environment with the requirements for this project installed.
+set -e
+
+MOONSTREAM_DB_SERVER_HOST="${MOONSTREAM_DB_SERVER_HOST:-0.0.0.0}"
+MOONSTREAM_DB_SERVER_PORT="${MOONSTREAM_DB_SERVER_PORT:-8080}"
+
+go run main.go -host "${MOONSTREAM_DB_SERVER_HOST}" -port "${MOONSTREAM_DB_SERVER_PORT}"
diff --git a/db/server/go.mod b/db/server/go.mod
index 5a8fa99b..570b3bc2 100644
--- a/db/server/go.mod
+++ b/db/server/go.mod
@@ -1,3 +1,23 @@
module moonstreamdb
go 1.17
+
+require (
+ gorm.io/driver/postgres v1.1.1
+ gorm.io/gorm v1.21.15
+)
+
+require (
+ github.com/jackc/chunkreader/v2 v2.0.1 // indirect
+ github.com/jackc/pgconn v1.10.0 // indirect
+ github.com/jackc/pgio v1.0.0 // indirect
+ github.com/jackc/pgpassfile v1.0.0 // indirect
+ github.com/jackc/pgproto3/v2 v2.1.1 // indirect
+ github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect
+ github.com/jackc/pgtype v1.8.1 // indirect
+ github.com/jackc/pgx/v4 v4.13.0 // indirect
+ github.com/jinzhu/inflection v1.0.0 // indirect
+ github.com/jinzhu/now v1.1.2 // indirect
+ golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 // indirect
+ golang.org/x/text v0.3.7 // indirect
+)
diff --git a/db/server/go.sum b/db/server/go.sum
new file mode 100644
index 00000000..33b809ff
--- /dev/null
+++ b/db/server/go.sum
@@ -0,0 +1,186 @@
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc=
+github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=
+github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I=
+github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ=
+github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
+github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
+github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw=
+github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
+github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/jackc/chunkreader v1.0.0 h1:4s39bBR8ByfqH+DKm8rQA3E1LHZWB9XWcrz8fqaZbe0=
+github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo=
+github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk=
+github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8=
+github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk=
+github.com/jackc/pgconn v0.0.0-20190420214824-7e0022ef6ba3/go.mod h1:jkELnwuX+w9qN5YIfX0fl88Ehu4XC3keFuOJJk9pcnA=
+github.com/jackc/pgconn v0.0.0-20190824142844-760dd75542eb/go.mod h1:lLjNuW/+OfW9/pnVKPazfWOgNfH2aPem8YQ7ilXGvJE=
+github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsUgOEh9hBm+xYTstcNHg7UPMVJqRfQxq4s=
+github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o=
+github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8/2JY=
+github.com/jackc/pgconn v1.9.1-0.20210724152538-d89c8390a530/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI=
+github.com/jackc/pgconn v1.10.0 h1:4EYhlDVEMsJ30nNj0mmgwIUXoq7e9sMJrVC2ED6QlCU=
+github.com/jackc/pgconn v1.10.0/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI=
+github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE=
+github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8=
+github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE=
+github.com/jackc/pgmock v0.0.0-20201204152224-4fe30f7445fd/go.mod h1:hrBW0Enj2AZTNpt/7Y5rr2xe/9Mn757Wtb2xeBzPv2c=
+github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65 h1:DadwsjnMwFjfWc9y5Wi/+Zz7xoE5ALHsRQlOctkOiHc=
+github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65/go.mod h1:5R2h2EEX+qri8jOWMbJCtaPWkrrNc7OHwsp2TCqp7ak=
+github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
+github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
+github.com/jackc/pgproto3 v1.1.0 h1:FYYE4yRw+AgI8wXIinMlNjBbp/UitDJwfj5LqqewP1A=
+github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78=
+github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA=
+github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg=
+github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM=
+github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM=
+github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA=
+github.com/jackc/pgproto3/v2 v2.1.1 h1:7PQ/4gLoqnl87ZxL7xjO0DR5gYuviDCZxQJsUlFW1eI=
+github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA=
+github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg=
+github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E=
+github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg=
+github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc=
+github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw=
+github.com/jackc/pgtype v1.8.1-0.20210724151600-32e20a603178/go.mod h1:C516IlIV9NKqfsMCXTdChteoXmwgUceqaLfjg2e3NlM=
+github.com/jackc/pgtype v1.8.1 h1:9k0IXtdJXHJbyAWQgbWr1lU+MEhPXZz6RIXxfR5oxXs=
+github.com/jackc/pgtype v1.8.1/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4=
+github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y=
+github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM=
+github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc=
+github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs=
+github.com/jackc/pgx/v4 v4.13.0 h1:JCjhT5vmhMAf/YwBHLvrBn4OGdIQBiFG6ym8Zmdx570=
+github.com/jackc/pgx/v4 v4.13.0/go.mod h1:9P4X524sErlaxj0XSGZk7s+LD0eOyu1ZDUrrpznYDF0=
+github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
+github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
+github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
+github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
+github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
+github.com/jinzhu/now v1.1.2 h1:eVKgfIdy9b6zbWBMgFpfDPoAMifwSZagU9HmEU6zgiI=
+github.com/jinzhu/now v1.1.2/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/lib/pq v1.10.2 h1:AqzbZs4ZoCBp+GtejcpCpcxM3zlSMx29dXbUSeVtJb8=
+github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
+github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
+github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
+github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
+github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
+github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU=
+github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc=
+github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
+github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
+github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ=
+github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
+github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
+github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q=
+go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
+go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
+go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
+go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
+go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
+go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4=
+go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU=
+go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA=
+go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
+go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
+go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE=
+golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
+golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 h1:HWj/xjIHfjYU5nVXpTM0s39J9CbLn7Cc5a7IC5rwsMQ=
+golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
+golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gorm.io/driver/postgres v1.1.1 h1:tWLmqYCyaoh89fi7DhM6QggujrOnmfo3H98AzgNAAu0=
+gorm.io/driver/postgres v1.1.1/go.mod h1:tpe2xN7aCst1NUdYyWQyxPtnHC+Zfp6NEux9PXD1OU0=
+gorm.io/gorm v1.21.15 h1:gAyaDoPw0lCyrSFWhBlahbUA1U4P5RViC1uIqoB+1Rk=
+gorm.io/gorm v1.21.15/go.mod h1:F+OptMscr0P2F2qU97WT1WimdH9GaQPoDW7AYd5i2Y0=
+honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
diff --git a/db/server/main.go b/db/server/main.go
index e9eb2811..caae18b1 100644
--- a/db/server/main.go
+++ b/db/server/main.go
@@ -2,25 +2,86 @@ package main
import (
"encoding/json"
- "fmt"
+ "flag"
+ "log"
"net/http"
+ "os"
+ "strings"
+
+ "gorm.io/driver/postgres"
+ "gorm.io/gorm"
)
+var MOONSTREAM_DB_URI = os.Getenv("MOONSTREAM_DB_URI")
+var MOONSTREAM_CORS_ALLOWED_ORIGINS = os.Getenv("MOONSTREAM_CORS_ALLOWED_ORIGINS")
+
type PingResponse struct {
Status string `json:"status"`
}
+type BlockResponse struct {
+ BlockNumber uint64 `json:"block_number"`
+}
+
+// Extends handler with allowed CORS policies
+func setupCorsResponse(w *http.ResponseWriter, req *http.Request) {
+ for _, allowedOrigin := range strings.Split(MOONSTREAM_CORS_ALLOWED_ORIGINS, ",") {
+ for _, reqOrigin := range req.Header["Origin"] {
+ if reqOrigin == allowedOrigin {
+ (*w).Header().Set("Access-Control-Allow-Origin", allowedOrigin)
+ }
+ }
+
+ }
+ (*w).Header().Set("Access-Control-Allow-Methods", "GET,OPTIONS")
+}
+
func ping(w http.ResponseWriter, req *http.Request) {
+ setupCorsResponse(&w, req)
+ log.Printf("%s, %s, %q", req.RemoteAddr, req.Method, req.URL.String())
+ if (*req).Method == "OPTIONS" {
+ return
+ }
+
w.Header().Set("Content-Type", "application/json")
response := PingResponse{Status: "ok"}
json.NewEncoder(w).Encode(response)
}
+func blockLatest(w http.ResponseWriter, req *http.Request) {
+ setupCorsResponse(&w, req)
+ log.Printf("%s, %s, %q", req.RemoteAddr, req.Method, req.URL.String())
+ if (*req).Method == "OPTIONS" {
+ return
+ }
+
+ w.Header().Set("Content-Type", "application/json")
+
+ var latestBlock BlockResponse
+ db, err := gorm.Open(postgres.Open(MOONSTREAM_DB_URI), &gorm.Config{})
+ if err != nil {
+ http.Error(w, http.StatusText(500), 500)
+ return
+ }
+
+ query := "SELECT block_number FROM ethereum_blocks ORDER BY block_number DESC LIMIT 1"
+ db.Raw(query, 1).Scan(&latestBlock.BlockNumber)
+
+ json.NewEncoder(w).Encode(latestBlock)
+}
+
func main() {
- address := "0.0.0.0:8931"
- fmt.Printf("Starting server at %s\n", address)
+ var listenAddr string
+ var listenPort string
+ flag.StringVar(&listenAddr, "host", "127.0.0.1", "Server listen address")
+ flag.StringVar(&listenPort, "port", "8080", "Server listen port")
+ flag.Parse()
+
+ address := listenAddr + ":" + listenPort
+ log.Printf("Starting server at %s\n", address)
http.HandleFunc("/ping", ping)
+ http.HandleFunc("/block/latest", blockLatest)
http.ListenAndServe(address, nil)
}
diff --git a/db/server/sample.env b/db/server/sample.env
new file mode 100644
index 00000000..ef5a4be3
--- /dev/null
+++ b/db/server/sample.env
@@ -0,0 +1,3 @@
+export MOONSTREAM_DB_SERVER_PORT="8080"
+export MOONSTREAM_DB_URI="postgresql://<username>:<password>@<db_host>:<db_port>/<db_name>"
+export MOONSTREAM_CORS_ALLOWED_ORIGINS="http://localhost:3000,https://moonstream.to,https://www.moonstream.to,https://alpha.moonstream.to"
diff --git a/frontend/src/layouts/RootLayout.js b/frontend/src/layouts/RootLayout.js
index 209dbe04..cd66abe9 100644
--- a/frontend/src/layouts/RootLayout.js
+++ b/frontend/src/layouts/RootLayout.js
@@ -59,7 +59,11 @@ const RootLayout = (props) => {
>
Join early. Our first 1000 users get free lifetime access to
blockchain analytics. Contact our team on{" "}
- <Link href={"https://discord.gg/V3tWaP36"} color="orange.900">
+ <Link
+ isExternal
+ href={"https://discord.gg/K56VNUQGvA"}
+ color="orange.900"
+ >
Discord
</Link>
</Text>