mirror
/
moonstream
kopia lustrzana https://github.com/bugout-dev/moonstream
1
0
Forkuj 0
Kod Zgłoszenia Projekty Wydania Wiki Aktywność

Merge pull request #364 from bugout-dev/additional-params-crawlers 

Additional parameters from checkenv with Route53 IP
pull/367/head
Sergei Sumarokov 2021-11-04 13:33:06 +00:00 zatwierdzone przez GitHub
rodzic 7f481ba703 0df96e053c
commit 4b6c9e4d34
Nie znaleziono w bazie danych klucza dla tego podpisu
ID klucza GPG: 4AEE18F83AFDEB23
7 zmienionych plików z 65 dodań i 23 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

51
crawlers/deploy/deploy.bash
Wyświetl plik

@ -2,6 +2,17 @@
# Deployment script - intended to run on Moonstream crawlers server
# Colors
C_RESET='\033[0m'
C_RED='\033[1;31m'
C_GREEN='\033[1;32m'
C_YELLOW='\033[1;33m'
# Logs
PREFIX_INFO="${C_GREEN}[INFO]${C_RESET} [$(date +%d-%m\ %T)]"
PREFIX_WARN="${C_YELLOW}[WARN]${C_RESET} [$(date +%d-%m\ %T)]"
PREFIX_CRIT="${C_RED}[CRIT]${C_RESET} [$(date +%d-%m\ %T)]"
# Main
AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION:-us-east-1}"
APP_DIR="${APP_DIR:-/home/ubuntu/moonstream}"
@ -14,17 +25,20 @@ PARAMETERS_ENV_PATH="${SECRETS_DIR}/app.env"
AWS_SSM_PARAMETER_PATH="${AWS_SSM_PARAMETER_PATH:-/moonstream/prod}"
SCRIPT_DIR="$(realpath $(dirname $0))"
PARAMETERS_SCRIPT="${SCRIPT_DIR}/parameters.py"
CHECKENV_REPO_URL="https://raw.githubusercontent.com/bugout-dev/checkenv/main/scripts"
CHECKENV_PARAMETERS_SCRIPT_URL="${CHECKENV_REPO_URL}/parameters.bash"
CHECKENV_NODES_CONNECTIONS_SCRIPT_URL="${CHECKENV_REPO_URL}/nodes-connections.bash"
ETHEREUM_SYNCHRONIZE_SERVICE="ethereum-synchronize.service"
ETHEREUM_TRENDING_SERVICE="ethereum-trending.service"
ETHEREUM_TRENDING_TIMER="ethereum-trending.service"
ETHEREUM_TXPOOL_SERVICE="ethereum-txpool.service"
SERVICE_FILE="moonstreamcrawlers.service"
ETHEREUM_CRAWLERS_SERVICE_FILE="moonstreamcrawlers.service"
set -eu
echo
echo
echo "Building executable Ethereum transaction pool crawler script with Go"
echo -e "${PREFIX_INFO} Building executable Ethereum transaction pool crawler script with Go"
EXEC_DIR=$(pwd)
cd "${APP_CRAWLERS_DIR}/ethtxpool"
HOME=/root /usr/local/go/bin/go build -o "${APP_CRAWLERS_DIR}/ethtxpool/ethtxpool" "${APP_CRAWLERS_DIR}/ethtxpool/main.go"
@ -32,7 +46,7 @@ cd "${EXEC_DIR}"
echo
echo
echo "Building executable server of moonstreamcrawlers with Go"
echo -e "${PREFIX_INFO} Building executable server of moonstreamcrawlers with Go"
EXEC_DIR=$(pwd)
cd "${APP_CRAWLERS_DIR}/server"
HOME=/root /usr/local/go/bin/go build -o "${APP_CRAWLERS_DIR}/server/moonstreamcrawlers" "${APP_CRAWLERS_DIR}/server/main.go"
@ -40,19 +54,29 @@ cd "${EXEC_DIR}"
echo
echo
echo "Updating Python dependencies"
echo -e "${PREFIX_INFO} Updating Python dependencies"
"${PIP}" install --upgrade pip
"${PIP}" install -r "${APP_CRAWLERS_DIR}/mooncrawl/requirements.txt"
echo
echo
echo "Retrieving deployment parameters"
echo -e "${PREFIX_INFO} Retrieving deployment parameters"
mkdir -p "${SECRETS_DIR}"
AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION}" "${PYTHON}" "${PARAMETERS_SCRIPT}" extract -p "${AWS_SSM_PARAMETER_PATH}" -o "${PARAMETERS_ENV_PATH}"
echo
echo
echo "Replacing existing Ethereum block with transactions syncronizer service definition with ${ETHEREUM_SYNCHRONIZE_SERVICE}"
echo -e "${PREFIX_INFO} Retrieving addition deployment parameters"
curl -s "${CHECKENV_PARAMETERS_SCRIPT_URL}" | bash /dev/stdin -v -p "moonstream" -o "${PARAMETERS_ENV_PATH}"
echo
echo
echo -e "${PREFIX_INFO} Updating nodes connection parameters"
curl -s "${CHECKENV_NODES_CONNECTIONS_SCRIPT_URL}" | bash /dev/stdin -v -f "${PARAMETERS_ENV_PATH}"
echo
echo
echo -e "${PREFIX_INFO} Replacing existing Ethereum block with transactions syncronizer service definition with ${ETHEREUM_SYNCHRONIZE_SERVICE}"
chmod 644 "${SCRIPT_DIR}/${ETHEREUM_SYNCHRONIZE_SERVICE}"
cp "${SCRIPT_DIR}/${ETHEREUM_SYNCHRONIZE_SERVICE}" "/etc/systemd/system/${ETHEREUM_SYNCHRONIZE_SERVICE}"
systemctl daemon-reload
@ -60,7 +84,7 @@ systemctl restart "${ETHEREUM_SYNCHRONIZE_SERVICE}"
echo
echo
echo "Replacing existing Ethereum trending service and timer with: ${ETHEREUM_TRENDING_SERVICE}, ${ETHEREUM_TRENDING_TIMER}"
echo -e "${PREFIX_INFO} Replacing existing Ethereum trending service and timer with: ${ETHEREUM_TRENDING_SERVICE}, ${ETHEREUM_TRENDING_TIMER}"
chmod 644 "${SCRIPT_DIR}/${ETHEREUM_TRENDING_SERVICE}" "${SCRIPT_DIR}/${ETHEREUM_TRENDING_TIMER}"
cp "${SCRIPT_DIR}/${ETHEREUM_TRENDING_SERVICE}" "/etc/systemd/system/${ETHEREUM_TRENDING_SERVICE}"
cp "${SCRIPT_DIR}/${ETHEREUM_TRENDING_TIMER}" "/etc/systemd/system/${ETHEREUM_TRENDING_TIMER}"
@ -69,7 +93,7 @@ systemctl restart "${ETHEREUM_TRENDING_TIMER}"
echo
echo
echo "Replacing existing Ethereum transaction pool crawler service definition with ${ETHEREUM_TXPOOL_SERVICE}"
echo -e "${PREFIX_INFO} Replacing existing Ethereum transaction pool crawler service definition with ${ETHEREUM_TXPOOL_SERVICE}"
chmod 644 "${SCRIPT_DIR}/${ETHEREUM_TXPOOL_SERVICE}"
cp "${SCRIPT_DIR}/${ETHEREUM_TXPOOL_SERVICE}" "/etc/systemd/system/${ETHEREUM_TXPOOL_SERVICE}"
systemctl daemon-reload
@ -77,9 +101,10 @@ systemctl restart "${ETHEREUM_TXPOOL_SERVICE}"
echo
echo
echo "Replacing existing moonstreamcrawlers service definition with ${SERVICE_FILE}"
chmod 644 "${SCRIPT_DIR}/${SERVICE_FILE}"
cp "${SCRIPT_DIR}/${SERVICE_FILE}" "/etc/systemd/system/${SERVICE_FILE}"
echo -e "${PREFIX_INFO} Replacing existing moonstreamcrawlers service definition with ${ETHEREUM_CRAWLERS_SERVICE_FILE}"
chmod 644 "${SCRIPT_DIR}/${ETHEREUM_CRAWLERS_SERVICE_FILE}"
cp "${SCRIPT_DIR}/${ETHEREUM_CRAWLERS_SERVICE_FILE}" "/etc/systemd/system/${ETHEREUM_CRAWLERS_SERVICE_FILE}"
systemctl daemon-reload
systemctl restart "${SERVICE_FILE}"
systemctl status "${SERVICE_FILE}"
systemctl restart "${ETHEREUM_CRAWLERS_SERVICE_FILE}"
systemctl status "${ETHEREUM_CRAWLERS_SERVICE_FILE}"

10
crawlers/ethtxpool/main.go
Wyświetl plik

@ -175,14 +175,16 @@ func PollTxpoolContent(gethClient *rpc.Client, interval int, reporter *humbug.Hu
}
func main() {
var gethConnectionString string
var intervalSeconds int
flag.StringVar(&gethConnectionString, "geth", "", "Geth IPC path/RPC url/Websockets URL")
flag.IntVar(&intervalSeconds, "interval", 1, "Number of seconds to wait between RPC calls to query the transaction pool (default: 1)")
flag.Parse()
var MOONSTREAM_NODE_ETHEREUM_IPC_ADDR = os.Getenv("MOONSTREAM_NODE_ETHEREUM_IPC_ADDR")
var MOONSTREAM_NODE_ETHEREUM_IPC_PORT = os.Getenv("MOONSTREAM_NODE_ETHEREUM_IPC_PORT")
var MOONSTREAM_IPC_PATH = fmt.Sprintf("http://%s:%s", MOONSTREAM_NODE_ETHEREUM_IPC_ADDR, MOONSTREAM_NODE_ETHEREUM_IPC_PORT)
sessionID := uuid.New().String()
// Humbug crash client to collect errors
crashReporter, err := humbugClient(sessionID, "moonstream-crawlers", os.Getenv("HUMBUG_REPORTER_CRAWLERS_TOKEN"))
if err != nil {
@ -199,7 +201,7 @@ func main() {
}()
// Set connection with Ethereum blockchain via geth
gethClient, err := rpc.Dial(gethConnectionString)
gethClient, err := rpc.Dial(MOONSTREAM_IPC_PATH)
if err != nil {
panic(fmt.Sprintf("Could not connect to geth: %s", err.Error()))
}

2
crawlers/ethtxpool/sample.env
Wyświetl plik

@ -1,3 +1,5 @@
export MOONSTREAM_NODE_ETHEREUM_IPC_ADDR="127.0.0.1"
export MOONSTREAM_NODE_ETHEREUM_IPC_PORT="8545"
export ETHTXPOOL_HUMBUG_CLIENT_ID="<client id for the crawling machine>"
export ETHTXPOOL_HUMBUG_TOKEN="<Generate an integration and a Humbug token from https://bugout.dev/account/teams>"
export HUMBUG_REPORTER_CRAWLERS_TOKEN="<Bugout Humbug token for crash reports>"

12
crawlers/mooncrawl/mooncrawl/settings.py
Wyświetl plik

@ -4,8 +4,16 @@ from typing import cast
# Bugout
HUMBUG_REPORTER_CRAWLERS_TOKEN = os.environ.get("HUMBUG_REPORTER_CRAWLERS_TOKEN")
# Geth
MOONSTREAM_IPC_PATH = os.environ.get("MOONSTREAM_IPC_PATH", None)
# Geth connection address
MOONSTREAM_NODE_ETHEREUM_IPC_ADDR = os.environ.get(
"MOONSTREAM_NODE_ETHEREUM_IPC_ADDR", "127.0.0.1"
)
MOONSTREAM_NODE_ETHEREUM_IPC_PORT = os.environ.get(
"MOONSTREAM_NODE_ETHEREUM_IPC_PORT", 8545
)
MOONSTREAM_IPC_PATH = (
f"http://{MOONSTREAM_NODE_ETHEREUM_IPC_ADDR}:{MOONSTREAM_NODE_ETHEREUM_IPC_PORT}"
)
MOONSTREAM_CRAWL_WORKERS = 4
MOONSTREAM_CRAWL_WORKERS_RAW = os.environ.get("MOONSTREAM_CRAWL_WORKERS")

5
crawlers/mooncrawl/sample.env
Wyświetl plik

@ -1,5 +1,6 @@
# Path to IPC socket to use for web3 connections
export MOONSTREAM_IPC_PATH=null
export MOONSTREAM_NODE_ETHEREUM_IPC_ADDR="127.0.0.1"
export MOONSTREAM_NODE_ETHEREUM_IPC_PORT="8545"
export MOONSTREAM_CRAWL_WORKERS=4
export MOONSTREAM_DB_URI="postgresql://<username>:<password>@<db_host>:<db_port>/<db_name>"
export MOONSTREAM_ETHERSCAN_TOKEN="<Token for etherscan>"
@ -9,4 +10,4 @@ export COINMARKETCAP_API_KEY="<API key to parse conmarketcap>"
export HUMBUG_REPORTER_CRAWLERS_TOKEN="<Bugout Humbug token for crash reports>"
export MOONSTREAM_DATA_JOURNAL_ID="<Bugout journal id for moonstream>"
export MOONSTREAM_ADMIN_ACCESS_TOKEN="<Bugout access token for moonstream>"
export NFT_HUMBUG_TOKEN="<Token for nft crawler>"
export NFT_HUMBUG_TOKEN="<Token for nft crawler>"

5
crawlers/server/configs/settings.go
Wyświetl plik

@ -1,11 +1,14 @@
package settings
import (
"fmt"
"os"
)
// Geth configs
var MOONSTREAM_IPC_PATH = os.Getenv("MOONSTREAM_IPC_PATH")
var MOONSTREAM_NODE_ETHEREUM_IPC_ADDR = os.Getenv("MOONSTREAM_NODE_ETHEREUM_IPC_ADDR")
var MOONSTREAM_NODE_ETHEREUM_IPC_PORT = os.Getenv("MOONSTREAM_NODE_ETHEREUM_IPC_PORT")
var MOONSTREAM_IPC_PATH = fmt.Sprintf("http://%s:%s", MOONSTREAM_NODE_ETHEREUM_IPC_ADDR, MOONSTREAM_NODE_ETHEREUM_IPC_PORT)
// CORS
var MOONSTREAM_CORS_ALLOWED_ORIGINS = os.Getenv("MOONSTREAM_CORS_ALLOWED_ORIGINS")

3
crawlers/server/sample.env
Wyświetl plik

@ -1,2 +1,3 @@
export MOONSTREAM_IPC_PATH=null
export MOONSTREAM_NODE_ETHEREUM_IPC_ADDR="127.0.0.1"
export MOONSTREAM_NODE_ETHEREUM_IPC_PORT="8545"
export MOONSTREAM_CORS_ALLOWED_ORIGINS="http://localhost:3000,https://moonstream.to,https://www.moonstream.to,https://alpha.moonstream.to"