diff --git a/crawlers/deploy/deploy.bash b/crawlers/deploy/deploy.bash index 2aa1d355..d0467a87 100755 --- a/crawlers/deploy/deploy.bash +++ b/crawlers/deploy/deploy.bash @@ -2,6 +2,17 @@ # Deployment script - intended to run on Moonstream crawlers server +# Colors +C_RESET='\033[0m' +C_RED='\033[1;31m' +C_GREEN='\033[1;32m' +C_YELLOW='\033[1;33m' + +# Logs +PREFIX_INFO="${C_GREEN}[INFO]${C_RESET} [$(date +%d-%m\ %T)]" +PREFIX_WARN="${C_YELLOW}[WARN]${C_RESET} [$(date +%d-%m\ %T)]" +PREFIX_CRIT="${C_RED}[CRIT]${C_RESET} [$(date +%d-%m\ %T)]" + # Main AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION:-us-east-1}" APP_DIR="${APP_DIR:-/home/ubuntu/moonstream}" @@ -14,17 +25,20 @@ PARAMETERS_ENV_PATH="${SECRETS_DIR}/app.env" AWS_SSM_PARAMETER_PATH="${AWS_SSM_PARAMETER_PATH:-/moonstream/prod}" SCRIPT_DIR="$(realpath $(dirname $0))" PARAMETERS_SCRIPT="${SCRIPT_DIR}/parameters.py" +CHECKENV_REPO_URL="https://raw.githubusercontent.com/bugout-dev/checkenv/main/scripts" +CHECKENV_PARAMETERS_SCRIPT_URL="${CHECKENV_REPO_URL}/parameters.bash" +CHECKENV_NODES_CONNECTIONS_SCRIPT_URL="${CHECKENV_REPO_URL}/nodes-connections.bash" ETHEREUM_SYNCHRONIZE_SERVICE="ethereum-synchronize.service" ETHEREUM_TRENDING_SERVICE="ethereum-trending.service" ETHEREUM_TRENDING_TIMER="ethereum-trending.service" ETHEREUM_TXPOOL_SERVICE="ethereum-txpool.service" -SERVICE_FILE="moonstreamcrawlers.service" +ETHEREUM_CRAWLERS_SERVICE_FILE="moonstreamcrawlers.service" set -eu echo echo -echo "Building executable Ethereum transaction pool crawler script with Go" +echo -e "${PREFIX_INFO} Building executable Ethereum transaction pool crawler script with Go" EXEC_DIR=$(pwd) cd "${APP_CRAWLERS_DIR}/ethtxpool" HOME=/root /usr/local/go/bin/go build -o "${APP_CRAWLERS_DIR}/ethtxpool/ethtxpool" "${APP_CRAWLERS_DIR}/ethtxpool/main.go" @@ -32,7 +46,7 @@ cd "${EXEC_DIR}" echo echo -echo "Building executable server of moonstreamcrawlers with Go" +echo -e "${PREFIX_INFO} Building executable server of moonstreamcrawlers with Go" EXEC_DIR=$(pwd) cd "${APP_CRAWLERS_DIR}/server" HOME=/root /usr/local/go/bin/go build -o "${APP_CRAWLERS_DIR}/server/moonstreamcrawlers" "${APP_CRAWLERS_DIR}/server/main.go" @@ -40,19 +54,29 @@ cd "${EXEC_DIR}" echo echo -echo "Updating Python dependencies" +echo -e "${PREFIX_INFO} Updating Python dependencies" "${PIP}" install --upgrade pip "${PIP}" install -r "${APP_CRAWLERS_DIR}/mooncrawl/requirements.txt" echo echo -echo "Retrieving deployment parameters" +echo -e "${PREFIX_INFO} Retrieving deployment parameters" mkdir -p "${SECRETS_DIR}" AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION}" "${PYTHON}" "${PARAMETERS_SCRIPT}" extract -p "${AWS_SSM_PARAMETER_PATH}" -o "${PARAMETERS_ENV_PATH}" echo echo -echo "Replacing existing Ethereum block with transactions syncronizer service definition with ${ETHEREUM_SYNCHRONIZE_SERVICE}" +echo -e "${PREFIX_INFO} Retrieving addition deployment parameters" +curl -s "${CHECKENV_PARAMETERS_SCRIPT_URL}" | bash /dev/stdin -v -p "moonstream" -o "${PARAMETERS_ENV_PATH}" + +echo +echo +echo -e "${PREFIX_INFO} Updating nodes connection parameters" +curl -s "${CHECKENV_NODES_CONNECTIONS_SCRIPT_URL}" | bash /dev/stdin -v -f "${PARAMETERS_ENV_PATH}" + +echo +echo +echo -e "${PREFIX_INFO} Replacing existing Ethereum block with transactions syncronizer service definition with ${ETHEREUM_SYNCHRONIZE_SERVICE}" chmod 644 "${SCRIPT_DIR}/${ETHEREUM_SYNCHRONIZE_SERVICE}" cp "${SCRIPT_DIR}/${ETHEREUM_SYNCHRONIZE_SERVICE}" "/etc/systemd/system/${ETHEREUM_SYNCHRONIZE_SERVICE}" systemctl daemon-reload @@ -60,7 +84,7 @@ systemctl restart "${ETHEREUM_SYNCHRONIZE_SERVICE}" echo echo -echo "Replacing existing Ethereum trending service and timer with: ${ETHEREUM_TRENDING_SERVICE}, ${ETHEREUM_TRENDING_TIMER}" +echo -e "${PREFIX_INFO} Replacing existing Ethereum trending service and timer with: ${ETHEREUM_TRENDING_SERVICE}, ${ETHEREUM_TRENDING_TIMER}" chmod 644 "${SCRIPT_DIR}/${ETHEREUM_TRENDING_SERVICE}" "${SCRIPT_DIR}/${ETHEREUM_TRENDING_TIMER}" cp "${SCRIPT_DIR}/${ETHEREUM_TRENDING_SERVICE}" "/etc/systemd/system/${ETHEREUM_TRENDING_SERVICE}" cp "${SCRIPT_DIR}/${ETHEREUM_TRENDING_TIMER}" "/etc/systemd/system/${ETHEREUM_TRENDING_TIMER}" @@ -69,7 +93,7 @@ systemctl restart "${ETHEREUM_TRENDING_TIMER}" echo echo -echo "Replacing existing Ethereum transaction pool crawler service definition with ${ETHEREUM_TXPOOL_SERVICE}" +echo -e "${PREFIX_INFO} Replacing existing Ethereum transaction pool crawler service definition with ${ETHEREUM_TXPOOL_SERVICE}" chmod 644 "${SCRIPT_DIR}/${ETHEREUM_TXPOOL_SERVICE}" cp "${SCRIPT_DIR}/${ETHEREUM_TXPOOL_SERVICE}" "/etc/systemd/system/${ETHEREUM_TXPOOL_SERVICE}" systemctl daemon-reload @@ -77,9 +101,10 @@ systemctl restart "${ETHEREUM_TXPOOL_SERVICE}" echo echo -echo "Replacing existing moonstreamcrawlers service definition with ${SERVICE_FILE}" -chmod 644 "${SCRIPT_DIR}/${SERVICE_FILE}" -cp "${SCRIPT_DIR}/${SERVICE_FILE}" "/etc/systemd/system/${SERVICE_FILE}" +echo -e "${PREFIX_INFO} Replacing existing moonstreamcrawlers service definition with ${ETHEREUM_CRAWLERS_SERVICE_FILE}" +chmod 644 "${SCRIPT_DIR}/${ETHEREUM_CRAWLERS_SERVICE_FILE}" +cp "${SCRIPT_DIR}/${ETHEREUM_CRAWLERS_SERVICE_FILE}" "/etc/systemd/system/${ETHEREUM_CRAWLERS_SERVICE_FILE}" systemctl daemon-reload -systemctl restart "${SERVICE_FILE}" -systemctl status "${SERVICE_FILE}" +systemctl restart "${ETHEREUM_CRAWLERS_SERVICE_FILE}" +systemctl status "${ETHEREUM_CRAWLERS_SERVICE_FILE}" + diff --git a/crawlers/ethtxpool/main.go b/crawlers/ethtxpool/main.go index e4aa4f55..6a4c9bc7 100644 --- a/crawlers/ethtxpool/main.go +++ b/crawlers/ethtxpool/main.go @@ -175,14 +175,16 @@ func PollTxpoolContent(gethClient *rpc.Client, interval int, reporter *humbug.Hu } func main() { - var gethConnectionString string var intervalSeconds int - flag.StringVar(&gethConnectionString, "geth", "", "Geth IPC path/RPC url/Websockets URL") flag.IntVar(&intervalSeconds, "interval", 1, "Number of seconds to wait between RPC calls to query the transaction pool (default: 1)") flag.Parse() + var MOONSTREAM_NODE_ETHEREUM_IPC_ADDR = os.Getenv("MOONSTREAM_NODE_ETHEREUM_IPC_ADDR") + var MOONSTREAM_NODE_ETHEREUM_IPC_PORT = os.Getenv("MOONSTREAM_NODE_ETHEREUM_IPC_PORT") + var MOONSTREAM_IPC_PATH = fmt.Sprintf("http://%s:%s", MOONSTREAM_NODE_ETHEREUM_IPC_ADDR, MOONSTREAM_NODE_ETHEREUM_IPC_PORT) + sessionID := uuid.New().String() - + // Humbug crash client to collect errors crashReporter, err := humbugClient(sessionID, "moonstream-crawlers", os.Getenv("HUMBUG_REPORTER_CRAWLERS_TOKEN")) if err != nil { @@ -199,7 +201,7 @@ func main() { }() // Set connection with Ethereum blockchain via geth - gethClient, err := rpc.Dial(gethConnectionString) + gethClient, err := rpc.Dial(MOONSTREAM_IPC_PATH) if err != nil { panic(fmt.Sprintf("Could not connect to geth: %s", err.Error())) } diff --git a/crawlers/ethtxpool/sample.env b/crawlers/ethtxpool/sample.env index 809cc325..5b375379 100644 --- a/crawlers/ethtxpool/sample.env +++ b/crawlers/ethtxpool/sample.env @@ -1,3 +1,5 @@ +export MOONSTREAM_NODE_ETHEREUM_IPC_ADDR="127.0.0.1" +export MOONSTREAM_NODE_ETHEREUM_IPC_PORT="8545" export ETHTXPOOL_HUMBUG_CLIENT_ID="" export ETHTXPOOL_HUMBUG_TOKEN="" export HUMBUG_REPORTER_CRAWLERS_TOKEN="" diff --git a/crawlers/mooncrawl/mooncrawl/settings.py b/crawlers/mooncrawl/mooncrawl/settings.py index 00a8fbd9..fe85b631 100644 --- a/crawlers/mooncrawl/mooncrawl/settings.py +++ b/crawlers/mooncrawl/mooncrawl/settings.py @@ -4,8 +4,16 @@ from typing import cast # Bugout HUMBUG_REPORTER_CRAWLERS_TOKEN = os.environ.get("HUMBUG_REPORTER_CRAWLERS_TOKEN") -# Geth -MOONSTREAM_IPC_PATH = os.environ.get("MOONSTREAM_IPC_PATH", None) +# Geth connection address +MOONSTREAM_NODE_ETHEREUM_IPC_ADDR = os.environ.get( + "MOONSTREAM_NODE_ETHEREUM_IPC_ADDR", "127.0.0.1" +) +MOONSTREAM_NODE_ETHEREUM_IPC_PORT = os.environ.get( + "MOONSTREAM_NODE_ETHEREUM_IPC_PORT", 8545 +) +MOONSTREAM_IPC_PATH = ( + f"http://{MOONSTREAM_NODE_ETHEREUM_IPC_ADDR}:{MOONSTREAM_NODE_ETHEREUM_IPC_PORT}" +) MOONSTREAM_CRAWL_WORKERS = 4 MOONSTREAM_CRAWL_WORKERS_RAW = os.environ.get("MOONSTREAM_CRAWL_WORKERS") diff --git a/crawlers/mooncrawl/sample.env b/crawlers/mooncrawl/sample.env index a49b2bfb..01cdfb64 100644 --- a/crawlers/mooncrawl/sample.env +++ b/crawlers/mooncrawl/sample.env @@ -1,5 +1,6 @@ # Path to IPC socket to use for web3 connections -export MOONSTREAM_IPC_PATH=null +export MOONSTREAM_NODE_ETHEREUM_IPC_ADDR="127.0.0.1" +export MOONSTREAM_NODE_ETHEREUM_IPC_PORT="8545" export MOONSTREAM_CRAWL_WORKERS=4 export MOONSTREAM_DB_URI="postgresql://:@:/" export MOONSTREAM_ETHERSCAN_TOKEN="" @@ -9,4 +10,4 @@ export COINMARKETCAP_API_KEY="" export HUMBUG_REPORTER_CRAWLERS_TOKEN="" export MOONSTREAM_DATA_JOURNAL_ID="" export MOONSTREAM_ADMIN_ACCESS_TOKEN="" -export NFT_HUMBUG_TOKEN="" \ No newline at end of file +export NFT_HUMBUG_TOKEN="" diff --git a/crawlers/server/configs/settings.go b/crawlers/server/configs/settings.go index 03efc939..87e63d79 100644 --- a/crawlers/server/configs/settings.go +++ b/crawlers/server/configs/settings.go @@ -1,11 +1,14 @@ package settings import ( + "fmt" "os" ) // Geth configs -var MOONSTREAM_IPC_PATH = os.Getenv("MOONSTREAM_IPC_PATH") +var MOONSTREAM_NODE_ETHEREUM_IPC_ADDR = os.Getenv("MOONSTREAM_NODE_ETHEREUM_IPC_ADDR") +var MOONSTREAM_NODE_ETHEREUM_IPC_PORT = os.Getenv("MOONSTREAM_NODE_ETHEREUM_IPC_PORT") +var MOONSTREAM_IPC_PATH = fmt.Sprintf("http://%s:%s", MOONSTREAM_NODE_ETHEREUM_IPC_ADDR, MOONSTREAM_NODE_ETHEREUM_IPC_PORT) // CORS var MOONSTREAM_CORS_ALLOWED_ORIGINS = os.Getenv("MOONSTREAM_CORS_ALLOWED_ORIGINS") diff --git a/crawlers/server/sample.env b/crawlers/server/sample.env index c555f436..7be46542 100644 --- a/crawlers/server/sample.env +++ b/crawlers/server/sample.env @@ -1,2 +1,3 @@ -export MOONSTREAM_IPC_PATH=null +export MOONSTREAM_NODE_ETHEREUM_IPC_ADDR="127.0.0.1" +export MOONSTREAM_NODE_ETHEREUM_IPC_PORT="8545" export MOONSTREAM_CORS_ALLOWED_ORIGINS="http://localhost:3000,https://moonstream.to,https://www.moonstream.to,https://alpha.moonstream.to"