From 2bfec90e872e0bb1fa759b409b3691527d30cfd5 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Tue, 17 May 2022 20:31:01 +0000 Subject: [PATCH 1/3] With backslash systemd throwing error with assignment outside --- crawlers/deploy/moonworm-unicorns-mainnet.service | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/crawlers/deploy/moonworm-unicorns-mainnet.service b/crawlers/deploy/moonworm-unicorns-mainnet.service index e1dc8a64..3e75467f 100644 --- a/crawlers/deploy/moonworm-unicorns-mainnet.service +++ b/crawlers/deploy/moonworm-unicorns-mainnet.service @@ -10,11 +10,7 @@ WorkingDirectory=/home/ubuntu EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env Restart=on-failure RestartSec=15s -ExecStart=/home/ubuntu/moonworm-env/bin/python -m moonworm.cli watch-cu \ - -w "${MOONSTREAM_POLYGON_WEB3_PROVIDER_URI}?access_id=${NB_CONTROLLER_ACCESS_ID}&data_source=blockchain" \ - -c 0xdC0479CC5BbA033B3e7De9F178607150B3AbCe1f \ - -d 21418707 \ - --confirmations 60 +ExecStart=/home/ubuntu/moonworm-env/bin/python -m moonworm.cli watch-cu -w "${MOONSTREAM_POLYGON_WEB3_PROVIDER_URI}?access_id=${NB_CONTROLLER_ACCESS_ID}&data_source=blockchain" -c 0xdC0479CC5BbA033B3e7De9F178607150B3AbCe1f -d 21418707 --confirmations 60 SyslogIdentifier=moonworm-unicorns-mainnet [Install] From 2cfdc2dc99ad7ad8e82c5824faaeba83f39fefb7 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Sat, 11 Jun 2022 12:06:01 +0000 Subject: [PATCH 2/3] Applied CPU Weight --- crawlers/deploy/ethereum-missing.service | 1 + crawlers/deploy/ethereum-synchronize.service | 1 + crawlers/deploy/ethereum-trending.service | 1 + crawlers/deploy/ethereum-txpool.service | 1 + crawlers/deploy/moonworm-unicorns-mainnet.service | 1 + crawlers/deploy/polygon-missing.service | 1 + crawlers/deploy/polygon-moonworm-crawler.service | 1 + crawlers/deploy/polygon-statistics.service | 1 + crawlers/deploy/polygon-synchronize.service | 1 + crawlers/deploy/polygon-txpool.service | 1 + crawlers/deploy/xdai-missing.service | 1 + crawlers/deploy/xdai-moonworm-crawler.service | 1 + crawlers/deploy/xdai-statistics.service | 1 + crawlers/deploy/xdai-synchronize.service | 1 + crawlers/mooncrawl/sample.env | 1 + 15 files changed, 15 insertions(+) diff --git a/crawlers/deploy/ethereum-missing.service b/crawlers/deploy/ethereum-missing.service index e0d4a947..81de4815 100644 --- a/crawlers/deploy/ethereum-missing.service +++ b/crawlers/deploy/ethereum-missing.service @@ -11,4 +11,5 @@ EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler \ --access-id "${NB_CONTROLLER_ACCESS_ID}" \ blocks missing --blockchain ethereum -n +CPUWeight=50 SyslogIdentifier=ethereum-missing diff --git a/crawlers/deploy/ethereum-synchronize.service b/crawlers/deploy/ethereum-synchronize.service index af6a0561..37fd2ad2 100644 --- a/crawlers/deploy/ethereum-synchronize.service +++ b/crawlers/deploy/ethereum-synchronize.service @@ -14,6 +14,7 @@ EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler \ --access-id "${NB_CONTROLLER_ACCESS_ID}" \ blocks synchronize --blockchain ethereum -c 6 -j 2 +CPUWeight=90 SyslogIdentifier=ethereum-synchronize [Install] diff --git a/crawlers/deploy/ethereum-trending.service b/crawlers/deploy/ethereum-trending.service index f875defc..5e29e075 100644 --- a/crawlers/deploy/ethereum-trending.service +++ b/crawlers/deploy/ethereum-trending.service @@ -11,4 +11,5 @@ EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler \ --access-id "${NB_CONTROLLER_ACCESS_ID}" \ trending +CPUWeight=30 SyslogIdentifier=ethereum-trending diff --git a/crawlers/deploy/ethereum-txpool.service b/crawlers/deploy/ethereum-txpool.service index c580e6e9..8d5cae5a 100644 --- a/crawlers/deploy/ethereum-txpool.service +++ b/crawlers/deploy/ethereum-txpool.service @@ -14,6 +14,7 @@ RestartSec=15s ExecStart=/home/ubuntu/moonstream/crawlers/txpool/txpool \ -blockchain ethereum \ -access-id "${NB_CONTROLLER_ACCESS_ID}" +CPUWeight=30 SyslogIdentifier=ethereum-txpool [Install] diff --git a/crawlers/deploy/moonworm-unicorns-mainnet.service b/crawlers/deploy/moonworm-unicorns-mainnet.service index 3e75467f..82644df1 100644 --- a/crawlers/deploy/moonworm-unicorns-mainnet.service +++ b/crawlers/deploy/moonworm-unicorns-mainnet.service @@ -11,6 +11,7 @@ EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env Restart=on-failure RestartSec=15s ExecStart=/home/ubuntu/moonworm-env/bin/python -m moonworm.cli watch-cu -w "${MOONSTREAM_POLYGON_WEB3_PROVIDER_URI}?access_id=${NB_CONTROLLER_ACCESS_ID}&data_source=blockchain" -c 0xdC0479CC5BbA033B3e7De9F178607150B3AbCe1f -d 21418707 --confirmations 60 +CPUWeight=70 SyslogIdentifier=moonworm-unicorns-mainnet [Install] diff --git a/crawlers/deploy/polygon-missing.service b/crawlers/deploy/polygon-missing.service index d65cf10a..3ceb3dc8 100644 --- a/crawlers/deploy/polygon-missing.service +++ b/crawlers/deploy/polygon-missing.service @@ -11,4 +11,5 @@ EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler \ --access-id "${NB_CONTROLLER_ACCESS_ID}" \ blocks missing --blockchain polygon -n +CPUWeight=50 SyslogIdentifier=polygon-missing diff --git a/crawlers/deploy/polygon-moonworm-crawler.service b/crawlers/deploy/polygon-moonworm-crawler.service index e8f988db..93bf4172 100644 --- a/crawlers/deploy/polygon-moonworm-crawler.service +++ b/crawlers/deploy/polygon-moonworm-crawler.service @@ -14,6 +14,7 @@ RestartSec=15s ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli \ --access-id "${NB_CONTROLLER_ACCESS_ID}" \ crawl -b polygon +CPUWeight=70 SyslogIdentifier=polygon-moonworm-crawler [Install] diff --git a/crawlers/deploy/polygon-statistics.service b/crawlers/deploy/polygon-statistics.service index 06f28681..61f1ea28 100644 --- a/crawlers/deploy/polygon-statistics.service +++ b/crawlers/deploy/polygon-statistics.service @@ -11,4 +11,5 @@ EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.stats_worker.dashboard \ --access-id "${NB_CONTROLLER_ACCESS_ID}" \ generate --blockchain polygon +CPUWeight=60 SyslogIdentifier=polygon-statistics diff --git a/crawlers/deploy/polygon-synchronize.service b/crawlers/deploy/polygon-synchronize.service index 51f8a374..b217fd96 100644 --- a/crawlers/deploy/polygon-synchronize.service +++ b/crawlers/deploy/polygon-synchronize.service @@ -14,6 +14,7 @@ EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler \ --access-id "${NB_CONTROLLER_ACCESS_ID}" \ blocks synchronize --blockchain polygon -c 60 -j 2 +CPUWeight=90 SyslogIdentifier=polygon-synchronize [Install] diff --git a/crawlers/deploy/polygon-txpool.service b/crawlers/deploy/polygon-txpool.service index fb07b49f..b63c1d6b 100644 --- a/crawlers/deploy/polygon-txpool.service +++ b/crawlers/deploy/polygon-txpool.service @@ -14,6 +14,7 @@ RestartSec=15s ExecStart=/home/ubuntu/moonstream/crawlers/txpool/txpool \ -blockchain polygon \ -access-id "${NB_CONTROLLER_ACCESS_ID}" +CPUWeight=30 SyslogIdentifier=polygon-txpool [Install] diff --git a/crawlers/deploy/xdai-missing.service b/crawlers/deploy/xdai-missing.service index 69155551..9866ea3b 100644 --- a/crawlers/deploy/xdai-missing.service +++ b/crawlers/deploy/xdai-missing.service @@ -11,4 +11,5 @@ EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler \ --access-id "${NB_CONTROLLER_ACCESS_ID}" \ blocks missing --blockchain xdai -n +CPUWeight=50 SyslogIdentifier=xdai-missing \ No newline at end of file diff --git a/crawlers/deploy/xdai-moonworm-crawler.service b/crawlers/deploy/xdai-moonworm-crawler.service index ef8f3cc6..669cc260 100644 --- a/crawlers/deploy/xdai-moonworm-crawler.service +++ b/crawlers/deploy/xdai-moonworm-crawler.service @@ -14,6 +14,7 @@ RestartSec=15s ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli \ --access-id "${NB_CONTROLLER_ACCESS_ID}" \ crawl -b xdai --start 21704746 +CPUWeight=70 SyslogIdentifier=xdai-moonworm-crawler [Install] diff --git a/crawlers/deploy/xdai-statistics.service b/crawlers/deploy/xdai-statistics.service index bef114e1..74c6411c 100644 --- a/crawlers/deploy/xdai-statistics.service +++ b/crawlers/deploy/xdai-statistics.service @@ -11,4 +11,5 @@ EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.stats_worker.dashboard \ --access-id "${NB_CONTROLLER_ACCESS_ID}" \ generate --blockchain xdai +CPUWeight=60 SyslogIdentifier=xdai-statistics diff --git a/crawlers/deploy/xdai-synchronize.service b/crawlers/deploy/xdai-synchronize.service index 1b33607e..32b8adb9 100644 --- a/crawlers/deploy/xdai-synchronize.service +++ b/crawlers/deploy/xdai-synchronize.service @@ -14,6 +14,7 @@ EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler \ --access-id "${NB_CONTROLLER_ACCESS_ID}" \ blocks synchronize --blockchain xdai -c 60 -j 2 +CPUWeight=90 SyslogIdentifier=xdai-synchronize [Install] diff --git a/crawlers/mooncrawl/sample.env b/crawlers/mooncrawl/sample.env index 287e1989..d5380c65 100644 --- a/crawlers/mooncrawl/sample.env +++ b/crawlers/mooncrawl/sample.env @@ -17,6 +17,7 @@ export NFT_HUMBUG_TOKEN="" # Blockchain nodes environment variables export MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI="https://" export MOONSTREAM_POLYGON_WEB3_PROVIDER_URI="https://" +export MOONSTREAM_XDAI_WEB3_PROVIDER_URI="https://" export NB_CONTROLLER_ACCESS_ID="" # AWS environment variables From b3fe75ae345c359369b68974db92b7d6e87e12f1 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Sat, 11 Jun 2022 12:07:20 +0000 Subject: [PATCH 3/3] Removed start block for moonworm xdai crawler --- crawlers/deploy/xdai-moonworm-crawler.service | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crawlers/deploy/xdai-moonworm-crawler.service b/crawlers/deploy/xdai-moonworm-crawler.service index 669cc260..17b4d9dd 100644 --- a/crawlers/deploy/xdai-moonworm-crawler.service +++ b/crawlers/deploy/xdai-moonworm-crawler.service @@ -13,7 +13,7 @@ Restart=on-failure RestartSec=15s ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli \ --access-id "${NB_CONTROLLER_ACCESS_ID}" \ - crawl -b xdai --start 21704746 + crawl -b xdai CPUWeight=70 SyslogIdentifier=xdai-moonworm-crawler