Merge branch 'main' into landing-improvements-19.05.2022

2022-06-16 08:27:24 -04:00 · 2022-06-16 08:27:24 -04:00 · c7af5d5243
commit c7af5d5243
--- a/crawlers/deploy/ethereum-missing.service
+++ b/crawlers/deploy/ethereum-missing.service
@ -11,4 +11,5 @@ EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
 ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler \
    --access-id "${NB_CONTROLLER_ACCESS_ID}" \
    blocks missing --blockchain ethereum -n
+CPUWeight=50
 SyslogIdentifier=ethereum-missing
--- a/crawlers/deploy/ethereum-synchronize.service
+++ b/crawlers/deploy/ethereum-synchronize.service
@ -14,6 +14,7 @@ EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
 ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler \
    --access-id "${NB_CONTROLLER_ACCESS_ID}" \
    blocks synchronize --blockchain ethereum -c 6 -j 2
+CPUWeight=90
 SyslogIdentifier=ethereum-synchronize

 [Install]
--- a/crawlers/deploy/ethereum-trending.service
+++ b/crawlers/deploy/ethereum-trending.service
@ -11,4 +11,5 @@ EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
 ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler \
    --access-id "${NB_CONTROLLER_ACCESS_ID}" \
    trending
+CPUWeight=30
 SyslogIdentifier=ethereum-trending
--- a/crawlers/deploy/ethereum-txpool.service
+++ b/crawlers/deploy/ethereum-txpool.service
@ -14,6 +14,7 @@ RestartSec=15s
 ExecStart=/home/ubuntu/moonstream/crawlers/txpool/txpool \
    -blockchain ethereum \
    -access-id "${NB_CONTROLLER_ACCESS_ID}"
+CPUWeight=30
 SyslogIdentifier=ethereum-txpool

 [Install]
--- a/crawlers/deploy/moonworm-unicorns-mainnet.service
+++ b/crawlers/deploy/moonworm-unicorns-mainnet.service
@ -10,11 +10,8 @@ WorkingDirectory=/home/ubuntu
 EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
 Restart=on-failure
 RestartSec=15s
-ExecStart=/home/ubuntu/moonworm-env/bin/python -m moonworm.cli watch-cu \
-    -w "${MOONSTREAM_POLYGON_WEB3_PROVIDER_URI}?access_id=${NB_CONTROLLER_ACCESS_ID}&data_source=blockchain" \
-    -c 0xdC0479CC5BbA033B3e7De9F178607150B3AbCe1f \
-    -d 21418707 \
-    --confirmations 60
+ExecStart=/home/ubuntu/moonworm-env/bin/python -m moonworm.cli watch-cu -w "${MOONSTREAM_POLYGON_WEB3_PROVIDER_URI}?access_id=${NB_CONTROLLER_ACCESS_ID}&data_source=blockchain" -c 0xdC0479CC5BbA033B3e7De9F178607150B3AbCe1f -d 21418707 --confirmations 60
+CPUWeight=70
 SyslogIdentifier=moonworm-unicorns-mainnet

 [Install]
--- a/crawlers/deploy/polygon-missing.service
+++ b/crawlers/deploy/polygon-missing.service
@ -11,4 +11,5 @@ EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
 ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler \
    --access-id "${NB_CONTROLLER_ACCESS_ID}" \
    blocks missing --blockchain polygon -n
+CPUWeight=50
 SyslogIdentifier=polygon-missing
--- a/crawlers/deploy/polygon-moonworm-crawler.service
+++ b/crawlers/deploy/polygon-moonworm-crawler.service
@ -14,6 +14,7 @@ RestartSec=15s
 ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli \
    --access-id "${NB_CONTROLLER_ACCESS_ID}" \
    crawl -b polygon
+CPUWeight=70
 SyslogIdentifier=polygon-moonworm-crawler

 [Install]
--- a/crawlers/deploy/polygon-statistics.service
+++ b/crawlers/deploy/polygon-statistics.service
@ -11,4 +11,5 @@ EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
 ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.stats_worker.dashboard \
    --access-id "${NB_CONTROLLER_ACCESS_ID}" \
    generate --blockchain polygon
+CPUWeight=60
 SyslogIdentifier=polygon-statistics
--- a/crawlers/deploy/polygon-synchronize.service
+++ b/crawlers/deploy/polygon-synchronize.service
@ -14,6 +14,7 @@ EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
 ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler \
    --access-id "${NB_CONTROLLER_ACCESS_ID}" \
    blocks synchronize --blockchain polygon -c 60 -j 2
+CPUWeight=90
 SyslogIdentifier=polygon-synchronize

 [Install]
--- a/crawlers/deploy/polygon-txpool.service
+++ b/crawlers/deploy/polygon-txpool.service
@ -14,6 +14,7 @@ RestartSec=15s
 ExecStart=/home/ubuntu/moonstream/crawlers/txpool/txpool \
    -blockchain polygon \
    -access-id "${NB_CONTROLLER_ACCESS_ID}"
+CPUWeight=30
 SyslogIdentifier=polygon-txpool

 [Install]
--- a/crawlers/deploy/xdai-missing.service
+++ b/crawlers/deploy/xdai-missing.service
@ -11,4 +11,5 @@ EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
 ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler \
    --access-id "${NB_CONTROLLER_ACCESS_ID}" \
    blocks missing --blockchain xdai -n
+CPUWeight=50
 SyslogIdentifier=xdai-missing
--- a/crawlers/deploy/xdai-moonworm-crawler.service
+++ b/crawlers/deploy/xdai-moonworm-crawler.service
@ -13,7 +13,8 @@ Restart=on-failure
 RestartSec=15s
 ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli \
    --access-id "${NB_CONTROLLER_ACCESS_ID}" \
-    crawl -b xdai --start 21704746
+    crawl -b xdai
+CPUWeight=70
 SyslogIdentifier=xdai-moonworm-crawler

 [Install]
--- a/crawlers/deploy/xdai-statistics.service
+++ b/crawlers/deploy/xdai-statistics.service
@ -11,4 +11,5 @@ EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
 ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.stats_worker.dashboard \
    --access-id "${NB_CONTROLLER_ACCESS_ID}" \
    generate --blockchain xdai
+CPUWeight=60
 SyslogIdentifier=xdai-statistics
--- a/crawlers/deploy/xdai-synchronize.service
+++ b/crawlers/deploy/xdai-synchronize.service
@ -14,6 +14,7 @@ EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
 ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler \
    --access-id "${NB_CONTROLLER_ACCESS_ID}" \
    blocks synchronize --blockchain xdai -c 60 -j 2
+CPUWeight=90
 SyslogIdentifier=xdai-synchronize

 [Install]
--- a/crawlers/mooncrawl/sample.env
+++ b/crawlers/mooncrawl/sample.env
@ -17,6 +17,7 @@ export NFT_HUMBUG_TOKEN="<Token_for_nft_crawler>"
 # Blockchain nodes environment variables
 export MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI="https://<connection_path_uri_to_ethereum_node>"
 export MOONSTREAM_POLYGON_WEB3_PROVIDER_URI="https://<connection_path_uri_to_polygon_node>"
+export MOONSTREAM_XDAI_WEB3_PROVIDER_URI="https://<connection_path_uri_to_xdai_node>"
 export NB_CONTROLLER_ACCESS_ID="<access_uuid_for_moonstream_nodebalancer>"

 # AWS environment variables
--- a/db/scm.nix
+++ b/db/scm.nix
@ -4,7 +4,7 @@ let
        (getEnv "SCM_GIT")
        (fetchGit {
            url = "git@gitlab.com:deltaex/schematic.git";
-            rev = "0d9227335ad83e0ed9a62d82375c1e85aadcc08d";
+            rev = "ba5d7b40255e5da9a74e666dd88e309dae40fbd2";
        })
    ];
    scm_repo = head (filter (x: x != "") scm_repos);
@ -15,7 +15,7 @@ let
            (getEnv "MDP_GIT")
            (fetchGit {
                url = "git@gitlab.com:mixrank/mdp.git";
-                rev = "76707e9d08178633471fa3c98ef1b08e1e7bbb1c";
+                rev = "57e5d9903235a171c5be76ebb876d66215e2ed7c";
            })
        ] ++ scm_repos;
    });
--- a/db/sync_conf.yml
+++ b/db/sync_conf.yml
@ -22,3 +22,10 @@
  tablename: polygon_labels
 - namespace: public
  tablename: polygon_transactions
+- namespace: public
+  tablename: xdai_blocks
+- namespace: public
+  tablename: xdai_transactions
+- namespace: public
+  tablename: xdai_labels
+