Merge pull request #1098 from moonstream-to/add-xai-state-scripts

Add xai state scripts

crawlers/deploy/deploy-state.bash

@@ -46,6 +46,23 @@ ZKSYNC_ERA_STATE_TIMER_FILE="zksync-era-state.timer"
 ZKSYNC_ERA_STATE_CLEAN_SERVICE_FILE="zksync-era-state-clean.service"
 ZKSYNC_ERA_STATE_CLEAN_TIMER_FILE="zksync-era-state-clean.timer"
 
+
+# Xai
+XAI_STATE_SERVICE_FILE="xai-state.service"
+XAI_STATE_TIMER_FILE="xai-state.timer"
+XAI_STATE_CLEAN_SERVICE_FILE="xai-state-clean.service"
+XAI_STATE_CLEAN_TIMER_FILE="xai-state-clean.timer"
+XAI_METADATA_SERVICE_FILE="xai-metadata.service"
+XAI_METADATA_TIMER_FILE="xai-metadata.timer"
+
+# Xai sepolia
+XAI_SEPOLIA_STATE_SERVICE_FILE="xai-sepolia-state.service"
+XAI_SEPOLIA_STATE_TIMER_FILE="xai-sepolia-state.timer"
+XAI_SEPOLIA_STATE_CLEAN_SERVICE_FILE="xai-sepolia-state-clean.service"
+XAI_SEPOLIA_STATE_CLEAN_TIMER_FILE="xai-sepolia-state-clean.timer"
+XAI_SEPOLIA_METADATA_SERVICE_FILE="xai-sepolia-metadata.service"
+XAI_SEPOLIA_METADATA_TIMER_FILE="xai-sepolia-metadata.timer"
+
 set -eu
 
 echo
@@ -156,3 +173,58 @@ cp "${SCRIPT_DIR}/${ZKSYNC_ERA_STATE_CLEAN_SERVICE_FILE}" "/home/ubuntu/.config/
 cp "${SCRIPT_DIR}/${ZKSYNC_ERA_STATE_CLEAN_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${ZKSYNC_ERA_STATE_CLEAN_TIMER_FILE}"
 XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
 XDG_RUNTIME_DIR="/run/user/1000"  systemctl --user restart --no-block "${ZKSYNC_ERA_STATE_CLEAN_TIMER_FILE}"
+
+
+echo
+echo
+echo -e "${PREFIX_INFO} Replacing existing Xai state service and timer with: ${XAI_STATE_SERVICE_FILE}, ${XAI_STATE_TIMER_FILE}"
+chmod 644 "${SCRIPT_DIR}/${XAI_STATE_SERVICE_FILE}" "${SCRIPT_DIR}/${XAI_STATE_TIMER_FILE}"
+cp "${SCRIPT_DIR}/${XAI_STATE_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${XAI_STATE_SERVICE_FILE}"
+cp "${SCRIPT_DIR}/${XAI_STATE_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${XAI_STATE_TIMER_FILE}"
+XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
+XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${XAI_STATE_TIMER_FILE}"
+
+echo
+echo
+echo -e "${PREFIX_INFO} Replacing existing Xai state clean service and timer with: ${XAI_STATE_CLEAN_SERVICE_FILE}, ${XAI_STATE_CLEAN_TIMER_FILE}"
+chmod 644 "${SCRIPT_DIR}/${XAI_STATE_CLEAN_SERVICE_FILE}" "${SCRIPT_DIR}/${XAI_STATE_CLEAN_TIMER_FILE}"
+cp "${SCRIPT_DIR}/${XAI_STATE_CLEAN_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${XAI_STATE_CLEAN_SERVICE_FILE}"
+cp "${SCRIPT_DIR}/${XAI_STATE_CLEAN_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${XAI_STATE_CLEAN_TIMER_FILE}"
+XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
+XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${XAI_STATE_CLEAN_TIMER_FILE}"
+
+echo
+echo
+echo -e "${PREFIX_INFO} Replacing existing Xai metadata service and timer with: ${XAI_METADATA_SERVICE_FILE}, ${XAI_METADATA_TIMER_FILE}"
+chmod 644 "${SCRIPT_DIR}/${XAI_METADATA_SERVICE_FILE}" "${SCRIPT_DIR}/${XAI_METADATA_TIMER_FILE}"
+cp "${SCRIPT_DIR}/${XAI_METADATA_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${XAI_METADATA_SERVICE_FILE}"
+cp "${SCRIPT_DIR}/${XAI_METADATA_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${XAI_METADATA_TIMER_FILE}"
+XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
+XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${XAI_METADATA_TIMER_FILE}"
+
+echo
+echo
+echo -e "${PREFIX_INFO} Replacing existing Xai sepolia state service and timer with: ${XAI_SEPOLIA_STATE_SERVICE_FILE}, ${XAI_SEPOLIA_STATE_TIMER_FILE}"
+chmod 644 "${SCRIPT_DIR}/${XAI_SEPOLIA_STATE_SERVICE_FILE}" "${SCRIPT_DIR}/${XAI_SEPOLIA_STATE_TIMER_FILE}"
+cp "${SCRIPT_DIR}/${XAI_SEPOLIA_STATE_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${XAI_SEPOLIA_STATE_SERVICE_FILE}"
+cp "${SCRIPT_DIR}/${XAI_SEPOLIA_STATE_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${XAI_SEPOLIA_STATE_TIMER_FILE}"
+XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
+XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${XAI_SEPOLIA_STATE_TIMER_FILE}"
+
+echo
+echo
+echo -e "${PREFIX_INFO} Replacing existing Xai sepolia state clean service and timer with: ${XAI_SEPOLIA_STATE_CLEAN_SERVICE_FILE}, ${XAI_SEPOLIA_STATE_CLEAN_TIMER_FILE}"
+chmod 644 "${SCRIPT_DIR}/${XAI_SEPOLIA_STATE_CLEAN_SERVICE_FILE}" "${SCRIPT_DIR}/${XAI_SEPOLIA_STATE_CLEAN_TIMER_FILE}"
+cp "${SCRIPT_DIR}/${XAI_SEPOLIA_STATE_CLEAN_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${XAI_SEPOLIA_STATE_CLEAN_SERVICE_FILE}"
+cp "${SCRIPT_DIR}/${XAI_SEPOLIA_STATE_CLEAN_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${XAI_SEPOLIA_STATE_CLEAN_TIMER_FILE}"
+XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
+XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${XAI_SEPOLIA_STATE_CLEAN_TIMER_FILE}"
+
+echo
+echo
+echo -e "${PREFIX_INFO} Replacing existing Xai sepolia metadata service and timer with: ${XAI_SEPOLIA_METADATA_SERVICE_FILE}, ${XAI_SEPOLIA_METADATA_TIMER_FILE}"
+chmod 644 "${SCRIPT_DIR}/${XAI_SEPOLIA_METADATA_SERVICE_FILE}" "${SCRIPT_DIR}/${XAI_SEPOLIA_METADATA_TIMER_FILE}"
+cp "${SCRIPT_DIR}/${XAI_SEPOLIA_METADATA_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${XAI_SEPOLIA_METADATA_SERVICE_FILE}"
+cp "${SCRIPT_DIR}/${XAI_SEPOLIA_METADATA_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${XAI_SEPOLIA_METADATA_TIMER_FILE}"
+XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
+XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${XAI_SEPOLIA_METADATA_TIMER_FILE}"

crawlers/deploy/deploy.bash

@@ -136,6 +136,12 @@ XAI_MISSING_SERVICE_FILE="xai-missing.service"
 XAI_MISSING_TIMER_FILE="xai-missing.timer"
 XAI_MOONWORM_CRAWLER_SERVICE_FILE="xai-moonworm-crawler.service"
 XAI_SYNCHRONIZE_SERVICE="xai-synchronize.service"
+XAI_STATE_SERVICE_FILE="xai-state.service"
+XAI_STATE_TIMER_FILE="xai-state.timer"
+XAI_STATE_CLEAN_SERVICE_FILE="xai-state-clean.service"
+XAI_STATE_CLEAN_TIMER_FILE="xai-state-clean.timer"
+XAI_METADATA_SERVICE_FILE="xai-metadata.service"
+XAI_METADATA_TIMER_FILE="xai-metadata.timer"
 XAI_HISTORICAL_CRAWL_EVENTS_SERVICE_FILE="xai-historical-crawl-events.service"
 XAI_HISTORICAL_CRAWL_EVENTS_TIMER_FILE="xai-historical-crawl-events.timer"
 XAI_HISTORICAL_CRAWL_TRANSACTIONS_SERVICE_FILE="xai-historical-crawl-transactions.service"
@@ -147,6 +153,12 @@ XAI_SEPOLIA_MISSING_SERVICE_FILE="xai-sepolia-missing.service"
 XAI_SEPOLIA_MISSING_TIMER_FILE="xai-sepolia-missing.timer"
 XAI_SEPOLIA_MOONWORM_CRAWLER_SERVICE_FILE="xai-sepolia-moonworm-crawler.service"
 XAI_SEPOLIA_SYNCHRONIZE_SERVICE="xai-sepolia-synchronize.service"
+XAI_SEPOLIA_STATE_SERVICE_FILE="xai-sepolia-state.service"
+XAI_SEPOLIA_STATE_TIMER_FILE="xai-sepolia-state.timer"
+XAI_SEPOLIA_STATE_CLEAN_SERVICE_FILE="xai-sepolia-state-clean.service"
+XAI_SEPOLIA_STATE_CLEAN_TIMER_FILE="xai-sepolia-state-clean.timer"
+XAI_SEPOLIA_METADATA_SERVICE_FILE="xai-sepolia-metadata.service"
+XAI_SEPOLIA_METADATA_TIMER_FILE="xai-sepolia-metadata.timer"
 XAI_SEPOLIA_HISTORICAL_CRAWL_EVENTS_SERVICE_FILE="xai-sepolia-historical-crawl-events.service"
 XAI_SEPOLIA_HISTORICAL_CRAWL_EVENTS_TIMER_FILE="xai-sepolia-historical-crawl-events.timer"
 XAI_SEPOLIA_HISTORICAL_CRAWL_TRANSACTIONS_SERVICE_FILE="xai-sepolia-historical-crawl-transactions.service"
@@ -722,6 +734,33 @@ cp "${SCRIPT_DIR}/${XAI_MOONWORM_CRAWLER_SERVICE_FILE}" "/home/ubuntu/.config/sy
 XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
 XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${XAI_MOONWORM_CRAWLER_SERVICE_FILE}"
 
+echo
+echo
+echo -e "${PREFIX_INFO} Replacing existing Xai state service and timer with: ${XAI_STATE_SERVICE_FILE}, ${XAI_STATE_TIMER_FILE}"
+chmod 644 "${SCRIPT_DIR}/${XAI_STATE_SERVICE_FILE}" "${SCRIPT_DIR}/${XAI_STATE_TIMER_FILE}"
+cp "${SCRIPT_DIR}/${XAI_STATE_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${XAI_STATE_SERVICE_FILE}"
+cp "${SCRIPT_DIR}/${XAI_STATE_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${XAI_STATE_TIMER_FILE}"
+XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
+XDG_RUNTIME_DIR="/run/user/1000"  systemctl --user restart --no-block "${XAI_STATE_TIMER_FILE}"
+
+echo
+echo
+echo -e "${PREFIX_INFO} Replacing existing Xai state clean service and timer with: ${XAI_STATE_CLEAN_SERVICE_FILE}, ${XAI_STATE_CLEAN_TIMER_FILE}"
+chmod 644 "${SCRIPT_DIR}/${XAI_STATE_CLEAN_SERVICE_FILE}" "${SCRIPT_DIR}/${XAI_STATE_CLEAN_TIMER_FILE}"
+cp "${SCRIPT_DIR}/${XAI_STATE_CLEAN_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${XAI_STATE_CLEAN_SERVICE_FILE}"
+cp "${SCRIPT_DIR}/${XAI_STATE_CLEAN_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${XAI_STATE_CLEAN_TIMER_FILE}"
+XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
+XDG_RUNTIME_DIR="/run/user/1000"  systemctl --user restart --no-block "${XAI_STATE_CLEAN_TIMER_FILE}"
+
+echo
+echo
+echo -e "${PREFIX_INFO} Replacing existing Xai metadata service and timer with: ${XAI_METADATA_SERVICE_FILE}, ${XAI_METADATA_TIMER_FILE}"
+chmod 644 "${SCRIPT_DIR}/${XAI_METADATA_SERVICE_FILE}" "${SCRIPT_DIR}/${XAI_METADATA_TIMER_FILE}"
+cp "${SCRIPT_DIR}/${XAI_METADATA_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${XAI_METADATA_SERVICE_FILE}"
+cp "${SCRIPT_DIR}/${XAI_METADATA_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${XAI_METADATA_TIMER_FILE}"
+XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
+XDG_RUNTIME_DIR="/run/user/1000"  systemctl --user restart --no-block "${XAI_METADATA_TIMER_FILE}"
+
 echo
 echo
 echo -e "${PREFIX_INFO} Replacing existing Xai historical transactions crawler service and timer with: ${XAI_HISTORICAL_CRAWL_TRANSACTIONS_SERVICE_FILE}, ${XAI_HISTORICAL_CRAWL_TRANSACTIONS_TIMER_FILE}"
@@ -768,6 +807,34 @@ cp "${SCRIPT_DIR}/${XAI_SEPOLIA_MOONWORM_CRAWLER_SERVICE_FILE}" "/home/ubuntu/.c
 XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
 XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${XAI_SEPOLIA_MOONWORM_CRAWLER_SERVICE_FILE}"
 
+echo
+echo
+echo -e "${PREFIX_INFO} Replacing existing Xai sepolia state service and timer with: ${XAI_SEPOLIA_STATE_SERVICE_FILE}, ${XAI_SEPOLIA_STATE_TIMER_FILE}"
+chmod 644 "${SCRIPT_DIR}/${XAI_SEPOLIA_STATE_SERVICE_FILE}" "${SCRIPT_DIR}/${XAI_SEPOLIA_STATE_TIMER_FILE}"
+cp "${SCRIPT_DIR}/${XAI_SEPOLIA_STATE_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${XAI_SEPOLIA_STATE_SERVICE_FILE}"
+cp "${SCRIPT_DIR}/${XAI_SEPOLIA_STATE_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${XAI_SEPOLIA_STATE_TIMER_FILE}"
+XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
+XDG_RUNTIME_DIR="/run/user/1000"  systemctl --user restart --no-block "${XAI_SEPOLIA_STATE_TIMER_FILE}"
+
+echo
+echo
+echo -e "${PREFIX_INFO} Replacing existing Xai sepolia state clean service and timer with: ${XAI_SEPOLIA_STATE_CLEAN_SERVICE_FILE}, ${XAI_SEPOLIA_STATE_CLEAN_TIMER_FILE}"
+chmod 644 "${SCRIPT_DIR}/${XAI_SEPOLIA_STATE_CLEAN_SERVICE_FILE}" "${SCRIPT_DIR}/${XAI_SEPOLIA_STATE_CLEAN_TIMER_FILE}"
+cp "${SCRIPT_DIR}/${XAI_SEPOLIA_STATE_CLEAN_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${XAI_SEPOLIA_STATE_CLEAN_SERVICE_FILE}"
+cp "${SCRIPT_DIR}/${XAI_SEPOLIA_STATE_CLEAN_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${XAI_SEPOLIA_STATE_CLEAN_TIMER_FILE}"
+XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
+XDG_RUNTIME_DIR="/run/user/1000"  systemctl --user restart --no-block "${XAI_SEPOLIA_STATE_CLEAN_TIMER_FILE}"
+
+echo
+echo
+echo -e "${PREFIX_INFO} Replacing existing Xai sepolia metadata service and timer with: ${XAI_SEPOLIA_METADATA_SERVICE_FILE}, ${XAI_SEPOLIA_METADATA_TIMER_FILE}"
+chmod 644 "${SCRIPT_DIR}/${XAI_SEPOLIA_METADATA_SERVICE_FILE}" "${SCRIPT_DIR}/${XAI_SEPOLIA_METADATA_TIMER_FILE}"
+cp "${SCRIPT_DIR}/${XAI_SEPOLIA_METADATA_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${XAI_SEPOLIA_METADATA_SERVICE_FILE}"
+cp "${SCRIPT_DIR}/${XAI_SEPOLIA_METADATA_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${XAI_SEPOLIA_METADATA_TIMER_FILE}"
+XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
+XDG_RUNTIME_DIR="/run/user/1000"  systemctl --user restart --no-block "${XAI_SEPOLIA_METADATA_TIMER_FILE}"
+
+
 echo
 echo
 echo -e "${PREFIX_INFO} Replacing existing Xai sepolia historical transactions crawler service and timer with: ${XAI_SEPOLIA_HISTORICAL_CRAWL_TRANSACTIONS_SERVICE_FILE}, ${XAI_SEPOLIA_HISTORICAL_CRAWL_TRANSACTIONS_TIMER_FILE}"

crawlers/deploy/xai-metadata.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=Execute metadata crawler
+After=network.target
+
+[Service]
+Type=oneshot
+WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
+EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.metadata_crawler.cli crawl --blockchain xai
+CPUWeight=60
+SyslogIdentifier=xai-metadata

crawlers/deploy/xai-metadata.timer

@@ -0,0 +1,9 @@
+[Unit]
+Description=Execute Xai metadata crawler each 10m
+
+[Timer]
+OnBootSec=20s
+OnUnitActiveSec=60m
+
+[Install]
+WantedBy=timers.target

crawlers/deploy/xai-sepolia-metadata.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=Execute metadata crawler
+After=network.target
+
+[Service]
+Type=oneshot
+WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
+EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.metadata_crawler.cli crawl --blockchain xai_sepolia
+CPUWeight=60
+SyslogIdentifier=xai-sepolia-metadata

crawlers/deploy/xai-sepolia-metadata.timer

@@ -0,0 +1,9 @@
+[Unit]
+Description=Execute Xai sepolia metadata crawler each 10m
+
+[Timer]
+OnBootSec=20s
+OnUnitActiveSec=60m
+
+[Install]
+WantedBy=timers.target

crawlers/deploy/xai-sepolia-state-clean.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=Execute state clean labels crawler
+After=network.target
+
+[Service]
+Type=oneshot
+WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
+EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli clean-state-labels --blockchain xai -N 10000
+CPUWeight=60
+SyslogIdentifier=xai-sepolia-state-clean

crawlers/deploy/xai-sepolia-state-clean.timer

@@ -0,0 +1,9 @@
+[Unit]
+Description=Execute Xai sepolia state clean labels crawler each 25m
+
+[Timer]
+OnBootSec=50s
+OnUnitActiveSec=25m
+
+[Install]
+WantedBy=timers.target

crawlers/deploy/xai-sepolia-state.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=Execute state crawler
+After=network.target
+
+[Service]
+Type=oneshot
+WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
+EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli crawl-jobs --moonstream-token "${MOONSTREAM_PUBLIC_QUERIES_DATA_ACCESS_TOKEN}" --blockchain xai_sepolia --infura 
+CPUWeight=60
+SyslogIdentifier=sepolia-state

crawlers/deploy/xai-sepolia-state.timer

@@ -0,0 +1,9 @@
+[Unit]
+Description=Execute Xai sepolia state crawler each 10m
+
+[Timer]
+OnBootSec=15s
+OnUnitActiveSec=10m
+
+[Install]
+WantedBy=timers.target

crawlers/deploy/xai-state-clean.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=Execute state clean labels crawler
+After=network.target
+
+[Service]
+Type=oneshot
+WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
+EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli clean-state-labels --blockchain xai -N 10000
+CPUWeight=60
+SyslogIdentifier=xai-state-clean

crawlers/deploy/xai-state-clean.timer

@@ -0,0 +1,9 @@
+[Unit]
+Description=Execute Xai state clean labels crawler each 25m
+
+[Timer]
+OnBootSec=50s
+OnUnitActiveSec=25m
+
+[Install]
+WantedBy=timers.target

crawlers/deploy/xai-state.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=Execute state crawler
+After=network.target
+
+[Service]
+Type=oneshot
+WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
+EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli crawl-jobs --moonstream-token "${MOONSTREAM_PUBLIC_QUERIES_DATA_ACCESS_TOKEN}" --blockchain xai --infura 
+CPUWeight=60
+SyslogIdentifier=xai-state

crawlers/deploy/xai-state.timer

@@ -0,0 +1,9 @@
+[Unit]
+Description=Execute Xai state crawler each 10m
+
+[Timer]
+OnBootSec=15s
+OnUnitActiveSec=10m
+
+[Install]
+WantedBy=timers.target

crawlers/mooncrawl/mooncrawl/reports_crawler/cli.py

@@ -476,39 +476,67 @@ def generate_report_nft_dashboard_handler(args: argparse.Namespace):
 
     client = Moonstream()
 
+    reports = {
+        # "cu_nft_dashboard_data": {},
+        "cu_land_nft_dashboard_data": {},
+        "cu_seaport_feed_polygon": {},
+        "cu_breeding_feed": {},
+        "cu_shadowcorns_owners": {},
+        "cu_shadowcorns_feed": {},
+        "cu_previous_day_distributed_rewards": {},
+        "cu_nft_dashboard_data_xai": {
+            "blockchain": "xai",
+            "customer": "CryptoUnicorns",
+        },
+        "cu_shadowcorns_feed_xai": {"blockchain": "xai", "customer": "CryptoUnicorns"},
+        "cu_land_nft_dashboard_data_xai": {
+            "blockchain": "xai",
+            "customer": "CryptoUnicorns",
+        },
+    }
+
     for query in client.list_queries(
         token=args.moonstream_token,
     ).queries:
         params = {}  # type: ignore
 
-        if query.name not in [
+        if query.name not in reports:
-            "cu_nft_dashboard_data",
-            "cu_land_nft_dashboard_data",
-            "cu_seaport_feed_polygon",
-            "cu_breeding_feed",
-            "cu_shadowcorns_owners",
-            "cu_shadowcorns_feed",
-            "cu_previous_day_distributed_rewards",
-        ]:
             continue
         try:
             logger.info(f"Generating report for {query.name}")
-            data = recive_S3_data_from_query(
+
-                client=client,
+            try:
-                token=args.moonstream_token,
+                data = recive_S3_data_from_query(
-                query_name=query.name,
+                    client=client,
-                params=params,
+                    token=args.moonstream_token,
-                time_await=4,
+                    query_name=query.name,
-            )
+                    params=params,
+                    time_await=4,
+                )
+            except Exception as err:
+                logger.error(f"Error while generating report for {query.name}. {err}")
+                continue
 
             logger.info(f"Data recived. Uploading report for {query.name} as json")
 
             # send as json
             ext = "json"
 
+            full_path = []
+
+            if reports[query.name].get("customer"):
+                full_path.append(reports[query.name]["customer"])
+
+            if reports[query.name].get("blockchain"):
+                full_path.append(reports[query.name]["blockchain"])
+
+            full_path.append(query.name)
+
+            path = f"queries/{'/'.join(full_path)}/data.{ext}"
+
             url = client.upload_query_results(
                 json.dumps(data),
-                key=f"queries/{query.name}/data.{ext}",
+                key=path,
                 bucket=MOONSTREAM_S3_PUBLIC_DATA_BUCKET,
             )
 
@@ -517,8 +545,13 @@ def generate_report_nft_dashboard_handler(args: argparse.Namespace):
             logger.info(f"Data recived. Uploading report for {query.name} as csv")
 
             ext = "csv"
+
+            path = f"queries/{'/'.join(full_path)}/data.{ext}"
             csv_buffer = StringIO()
 
+            if len(data["data"]) == 0:
+                continue
+
             dict_csv_writer = csv.DictWriter(
                 csv_buffer, fieldnames=data["data"][0].keys(), delimiter=","
             )
@@ -529,7 +562,7 @@ def generate_report_nft_dashboard_handler(args: argparse.Namespace):
 
             url = client.upload_query_results(
                 data=csv_buffer.getvalue().encode("utf-8"),
-                key=f"queries/{query.name}/data.{ext}",
+                key=path,
                 bucket=MOONSTREAM_S3_PUBLIC_DATA_BUCKET,
             )
 

crawlers/mooncrawl/mooncrawl/version.py

@@ -2,4 +2,4 @@
 Moonstream crawlers version.
 """
 
-MOONCRAWL_VERSION = "0.4.7"
+MOONCRAWL_VERSION = "0.4.8"