From 348d115819144969d15d91143527eebafa2e6236 Mon Sep 17 00:00:00 2001 From: Andrey Date: Mon, 13 May 2024 16:20:20 +0300 Subject: [PATCH] Add headers. --- crawlers/mooncrawl/mooncrawl/metadata_crawler/cli.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crawlers/mooncrawl/mooncrawl/metadata_crawler/cli.py b/crawlers/mooncrawl/mooncrawl/metadata_crawler/cli.py index dbda8d6e..b7586f5a 100644 --- a/crawlers/mooncrawl/mooncrawl/metadata_crawler/cli.py +++ b/crawlers/mooncrawl/mooncrawl/metadata_crawler/cli.py @@ -55,8 +55,11 @@ def crawl_uri(metadata_uri: str) -> Any: metadata_uri = metadata_uri.replace( "ipfs://", "https://ipfs.io/ipfs/", 1 ) + req = urllib.request.Request( + metadata_uri, headers={"User-Agent": "Mozilla/5.0"} + ) - response = urllib.request.urlopen(metadata_uri, timeout=10) + response = urllib.request.urlopen(req, timeout=10) if ( metadata_uri.startswith("data:application/json")