From 46c708db4ce88babd1d48f87f5723b6a0837a12a Mon Sep 17 00:00:00 2001 From: Andrey Date: Tue, 2 Apr 2024 13:56:44 +0300 Subject: [PATCH] Add ipfs gateaway for metadata crawler. --- crawlers/mooncrawl/mooncrawl/metadata_crawler/cli.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/crawlers/mooncrawl/mooncrawl/metadata_crawler/cli.py b/crawlers/mooncrawl/mooncrawl/metadata_crawler/cli.py index 81d68c55..dbda8d6e 100644 --- a/crawlers/mooncrawl/mooncrawl/metadata_crawler/cli.py +++ b/crawlers/mooncrawl/mooncrawl/metadata_crawler/cli.py @@ -50,6 +50,12 @@ def crawl_uri(metadata_uri: str) -> Any: result = None while retry < 3: try: + + if metadata_uri.startswith("ipfs://"): + metadata_uri = metadata_uri.replace( + "ipfs://", "https://ipfs.io/ipfs/", 1 + ) + response = urllib.request.urlopen(metadata_uri, timeout=10) if (