From 8f42791ee2bdc70063cf846701345cb5aab38033 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=A0t=C4=9Bp=C3=A1n=20=C5=A0korpil?= <stepan@skorpil.cz>
Date: Tue, 22 Nov 2022 18:52:24 +0100
Subject: [PATCH] Added timeout to robots.txt fetching

---
 application/src/Fediverse/RobotsTxt/fetchRobotsTxt.ts | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/application/src/Fediverse/RobotsTxt/fetchRobotsTxt.ts b/application/src/Fediverse/RobotsTxt/fetchRobotsTxt.ts
index a2b3743..13cb1a8 100644
--- a/application/src/Fediverse/RobotsTxt/fetchRobotsTxt.ts
+++ b/application/src/Fediverse/RobotsTxt/fetchRobotsTxt.ts
@@ -1,5 +1,6 @@
 import axios, { AxiosRequestConfig, AxiosResponse } from 'axios'
 import robotsParser from 'robots-parser'
+import { getDefaultTimeoutMilliseconds } from '../getDefaultTimeoutMilliseconds.js'
 import RobotsTxt from './RobotsTxt.js'
 import { RobotsTxtError } from './RobotsTxtError.js'
 
@@ -10,7 +11,10 @@ export default async function fetchRobotsTxt (domain: string): Promise<RobotsTxt
   const url = `https://${domain}/robots.txt`
   let content = ''
   try {
-    const robotsTxt = await axios.get(url)
+    const robotsTxt = await axios.get(url, {
+      headers: { 'User-Agent': userAgent },
+      timeout: getDefaultTimeoutMilliseconds()
+    })
     content = robotsTxt.data
   } catch (error) {
     console.info('Robots.txt not found', { error, url })