Fixed robots.txt fetching

main
Štěpán Škorpil 2022-11-23 23:34:05 +01:00
rodzic 4064db521f
commit 0723c2508d
2 zmienionych plików z 3 dodań i 3 usunięć

Wyświetl plik

@ -15,7 +15,7 @@ export default async function fetchRobotsTxt (domain: string): Promise<RobotsTxt
headers: { 'User-Agent': userAgent },
timeout: getDefaultTimeoutMilliseconds()
})
content = robotsTxt.data
content = String(robotsTxt.data)
} catch (error) {
console.info('Robots.txt not found', { error, url })
}

Wyświetl plik

@ -11,8 +11,8 @@ export const assertSuccessJsonResponse = (
throw new UnexpectedResponseStatusError(expectedStatus, actualStatus)
}
const expectedContentType = 'application/json'
const actualContentType = response.headers['content-type']
const actualContentType = String(response.headers['content-type'])
if (!actualContentType.startsWith(expectedContentType)) {
throw new UnexpectedContentTypeError(expectedContentType, actualContentType)
throw new UnexpectedContentTypeError(actualContentType, expectedContentType)
}
}