kopia lustrzana https://github.com/Stopka/fedicrawl
Fixed robots.txt fetching
rodzic
4064db521f
commit
0723c2508d
|
@ -15,7 +15,7 @@ export default async function fetchRobotsTxt (domain: string): Promise<RobotsTxt
|
||||||
headers: { 'User-Agent': userAgent },
|
headers: { 'User-Agent': userAgent },
|
||||||
timeout: getDefaultTimeoutMilliseconds()
|
timeout: getDefaultTimeoutMilliseconds()
|
||||||
})
|
})
|
||||||
content = robotsTxt.data
|
content = String(robotsTxt.data)
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.info('Robots.txt not found', { error, url })
|
console.info('Robots.txt not found', { error, url })
|
||||||
}
|
}
|
||||||
|
|
|
@ -11,8 +11,8 @@ export const assertSuccessJsonResponse = (
|
||||||
throw new UnexpectedResponseStatusError(expectedStatus, actualStatus)
|
throw new UnexpectedResponseStatusError(expectedStatus, actualStatus)
|
||||||
}
|
}
|
||||||
const expectedContentType = 'application/json'
|
const expectedContentType = 'application/json'
|
||||||
const actualContentType = response.headers['content-type']
|
const actualContentType = String(response.headers['content-type'])
|
||||||
if (!actualContentType.startsWith(expectedContentType)) {
|
if (!actualContentType.startsWith(expectedContentType)) {
|
||||||
throw new UnexpectedContentTypeError(expectedContentType, actualContentType)
|
throw new UnexpectedContentTypeError(actualContentType, expectedContentType)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
Ładowanie…
Reference in New Issue