feat: take options into account for caching

old-agentic-v1^2
Philipp Burckhardt 2023-06-16 11:30:42 -04:00
rodzic b865b58242
commit 0099d86da5
Nie znaleziono w bazie danych klucza dla tego podpisu
ID klucza GPG: A2C3BCA4F31D1DDD
1 zmienionych plików z 51 dodań i 10 usunięć

Wyświetl plik

@ -7,22 +7,54 @@ const normalizedUrlCache = new QuickLRU<string, string | null>({
maxSize: 4000
})
/**
* Generates a hash string from normalization options.
*
* The hash string is used as a key in the normalized URL cache to avoid re-normalizing the same URL multiple times. The function assumes that the full options object is passed, not just a subset of options.
*
*
* @param options - normalization options object
* @returns hash string representing the options
*/
function hashCustomOptions(options: Required<Options>): string {
let hashString = ''
// Sort keys to ensure the same hash for identical options regardless of their order:
const keys = Object.keys(options).sort()
for (let i = 0; i < keys.length; i++) {
const key = keys[i]
let value = options[key]
if (Array.isArray(value)) {
value = value.map((v) => v.toString()).join(',') // sufficient for RegExp and string arrays
} else if (typeof value === 'boolean') {
value = value ? 'T' : 'F'
}
hashString += `${i}:${value},`
}
return hashString
}
/**
* Normalizes a URL string.
*
* @param url - URL string to normalize
* @param options - options for normalization.
* @returns normalized URL string or null if an invalid URL was passed
*/
export function normalizeUrl(url: string, options?: Options): string | null {
let normalizedUrl: string | null | undefined
let cacheKey
try {
if (!url || (isRelativeUrl(url) && !url.startsWith('//'))) {
return null
}
// TODO: caching doesn't take into account `options`
normalizedUrl = normalizedUrlCache.get(url)
if (normalizedUrl !== undefined) {
return normalizedUrl
}
normalizedUrl = normalizeUrlImpl(url, {
const opts = {
stripWWW: false,
defaultProtocol: 'https',
normalizeProtocol: true,
@ -35,12 +67,21 @@ export function normalizeUrl(url: string, options?: Options): string | null {
removeExplicitPort: true,
sortQueryParameters: true,
...options
})
} as Required<Options>
const optionsHash = hashCustomOptions(opts)
cacheKey = `${url}-${optionsHash}`
normalizedUrl = normalizedUrlCache.get(cacheKey)
if (normalizedUrl !== undefined) {
return normalizedUrl
}
normalizedUrl = normalizeUrlImpl(url, opts)
} catch (err) {
// ignore invalid urls
normalizedUrl = null
}
normalizedUrlCache.set(url, normalizedUrl!)
normalizedUrlCache.set(cacheKey, normalizedUrl!)
return normalizedUrl
}