feat: add diffbot; refactor ky and services; add optional ky caching for tests

.eslintrc

@@ -27,7 +27,12 @@
     "padding-line-between-statements": [
       "error",
       { "blankLine": "always", "prev": ["block", "block-like"], "next": "*" },
-      { "blankLine": "always", "prev": "*", "next": ["block", "block-like"] }
+      { "blankLine": "always", "prev": "*", "next": ["block", "block-like"] },
+      {
+        "blankLine": "any",
+        "prev": ["singleline-const", "singleline-let", "expression"],
+        "next": ["block", "block-like"]
+      }
     ]
   },
   "overrides": [

package.json

@@ -45,14 +45,16 @@
     "debug": "^4.3.4",
     "expr-eval": "^2.0.2",
     "handlebars": "^4.7.7",
+    "is-relative-url": "^4.0.0",
     "js-tiktoken": "^1.0.6",
     "jsonrepair": "^3.1.0",
     "ky": "^0.33.3",
     "nanoid": "^4.0.2",
+    "normalize-url": "^8.0.0",
     "openai-fetch": "^1.5.1",
     "p-map": "^6.0.0",
     "p-retry": "^5.1.2",
-    "p-timeout": "^6.1.1",
+    "p-timeout": "^6.1.2",
     "quick-lru": "^6.1.1",
     "ts-dedent": "^2.2.0",
     "uuid": "^9.0.0",
@@ -64,11 +66,11 @@
     "@keyv/redis": "^2.6.1",
     "@trivago/prettier-plugin-sort-imports": "^4.1.1",
     "@types/debug": "^4.1.8",
-    "@types/node": "^20.3.0",
+    "@types/node": "^20.3.1",
     "@types/sinon": "^10.0.15",
     "@types/uuid": "^9.0.2",
-    "@typescript-eslint/eslint-plugin": "^5.59.9",
+    "@typescript-eslint/eslint-plugin": "^5.59.11",
-    "@typescript-eslint/parser": "^5.59.9",
+    "@typescript-eslint/parser": "^5.59.11",
     "ava": "^5.3.0",
     "del-cli": "^5.0.0",
     "dotenv": "^16.1.4",
@@ -83,7 +85,7 @@
     "p-memoize": "^7.1.1",
     "prettier": "^2.8.8",
     "react": "^18.2.0",
-    "sinon": "^15.1.0",
+    "sinon": "^15.1.2",
     "tsup": "^6.7.0",
     "tsx": "^3.12.7",
     "type-fest": "^3.11.1",

pnpm-lock.yaml

@@ -1,4 +1,4 @@
-lockfileVersion: '6.1'
+lockfileVersion: '6.0'
 
 settings:
   autoInstallPeers: true
@@ -26,6 +26,9 @@ dependencies:
   handlebars:
     specifier: ^4.7.7
     version: 4.7.7
+  is-relative-url:
+    specifier: ^4.0.0
+    version: 4.0.0
   js-tiktoken:
     specifier: ^1.0.6
     version: 1.0.6
@@ -38,6 +41,9 @@ dependencies:
   nanoid:
     specifier: ^4.0.2
     version: 4.0.2
+  normalize-url:
+    specifier: ^8.0.0
+    version: 8.0.0
   openai-fetch:
     specifier: ^1.5.1
     version: 1.5.1
@@ -48,8 +54,8 @@ dependencies:
     specifier: ^5.1.2
     version: 5.1.2
   p-timeout:
-    specifier: ^6.1.1
+    specifier: ^6.1.2
-    version: 6.1.1
+    version: 6.1.2
   quick-lru:
     specifier: ^6.1.1
     version: 6.1.1
@@ -80,8 +86,8 @@ devDependencies:
     specifier: ^4.1.8
     version: 4.1.8
   '@types/node':
-    specifier: ^20.3.0
+    specifier: ^20.3.1
-    version: 20.3.0
+    version: 20.3.1
   '@types/sinon':
     specifier: ^10.0.15
     version: 10.0.15
@@ -89,11 +95,11 @@ devDependencies:
     specifier: ^9.0.2
     version: 9.0.2
   '@typescript-eslint/eslint-plugin':
-    specifier: ^5.59.9
+    specifier: ^5.59.11
-    version: 5.59.9(@typescript-eslint/parser@5.59.9)(eslint@8.42.0)(typescript@5.1.3)
+    version: 5.59.11(@typescript-eslint/parser@5.59.11)(eslint@8.42.0)(typescript@5.1.3)
   '@typescript-eslint/parser':
-    specifier: ^5.59.9
+    specifier: ^5.59.11
-    version: 5.59.9(eslint@8.42.0)(typescript@5.1.3)
+    version: 5.59.11(eslint@8.42.0)(typescript@5.1.3)
   ava:
     specifier: ^5.3.0
     version: 5.3.0
@@ -137,8 +143,8 @@ devDependencies:
     specifier: ^18.2.0
     version: 18.2.0
   sinon:
-    specifier: ^15.1.0
+    specifier: ^15.1.2
-    version: 15.1.0
+    version: 15.1.2
   tsup:
     specifier: ^6.7.0
     version: 6.7.0(typescript@5.1.3)
@@ -692,8 +698,8 @@ packages:
       type-detect: 4.0.8
     dev: true
 
-  /@sinonjs/fake-timers@10.2.0:
+  /@sinonjs/fake-timers@10.1.0:
-    resolution: {integrity: sha512-OPwQlEdg40HAj5KNF8WW6q2KG4Z+cBCZb3m4ninfTZKaBmbIJodviQsDBoYMPHkOyJJMHnOJo5j2+LKDOhOACg==}
+    resolution: {integrity: sha512-w1qd368vtrwttm1PRJWPW1QHlbmHrVDGs1eBH/jZvRPUFS4MNXV9Q33EQdjOdeAxZ7O8+3wM7zxztm2nfUSyKw==}
     dependencies:
       '@sinonjs/commons': 3.0.0
     dev: true
@@ -748,8 +754,8 @@ packages:
     resolution: {integrity: sha512-iiUgKzV9AuaEkZqkOLDIvlQiL6ltuZd9tGcW3gwpnX8JbuiuhFlEGmmFXEXkN50Cvq7Os88IY2v0dkDqXYWVgA==}
     dev: true
 
-  /@types/node@20.3.0:
+  /@types/node@20.3.1:
-    resolution: {integrity: sha512-cumHmIAf6On83X7yP+LrsEyUOf/YlociZelmpRYaGFydoaPdxdt80MAbu6vWerQT2COCp2nPvHdsbD7tHn/YlQ==}
+    resolution: {integrity: sha512-EhcH/wvidPy1WeML3TtYFGR83UzjxeWRen9V402T8aUGYsCHOmfoisV3ZSg03gAFIbLq8TnWOJ0f4cALtnSEUg==}
     dev: true
 
   /@types/normalize-package-data@2.4.1:
@@ -778,8 +784,8 @@ packages:
     resolution: {integrity: sha512-kNnC1GFBLuhImSnV7w4njQkUiJi0ZXUycu1rUaouPqiKlXkh77JKgdRnTAp1x5eBwcIwbtI+3otwzuIDEuDoxQ==}
     dev: true
 
-  /@typescript-eslint/eslint-plugin@5.59.9(@typescript-eslint/parser@5.59.9)(eslint@8.42.0)(typescript@5.1.3):
+  /@typescript-eslint/eslint-plugin@5.59.11(@typescript-eslint/parser@5.59.11)(eslint@8.42.0)(typescript@5.1.3):
-    resolution: {integrity: sha512-4uQIBq1ffXd2YvF7MAvehWKW3zVv/w+mSfRAu+8cKbfj3nwzyqJLNcZJpQ/WZ1HLbJDiowwmQ6NO+63nCA+fqA==}
+    resolution: {integrity: sha512-XxuOfTkCUiOSyBWIvHlUraLw/JT/6Io1365RO6ZuI88STKMavJZPNMU0lFcUTeQXEhHiv64CbxYxBNoDVSmghg==}
     engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0}
     peerDependencies:
       '@typescript-eslint/parser': ^5.0.0
@@ -790,10 +796,10 @@ packages:
         optional: true
     dependencies:
       '@eslint-community/regexpp': 4.5.1
-      '@typescript-eslint/parser': 5.59.9(eslint@8.42.0)(typescript@5.1.3)
+      '@typescript-eslint/parser': 5.59.11(eslint@8.42.0)(typescript@5.1.3)
-      '@typescript-eslint/scope-manager': 5.59.9
+      '@typescript-eslint/scope-manager': 5.59.11
-      '@typescript-eslint/type-utils': 5.59.9(eslint@8.42.0)(typescript@5.1.3)
+      '@typescript-eslint/type-utils': 5.59.11(eslint@8.42.0)(typescript@5.1.3)
-      '@typescript-eslint/utils': 5.59.9(eslint@8.42.0)(typescript@5.1.3)
+      '@typescript-eslint/utils': 5.59.11(eslint@8.42.0)(typescript@5.1.3)
       debug: 4.3.4
       eslint: 8.42.0
       grapheme-splitter: 1.0.4
@@ -806,8 +812,8 @@ packages:
       - supports-color
     dev: true
 
-  /@typescript-eslint/parser@5.59.9(eslint@8.42.0)(typescript@5.1.3):
+  /@typescript-eslint/parser@5.59.11(eslint@8.42.0)(typescript@5.1.3):
-    resolution: {integrity: sha512-FsPkRvBtcLQ/eVK1ivDiNYBjn3TGJdXy2fhXX+rc7czWl4ARwnpArwbihSOHI2Peg9WbtGHrbThfBUkZZGTtvQ==}
+    resolution: {integrity: sha512-s9ZF3M+Nym6CAZEkJJeO2TFHHDsKAM3ecNkLuH4i4s8/RCPnF5JRip2GyviYkeEAcwGMJxkqG9h2dAsnA1nZpA==}
     engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0}
     peerDependencies:
       eslint: ^6.0.0 || ^7.0.0 || ^8.0.0
@@ -816,9 +822,9 @@ packages:
       typescript:
         optional: true
     dependencies:
-      '@typescript-eslint/scope-manager': 5.59.9
+      '@typescript-eslint/scope-manager': 5.59.11
-      '@typescript-eslint/types': 5.59.9
+      '@typescript-eslint/types': 5.59.11
-      '@typescript-eslint/typescript-estree': 5.59.9(typescript@5.1.3)
+      '@typescript-eslint/typescript-estree': 5.59.11(typescript@5.1.3)
       debug: 4.3.4
       eslint: 8.42.0
       typescript: 5.1.3
@@ -826,16 +832,16 @@ packages:
       - supports-color
     dev: true
 
-  /@typescript-eslint/scope-manager@5.59.9:
+  /@typescript-eslint/scope-manager@5.59.11:
-    resolution: {integrity: sha512-8RA+E+w78z1+2dzvK/tGZ2cpGigBZ58VMEHDZtpE1v+LLjzrYGc8mMaTONSxKyEkz3IuXFM0IqYiGHlCsmlZxQ==}
+    resolution: {integrity: sha512-dHFOsxoLFtrIcSj5h0QoBT/89hxQONwmn3FOQ0GOQcLOOXm+MIrS8zEAhs4tWl5MraxCY3ZJpaXQQdFMc2Tu+Q==}
     engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0}
     dependencies:
-      '@typescript-eslint/types': 5.59.9
+      '@typescript-eslint/types': 5.59.11
-      '@typescript-eslint/visitor-keys': 5.59.9
+      '@typescript-eslint/visitor-keys': 5.59.11
     dev: true
 
-  /@typescript-eslint/type-utils@5.59.9(eslint@8.42.0)(typescript@5.1.3):
+  /@typescript-eslint/type-utils@5.59.11(eslint@8.42.0)(typescript@5.1.3):
-    resolution: {integrity: sha512-ksEsT0/mEHg9e3qZu98AlSrONAQtrSTljL3ow9CGej8eRo7pe+yaC/mvTjptp23Xo/xIf2mLZKC6KPv4Sji26Q==}
+    resolution: {integrity: sha512-LZqVY8hMiVRF2a7/swmkStMYSoXMFlzL6sXV6U/2gL5cwnLWQgLEG8tjWPpaE4rMIdZ6VKWwcffPlo1jPfk43g==}
     engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0}
     peerDependencies:
       eslint: '*'
@@ -844,8 +850,8 @@ packages:
       typescript:
         optional: true
     dependencies:
-      '@typescript-eslint/typescript-estree': 5.59.9(typescript@5.1.3)
+      '@typescript-eslint/typescript-estree': 5.59.11(typescript@5.1.3)
-      '@typescript-eslint/utils': 5.59.9(eslint@8.42.0)(typescript@5.1.3)
+      '@typescript-eslint/utils': 5.59.11(eslint@8.42.0)(typescript@5.1.3)
       debug: 4.3.4
       eslint: 8.42.0
       tsutils: 3.21.0(typescript@5.1.3)
@@ -854,13 +860,13 @@ packages:
       - supports-color
     dev: true
 
-  /@typescript-eslint/types@5.59.9:
+  /@typescript-eslint/types@5.59.11:
-    resolution: {integrity: sha512-uW8H5NRgTVneSVTfiCVffBb8AbwWSKg7qcA4Ot3JI3MPCJGsB4Db4BhvAODIIYE5mNj7Q+VJkK7JxmRhk2Lyjw==}
+    resolution: {integrity: sha512-epoN6R6tkvBYSc+cllrz+c2sOFWkbisJZWkOE+y3xHtvYaOE6Wk6B8e114McRJwFRjGvYdJwLXQH5c9osME/AA==}
     engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0}
     dev: true
 
-  /@typescript-eslint/typescript-estree@5.59.9(typescript@5.1.3):
+  /@typescript-eslint/typescript-estree@5.59.11(typescript@5.1.3):
-    resolution: {integrity: sha512-pmM0/VQ7kUhd1QyIxgS+aRvMgw+ZljB3eDb+jYyp6d2bC0mQWLzUDF+DLwCTkQ3tlNyVsvZRXjFyV0LkU/aXjA==}
+    resolution: {integrity: sha512-YupOpot5hJO0maupJXixi6l5ETdrITxeo5eBOeuV7RSKgYdU3G5cxO49/9WRnJq9EMrB7AuTSLH/bqOsXi7wPA==}
     engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0}
     peerDependencies:
       typescript: '*'
@@ -868,8 +874,8 @@ packages:
       typescript:
         optional: true
     dependencies:
-      '@typescript-eslint/types': 5.59.9
+      '@typescript-eslint/types': 5.59.11
-      '@typescript-eslint/visitor-keys': 5.59.9
+      '@typescript-eslint/visitor-keys': 5.59.11
       debug: 4.3.4
       globby: 11.1.0
       is-glob: 4.0.3
@@ -880,8 +886,8 @@ packages:
       - supports-color
     dev: true
 
-  /@typescript-eslint/utils@5.59.9(eslint@8.42.0)(typescript@5.1.3):
+  /@typescript-eslint/utils@5.59.11(eslint@8.42.0)(typescript@5.1.3):
-    resolution: {integrity: sha512-1PuMYsju/38I5Ggblaeb98TOoUvjhRvLpLa1DoTOFaLWqaXl/1iQ1eGurTXgBY58NUdtfTXKP5xBq7q9NDaLKg==}
+    resolution: {integrity: sha512-didu2rHSOMUdJThLk4aZ1Or8IcO3HzCw/ZvEjTTIfjIrcdd5cvSIwwDy2AOlE7htSNp7QIZ10fLMyRCveesMLg==}
     engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0}
     peerDependencies:
       eslint: ^6.0.0 || ^7.0.0 || ^8.0.0
@@ -889,9 +895,9 @@ packages:
       '@eslint-community/eslint-utils': 4.4.0(eslint@8.42.0)
       '@types/json-schema': 7.0.12
       '@types/semver': 7.5.0
-      '@typescript-eslint/scope-manager': 5.59.9
+      '@typescript-eslint/scope-manager': 5.59.11
-      '@typescript-eslint/types': 5.59.9
+      '@typescript-eslint/types': 5.59.11
-      '@typescript-eslint/typescript-estree': 5.59.9(typescript@5.1.3)
+      '@typescript-eslint/typescript-estree': 5.59.11(typescript@5.1.3)
       eslint: 8.42.0
       eslint-scope: 5.1.1
       semver: 7.5.1
@@ -900,11 +906,11 @@ packages:
       - typescript
     dev: true
 
-  /@typescript-eslint/visitor-keys@5.59.9:
+  /@typescript-eslint/visitor-keys@5.59.11:
-    resolution: {integrity: sha512-bT7s0td97KMaLwpEBckbzj/YohnvXtqbe2XgqNvTl6RJVakY5mvENOTPvw5u66nljfZxthESpDozs86U+oLY8Q==}
+    resolution: {integrity: sha512-KGYniTGG3AMTuKF9QBD7EIrvufkB6O6uX3knP73xbKLMpH+QRPcgnCxjWXSHjMRuOxFLovljqQgQpR0c7GvjoA==}
     engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0}
     dependencies:
-      '@typescript-eslint/types': 5.59.9
+      '@typescript-eslint/types': 5.59.11
       eslint-visitor-keys: 3.4.1
     dev: true
 
@@ -2243,6 +2249,11 @@ packages:
     engines: {node: '>=8'}
     dev: true
 
+  /is-absolute-url@4.0.1:
+    resolution: {integrity: sha512-/51/TKE88Lmm7Gc4/8btclNXWS+g50wXhYJq8HWIBAGUBnoAdRu1aXeh364t/O7wXDAcTJDP8PNuNKWUDWie+A==}
+    engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}
+    dev: false
+
   /is-array-buffer@3.0.2:
     resolution: {integrity: sha512-y+FyyR/w8vfIRq4eQcM1EYgSTnmHXPqaF+IgzgraytCFq5Xh8lllDVmAZolPJiZttZLeFSINPYMaEJ7/vWUa1w==}
     dependencies:
@@ -2383,6 +2394,13 @@ packages:
       has-tostringtag: 1.0.0
     dev: true
 
+  /is-relative-url@4.0.0:
+    resolution: {integrity: sha512-PkzoL1qKAYXNFct5IKdKRH/iBQou/oCC85QhXj6WKtUQBliZ4Yfd3Zk27RHu9KQG8r6zgvAA2AQKC9p+rqTszg==}
+    engines: {node: '>=14.16'}
+    dependencies:
+      is-absolute-url: 4.0.1
+    dev: false
+
   /is-shared-array-buffer@1.0.2:
     resolution: {integrity: sha512-sqN2UDu1/0y6uvXyStCOzyhAjCSlHceFoMKJW8W9EU9cvic/QdsZ0kEU93HEy3IUEFZIiH/3w+AH/UQbPHNdhA==}
     dependencies:
@@ -2833,7 +2851,7 @@ packages:
     resolution: {integrity: sha512-8+Ib8rRJ4L0o3kfmyVCL7gzrohyDe0cMFTBa2d364yIrEGMEoetznKJx899YxjybU6bL9SQkYPSBBs1gyYs8Xg==}
     dependencies:
       '@sinonjs/commons': 2.0.0
-      '@sinonjs/fake-timers': 10.2.0
+      '@sinonjs/fake-timers': 10.1.0
       '@sinonjs/text-encoding': 0.7.2
       just-extend: 4.2.1
       path-to-regexp: 1.8.0
@@ -2880,6 +2898,11 @@ packages:
     engines: {node: '>=0.10.0'}
     dev: true
 
+  /normalize-url@8.0.0:
+    resolution: {integrity: sha512-uVFpKhj5MheNBJRTiMZ9pE/7hD1QTeEvugSJW/OmLzAp78PB5O6adfMNTvmfKhXBkvCzC+rqifWcVYpGFwTjnw==}
+    engines: {node: '>=14.16'}
+    dev: false
+
   /npm-run-all@4.1.5:
     resolution: {integrity: sha512-Oo82gJDAVcaMdi3nuoKFavkIHBRVqQ1qvMb+9LHk/cF4P6B2m8aP04hGf7oL6wZ9BuGwX1onlLhpuoofSyoQDQ==}
     engines: {node: '>= 4'}
@@ -3058,8 +3081,8 @@ packages:
     engines: {node: '>=12'}
     dev: true
 
-  /p-timeout@6.1.1:
+  /p-timeout@6.1.2:
-    resolution: {integrity: sha512-yqz2Wi4fiFRpMmK0L2pGAU49naSUaP23fFIQL2Y6YT+qDGPoFwpvgQM/wzc6F8JoenUkIlAFa4Ql7NguXBxI7w==}
+    resolution: {integrity: sha512-UbD77BuZ9Bc9aABo74gfXhNvzC9Tx7SxtHSh1fxvx3jTLLYvmVhiQZZrJzqqU0jKbN32kb5VOKiLEQI/3bIjgQ==}
     engines: {node: '>=14.16'}
     dev: false
 
@@ -3375,8 +3398,8 @@ packages:
       glob: 7.2.3
     dev: true
 
-  /rollup@3.24.1:
+  /rollup@3.25.1:
-    resolution: {integrity: sha512-REHe5dx30ERBRFS0iENPHy+t6wtSEYkjrhwNsLyh3qpRaZ1+aylvMUdMBUHWUD/RjjLmLzEvY8Z9XRlpcdIkHA==}
+    resolution: {integrity: sha512-tywOR+rwIt5m2ZAWSe5AIJcTat8vGlnPFAv15ycCrw33t6iFsXZ6mzHVFh2psSjxQPmI+xgzMZZizUAukBI4aQ==}
     engines: {node: '>=14.18.0', npm: '>=8.0.0'}
     hasBin: true
     optionalDependencies:
@@ -3477,11 +3500,11 @@ packages:
     engines: {node: '>=14'}
     dev: true
 
-  /sinon@15.1.0:
+  /sinon@15.1.2:
-    resolution: {integrity: sha512-cS5FgpDdE9/zx7no8bxROHymSlPLZzq0ChbbLk1DrxBfc+eTeBK3y8nIL+nu/0QeYydhhbLIr7ecHJpywjQaoQ==}
+    resolution: {integrity: sha512-uG1pU54Fis4EfYOPoEi13fmRHgZNg/u+3aReSEzHsN52Bpf+bMVfsBQS5MjouI+rTuG6UBIINlpuuO2Epr7SiA==}
     dependencies:
       '@sinonjs/commons': 3.0.0
-      '@sinonjs/fake-timers': 10.2.0
+      '@sinonjs/fake-timers': 10.1.0
       '@sinonjs/samsam': 8.0.0
       diff: 5.1.0
       nise: 5.1.4
@@ -3842,7 +3865,7 @@ packages:
       joycon: 3.1.1
       postcss-load-config: 3.1.4
       resolve-from: 5.0.0
-      rollup: 3.24.1
+      rollup: 3.25.1
       source-map: 0.8.0-beta.0
       sucrase: 3.32.0
       tree-kill: 1.2.2

scratch/apify.ts

@@ -0,0 +1,13 @@
+import { ApifyClient } from 'apify-client'
+import 'dotenv/config'
+
+async function main() {
+  const apify = new ApifyClient({
+    token: process.env.APIFY_API_KEY
+  })
+
+  const actor = await apify.actor('apify/website-content-crawler').get()
+  console.log(actor)
+}
+
+main()

src/services/bing-web-search.ts

@@ -1,6 +1,6 @@
-import ky from 'ky'
+import defaultKy from 'ky'
 
-export const BING_BASE_URL = 'https://api.bing.microsoft.com'
+export const BING_API_BASE_URL = 'https://api.bing.microsoft.com'
 
 export interface BingWebSearchQuery {
   q: string
@@ -235,22 +235,30 @@ interface FluffyContractualRule {
 }
 
 export class BingWebSearchClient {
+  api: typeof defaultKy
+
   apiKey: string
-  baseUrl: string
+  apiBaseUrl: string
 
   constructor({
     apiKey = process.env.BING_API_KEY,
-    baseUrl = BING_BASE_URL
+    apiBaseUrl = BING_API_BASE_URL,
+    ky = defaultKy
   }: {
     apiKey?: string
-    baseUrl?: string
+    apiBaseUrl?: string
+    ky?: typeof defaultKy
   } = {}) {
     if (!apiKey) {
       throw new Error(`Error BingWebSearchClient missing required "apiKey"`)
     }
 
     this.apiKey = apiKey
-    this.baseUrl = baseUrl
+    this.apiBaseUrl = apiBaseUrl
+
+    this.api = ky.extend({
+      prefixUrl: this.apiBaseUrl
+    })
   }
 
   async search(queryOrOpts: string | BingWebSearchQuery) {
@@ -269,9 +277,9 @@ export class BingWebSearchClient {
             ...queryOrOpts
           }
 
-    console.log(searchParams)
+    // console.log(searchParams)
-    return ky
+    return this.api
-      .get(`${this.baseUrl}/v7.0/search`, {
+      .get('v7.0/search', {
         headers: {
           'Ocp-Apim-Subscription-Key': this.apiKey
         },

src/services/diffbot.ts

@@ -0,0 +1,303 @@
+import defaultKy from 'ky'
+
+export const DIFFBOT_API_BASE_URL = 'https://api.diffbot.com'
+export const DIFFBOT_KNOWLEDGE_GRAPH_API_BASE_URL = 'https://kg.diffbot.com'
+
+export interface DiffbotExtractOptions {
+  /** Specify optional fields to be returned from any fully-extracted pages, e.g.: &fields=querystring,links. See available fields within each API's individual documentation pages.
+   * @see https://docs.diffbot.com/reference/extract-optional-fields
+   */
+  fields?: string[]
+
+  /** (*Undocumented*) Pass paging=false to disable automatic concatenation of multiple-page articles. (By default, Diffbot will concatenate up to 20 pages of a single article.) */
+  paging?: boolean
+
+  /** Pass discussion=false to disable automatic extraction of comments or reviews from pages identified as articles or products. This will not affect pages identified as discussions. */
+  discussion?: boolean
+
+  /** Sets a value in milliseconds to wait for the retrieval/fetch of content from the requested URL. The default timeout for the third-party response is 30 seconds (30000). */
+  timeout?: number
+
+  /** Used to specify the IP address of a custom proxy that will be used to fetch the target page, instead of Diffbot's default IPs/proxies. (Ex: &proxy=168.212.226.204) */
+  proxy?: string
+
+  /** Used to specify the authentication parameters that will be used with the proxy specified in the &proxy parameter. (Ex: &proxyAuth=username:password) */
+  proxyAuth?: string
+
+  /** `none` will instruct Extract to not use proxies, even if proxies have been enabled for this particular URL globally. */
+  useProxy?: string
+
+  /** @see https://docs.diffbot.com/reference/extract-custom-javascript */
+  customJs?: string
+
+  /** @see https://docs.diffbot.com/reference/extract-custom-headers */
+  customHeaders?: Record<string, string>
+}
+
+export interface DiffbotExtractAnalyzeOptions extends DiffbotExtractOptions {
+  /** Web page URL of the analyze to process */
+  url: string
+
+  /** By default the Analyze API will fully extract all pages that match an existing Automatic API -- articles, products or image pages. Set mode to a specific page-type (e.g., mode=article) to extract content only from that specific page-type. All other pages will simply return the default Analyze fields. */
+  mode?: string
+
+  /** Force any non-extracted pages (those with a type of "other") through a specific API. For example, to route all "other" pages through the Article API, pass &fallback=article. Pages that utilize this functionality will return a fallbackType field at the top-level of the response and a originalType field within each extracted object, both of which will indicate the fallback API used. */
+  fallback?: string
+}
+
+export interface DiffbotExtractArticleOptions extends DiffbotExtractOptions {
+  /** Web page URL of the analyze to process */
+  url: string
+
+  /** Set the maximum number of automatically-generated tags to return. By default a maximum of ten tags will be returned. */
+  maxTags?: number
+
+  /** Set the minimum relevance score of tags to return, between 0.0 and 1.0. By default only tags with a score equal to or above 0.5 will be returned. */
+  tagConfidence?: number
+
+  /** Used to request the output of the Diffbot Natural Language API in the field naturalLanguage. Example: &naturalLanguage=entities,facts,categories,sentiment. */
+  naturalLanguage?: string[]
+}
+
+export interface DiffbotExtractResponse {
+  request: DiffbotRequest
+  objects: DiffbotObject[]
+}
+
+export type DiffbotExtractArticleResponse = DiffbotExtractResponse
+
+export interface DiffbotExtractAnalyzeResponse extends DiffbotExtractResponse {
+  type: string
+  title: string
+  humanLanguage: string
+}
+
+export interface DiffbotObject {
+  date: string
+  sentiment: number
+  images: DiffbotImage[]
+  author: string
+  estimatedDate: string
+  publisherRegion: string
+  icon: string
+  diffbotUri: string
+  siteName: string
+  type: string
+  title: string
+  tags: DiffbotTag[]
+  publisherCountry: string
+  humanLanguage: string
+  authorUrl: string
+  pageUrl: string
+  html: string
+  text: string
+  categories?: DiffbotCategory[]
+  authors: DiffbotAuthor[]
+  breadcrumb?: DiffbotBreadcrumb[]
+  meta?: any
+}
+
+interface DiffbotAuthor {
+  name: string
+  link: string
+}
+
+interface DiffbotCategory {
+  score: number
+  name: string
+  id: string
+}
+
+export interface DiffbotBreadcrumb {
+  link: string
+  name: string
+}
+
+interface DiffbotImage {
+  url: string
+  diffbotUri: string
+
+  naturalWidth: number
+  naturalHeight: number
+  width: number
+  height: number
+
+  isCached?: boolean
+  primary?: boolean
+}
+
+interface DiffbotTag {
+  score: number
+  sentiment: number
+  count: number
+  label: string
+  uri: string
+  rdfTypes: string[]
+}
+
+interface DiffbotRequest {
+  pageUrl: string
+  api: string
+  version: number
+}
+
+export interface Image {
+  naturalHeight: number
+  diffbotUri: string
+  url: string
+  naturalWidth: number
+  primary: boolean
+}
+
+export interface Tag {
+  score: number
+  sentiment: number
+  count: number
+  label: string
+  uri: string
+  rdfTypes: string[]
+}
+
+export interface Request {
+  pageUrl: string
+  api: string
+  version: number
+}
+
+export interface DiffbotSearchKnowledgeGraphOptions {
+  type?: 'query' | 'text' | 'queryTextFallback' | 'crawl'
+  query: string
+  col?: string
+  from?: number
+  size?: number
+
+  // NOTE: we only support `json`, so these options are not needed
+  // We can always convert from json to another format if needed.
+  // format?: 'json' | 'jsonl' | 'csv' | 'xls' | 'xlsx'
+  // exportspec?: string
+  // exportseparator?: string
+  // exportfile?: string
+
+  filter?: string
+  jsonmode?: 'extended' | 'id'
+  nonCanonicalFacts?: boolean
+  noDedupArticles?: boolean
+  cluster?: 'all' | 'best' | 'dedupe'
+  report?: boolean
+}
+
+export interface DiffbotSearchKnowledgeGraphResponse {
+  version: number
+  hits: number
+  results: number
+  kgversion: string
+  diffbot_type: string
+  facet: boolean
+  data: DiffbotKnowledgeGraphNode[]
+}
+
+export interface DiffbotKnowledgeGraphNode {
+  score: number
+  entity: DiffbotKnowledgeGraphEntity
+  entity_ctx: any
+  errors: string[]
+  callbackQuery: string
+  upperBound: number
+  lowerBound: number
+  count: number
+  value: string
+  uri: string
+}
+
+export interface DiffbotKnowledgeGraphEntity {
+  id: string
+  images: DiffbotImage[]
+  diffbotUri: string
+  name: string
+  origins: string[]
+}
+
+export class DiffbotClient {
+  api: typeof defaultKy
+  apiKnowledgeGraph: typeof defaultKy
+
+  apiKey: string
+  apiBaseUrl: string
+  apiKnowledgeGraphBaseUrl: string
+
+  constructor({
+    apiKey = process.env.DIFFBOT_API_KEY,
+    apiBaseUrl = DIFFBOT_API_BASE_URL,
+    apiKnowledgeGraphBaseUrl = DIFFBOT_KNOWLEDGE_GRAPH_API_BASE_URL,
+    timeoutMs = 60_000,
+    ky = defaultKy
+  }: {
+    apiKey?: string
+    apiBaseUrl?: string
+    apiKnowledgeGraphBaseUrl?: string
+    timeoutMs?: number
+    ky?: typeof defaultKy
+  } = {}) {
+    if (!apiKey) {
+      throw new Error(`Error DiffbotClient missing required "apiKey"`)
+    }
+
+    this.apiKey = apiKey
+    this.apiBaseUrl = apiBaseUrl
+    this.apiKnowledgeGraphBaseUrl = apiKnowledgeGraphBaseUrl
+
+    this.api = ky.extend({ prefixUrl: apiBaseUrl, timeout: timeoutMs })
+    this.apiKnowledgeGraph = ky.extend({
+      prefixUrl: apiKnowledgeGraphBaseUrl,
+      timeout: timeoutMs
+    })
+  }
+
+  protected async _extract<
+    T extends DiffbotExtractResponse = DiffbotExtractResponse
+  >(endpoint: string, options: DiffbotExtractOptions): Promise<T> {
+    const { customJs, customHeaders, ...rest } = options
+    const searchParams: Record<string, any> = {
+      ...rest,
+      token: this.apiKey
+    }
+    const headers = {
+      ...Object.fromEntries(
+        [['X-Forward-X-Evaluate', customJs]].filter(([, value]) => value)
+      ),
+      ...customHeaders
+    }
+
+    for (const [key, value] of Object.entries(rest)) {
+      if (Array.isArray(value)) {
+        searchParams[key] = value.join(',')
+      }
+    }
+
+    return this.api
+      .get(endpoint, {
+        searchParams,
+        headers
+      })
+      .json<T>()
+  }
+
+  async extractAnalyze(options: DiffbotExtractAnalyzeOptions) {
+    return this._extract<DiffbotExtractAnalyzeResponse>('v3/analyze', options)
+  }
+
+  async extractArticle(options: DiffbotExtractArticleOptions) {
+    return this._extract<DiffbotExtractArticleResponse>('v3/article', options)
+  }
+
+  async searchKnowledgeGraph(options: DiffbotSearchKnowledgeGraphOptions) {
+    return this.apiKnowledgeGraph
+      .get('kg/v3/dql', {
+        searchParams: {
+          ...options,
+          token: this.apiKey
+        }
+      })
+      .json<DiffbotSearchKnowledgeGraphResponse>()
+  }
+}

src/services/index.ts

@@ -1,4 +1,5 @@
 export * from './bing-web-search'
+export * from './diffbot'
 export * from './metaphor'
 export * from './novu'
 export * from './serpapi'

src/services/metaphor.ts

@@ -1,7 +1,7 @@
-import ky from 'ky'
+import defaultKy from 'ky'
 import { z } from 'zod'
 
-export const METAPHOR_BASE_URL = 'https://api.metaphor.systems'
+export const METAPHOR_API_BASE_URL = 'https://api.metaphor.systems'
 
 // https://metaphorapi.readme.io/reference/search
 export const MetaphorSearchInputSchema = z.object({
@@ -33,27 +33,35 @@ export const MetaphorSearchOutputSchema = z.object({
 export type MetaphorSearchOutput = z.infer<typeof MetaphorSearchOutputSchema>
 
 export class MetaphorClient {
+  api: typeof defaultKy
+
   apiKey: string
-  baseUrl: string
+  apiBaseUrl: string
 
   constructor({
     apiKey = process.env.METAPHOR_API_KEY,
-    baseUrl = METAPHOR_BASE_URL
+    apiBaseUrl = METAPHOR_API_BASE_URL,
+    ky = defaultKy
   }: {
     apiKey?: string
-    baseUrl?: string
+    apiBaseUrl?: string
+    ky?: typeof defaultKy
   } = {}) {
     if (!apiKey) {
       throw new Error(`Error MetaphorClient missing required "apiKey"`)
     }
 
     this.apiKey = apiKey
-    this.baseUrl = baseUrl
+    this.apiBaseUrl = apiBaseUrl
+
+    this.api = ky.extend({
+      prefixUrl: this.apiBaseUrl
+    })
   }
 
   async search(params: MetaphorSearchInput) {
-    return ky
+    return this.api
-      .post(`${this.baseUrl}/search`, {
+      .post('search', {
         headers: {
           'x-api-key': this.apiKey
         },

src/services/novu.ts

@@ -1,6 +1,6 @@
-import ky from 'ky'
+import defaultKy from 'ky'
 
-export const NOVU_BASE_URL = 'https://api.novu.co/v1'
+export const NOVU_API_BASE_URL = 'https://api.novu.co/v1'
 
 export type NovuSubscriber = {
   subscriberId: string
@@ -19,22 +19,30 @@ export type NovuTriggerEventResponse = {
 }
 
 export class NovuClient {
+  api: typeof defaultKy
+
   apiKey: string
-  baseUrl: string
+  apiBaseUrl: string
 
   constructor({
     apiKey = process.env.NOVU_API_KEY,
-    baseUrl = NOVU_BASE_URL
+    apiBaseUrl = NOVU_API_BASE_URL,
+    ky = defaultKy
   }: {
     apiKey?: string
-    baseUrl?: string
+    apiBaseUrl?: string
+    ky?: typeof defaultKy
   } = {}) {
     if (!apiKey) {
       throw new Error(`Error NovuClient missing required "apiKey"`)
     }
 
     this.apiKey = apiKey
-    this.baseUrl = baseUrl
+    this.apiBaseUrl = apiBaseUrl
+
+    this.api = ky.extend({
+      prefixUrl: this.apiBaseUrl
+    })
   }
 
   async triggerEvent({
@@ -46,19 +54,18 @@ export class NovuClient {
     payload: Record<string, unknown>
     to: NovuSubscriber[]
   }) {
-    const url = `${this.baseUrl}/events/trigger`
+    return this.api
-    const headers = {
+      .post('events/trigger', {
-      Authorization: `ApiKey ${this.apiKey}`,
+        headers: {
-      'Content-Type': 'application/json'
+          Authorization: `ApiKey ${this.apiKey}`,
-    }
+          'Content-Type': 'application/json'
-    const response = await ky.post(url, {
+        },
-      headers,
+        json: {
-      json: {
+          name,
-        name,
+          payload,
-        payload,
+          to
-        to
+        }
-      }
+      })
-    })
+      .json<NovuTriggerEventResponse>()
-    return response.json<NovuTriggerEventResponse>()
   }
 }

src/services/serpapi.ts

@@ -1,4 +1,4 @@
-import ky from 'ky'
+import defaultKy from 'ky'
 
 /**
  * All types have been exported from the `serpapi` package, which we're
@@ -350,7 +350,8 @@ export type SerpAPISearchResponse = BaseResponse<GoogleParameters>
 
 export interface SerpAPIClientOptions extends Partial<SerpAPIParams> {
   apiKey?: string
-  baseUrl?: string
+  apiBaseUrl?: string
+  ky?: typeof defaultKy
 }
 
 export const SERPAPI_BASE_URL = 'https://serpapi.com'
@@ -361,13 +362,16 @@ export const SERPAPI_BASE_URL = 'https://serpapi.com'
  * @see https://serpapi.com/search-api
  */
 export class SerpAPIClient {
+  api: typeof defaultKy
+
   apiKey: string
-  baseUrl: string
+  apiBaseUrl: string
   params: Partial<SerpAPIParams>
 
   constructor({
     apiKey = process.env.SERPAPI_API_KEY ?? process.env.SERP_API_KEY,
-    baseUrl = SERPAPI_BASE_URL,
+    apiBaseUrl = SERPAPI_BASE_URL,
+    ky = defaultKy,
     ...params
   }: SerpAPIClientOptions = {}) {
     if (!apiKey) {
@@ -375,8 +379,12 @@ export class SerpAPIClient {
     }
 
     this.apiKey = apiKey
-    this.baseUrl = baseUrl
+    this.apiBaseUrl = apiBaseUrl
     this.params = params
+
+    this.api = ky.extend({
+      prefixUrl: this.apiBaseUrl
+    })
   }
 
   async search(queryOrOpts: string | { query: string }) {
@@ -384,8 +392,8 @@ export class SerpAPIClient {
       typeof queryOrOpts === 'string' ? queryOrOpts : queryOrOpts.query
     const { timeout, ...rest } = this.params
 
-    return ky
+    return this.api
-      .get(`${this.baseUrl}/search`, {
+      .get('search', {
         searchParams: {
           ...rest,
           engine: 'google',

src/services/slack.ts

@@ -1,4 +1,4 @@
-import ky from 'ky'
+import defaultKy from 'ky'
 
 import { sleep } from '@/utils'
 
@@ -256,30 +256,33 @@ export type SlackSendAndWaitOptions = {
 }
 
 export class SlackClient {
-  private api: typeof ky
+  protected api: typeof defaultKy
 
   protected defaultChannel?: string
 
   constructor({
     apiKey = process.env.SLACK_API_KEY,
     baseUrl = SLACK_API_BASE_URL,
-    defaultChannel = process.env.SLACK_DEFAULT_CHANNEL
+    defaultChannel = process.env.SLACK_DEFAULT_CHANNEL,
+    ky = defaultKy
   }: {
     apiKey?: string
     baseUrl?: string
     defaultChannel?: string
+    ky?: typeof defaultKy
   } = {}) {
     if (!apiKey) {
       throw new Error(`Error SlackClient missing required "apiKey"`)
     }
 
-    this.api = ky.create({
+    this.defaultChannel = defaultChannel
+
+    this.api = ky.extend({
       prefixUrl: baseUrl,
       headers: {
         Authorization: `Bearer ${apiKey}`
       }
     })
-    this.defaultChannel = defaultChannel
   }
 
   /**
@@ -328,10 +331,11 @@ export class SlackClient {
    * Returns a list of messages that were sent in a channel after a given timestamp both directly and in threads.
    */
   private async fetchCandidates(channel: string, ts: string) {
-    let candidates: SlackMessage[] = []
     const history = await this.fetchConversationHistory({ channel })
     const directReplies = await this.fetchReplies({ channel, ts })
 
+    let candidates: SlackMessage[] = []
+
     if (directReplies.ok) {
       candidates = candidates.concat(directReplies.messages)
     }
@@ -349,6 +353,7 @@ export class SlackClient {
     candidates.sort((a, b) => {
       return parseFloat(b.ts) - parseFloat(a.ts)
     })
+
     return candidates
   }
 
@@ -357,7 +362,7 @@ export class SlackClient {
    *
    * ### Notes
    *
-   * -   The implementation will poll for replies to the message until the timeout is reached. This is not ideal, but it is the only way to retrieve replies to a message in Slack without spinning up a local server to receive webhook events.
+   * - The implementation will poll for replies to the message until the timeout is reached. This is not ideal, but it is the only way to retrieve replies to a message in Slack without spinning up a local server to receive webhook events.
    */
   public async sendAndWaitForReply({
     text,

src/services/twilio-conversation.ts

@@ -1,9 +1,9 @@
-import ky, { type KyResponse } from 'ky'
+import defaultKy from 'ky'
 
 import { DEFAULT_BOT_NAME } from '@/constants'
 import { sleep } from '@/utils'
 
-export const TWILIO_CONVERSATION_BASE_URL =
+export const TWILIO_CONVERSATION_API_BASE_URL =
   'https://conversations.twilio.com/v1'
 
 export const DEFAULT_TWILIO_TIMEOUT_MS = 120_000
@@ -129,7 +129,8 @@ export type TwilioSendAndWaitOptions = {
  * @see {@link https://www.twilio.com/docs/conversations/api}
  */
 export class TwilioConversationClient {
-  api: typeof ky
+  api: typeof defaultKy
+
   phoneNumber: string
   botName: string
 
@@ -137,18 +138,26 @@ export class TwilioConversationClient {
     accountSid = process.env.TWILIO_ACCOUNT_SID,
     authToken = process.env.TWILIO_AUTH_TOKEN,
     phoneNumber = process.env.TWILIO_PHONE_NUMBER,
-    baseUrl = TWILIO_CONVERSATION_BASE_URL,
+    apiBaseUrl = TWILIO_CONVERSATION_API_BASE_URL,
-    botName = DEFAULT_BOT_NAME
+    botName = DEFAULT_BOT_NAME,
+    ky = defaultKy
   }: {
     accountSid?: string
     authToken?: string
     phoneNumber?: string
-    baseUrl?: string
+    apiBaseUrl?: string
     botName?: string
+    ky?: typeof defaultKy
   } = {}) {
-    if (!accountSid || !authToken) {
+    if (!accountSid) {
       throw new Error(
-        `Error TwilioConversationClient missing required "accountSid" and/or "authToken"`
+        `Error TwilioConversationClient missing required "accountSid"`
+      )
+    }
+
+    if (!authToken) {
+      throw new Error(
+        `Error TwilioConversationClient missing required "authToken"`
       )
     }
 
@@ -161,8 +170,8 @@ export class TwilioConversationClient {
     this.botName = botName
     this.phoneNumber = phoneNumber
 
-    this.api = ky.create({
+    this.api = ky.extend({
-      prefixUrl: baseUrl,
+      prefixUrl: apiBaseUrl,
       headers: {
         Authorization:
           'Basic ' +
@@ -175,7 +184,7 @@ export class TwilioConversationClient {
   /**
    * Deletes a conversation and all its messages.
    */
-  async deleteConversation(conversationSid: string): Promise<KyResponse> {
+  async deleteConversation(conversationSid: string) {
     return this.api.delete(`Conversations/${conversationSid}`)
   }
 

src/url-utils.ts

@@ -0,0 +1,44 @@
+import isRelativeUrl from 'is-relative-url'
+import normalizeUrlImpl from 'normalize-url'
+import QuickLRU from 'quick-lru'
+
+// const protocolAllowList = new Set(['https:', 'http:'])
+const normalizedUrlCache = new QuickLRU<string, string | null>({
+  maxSize: 4000
+})
+
+export function normalizeUrl(url: string): string | null {
+  let normalizedUrl: string | null | undefined
+
+  try {
+    if (!url || (isRelativeUrl(url) && !url.startsWith('//'))) {
+      return null
+    }
+
+    normalizedUrl = normalizedUrlCache.get(url)
+
+    if (normalizedUrl !== undefined) {
+      return normalizedUrl
+    }
+
+    normalizedUrl = normalizeUrlImpl(url, {
+      stripWWW: false,
+      defaultProtocol: 'https',
+      normalizeProtocol: true,
+      forceHttps: false,
+      stripHash: false,
+      stripTextFragment: true,
+      removeQueryParameters: [/^utm_\w+/i, 'ref'],
+      removeTrailingSlash: true,
+      removeSingleSlash: true,
+      removeExplicitPort: true,
+      sortQueryParameters: true
+    })
+  } catch (err) {
+    // ignore invalid urls
+    normalizedUrl = null
+  }
+
+  normalizedUrlCache.set(url, normalizedUrl!)
+  return normalizedUrl
+}

test/_utils.ts

@@ -4,10 +4,12 @@ import 'dotenv/config'
 import hashObject from 'hash-obj'
 import Redis from 'ioredis'
 import Keyv from 'keyv'
+import defaultKy from 'ky'
 import { OpenAIClient } from 'openai-fetch'
 import pMemoize from 'p-memoize'
 
 import { Agentic } from '@/agentic'
+import { normalizeUrl } from '@/url-utils'
 
 export const fakeOpenAIAPIKey = 'fake-openai-api-key'
 export const fakeAnthropicAPIKey = 'fake-anthropic-api-key'
@@ -39,6 +41,97 @@ keyv.has = async (key, ...rest) => {
   return res
 }
 
+function getCacheKeyForRequest(request: Request): string | null {
+  const method = request.method.toLowerCase()
+
+  if (method === 'get') {
+    const url = normalizeUrl(request.url)
+
+    if (url) {
+      const cacheParams = {
+        // TODO: request.headers isn't a normal JS object...
+        headers: { ...request.headers }
+      }
+
+      // console.log('getCacheKeyForRequest', { url, cacheParams })
+      const cacheKey = getCacheKey(`http:${method} ${url}`, cacheParams)
+      return cacheKey
+    }
+  }
+
+  return null
+}
+
+/**
+ * Custom `ky` instance that caches GET JSON requests.
+ *
+ * TODO:
+ *  - support non-GET requests
+ *  - support non-JSON responses
+ */
+export const ky = defaultKy.extend({
+  hooks: {
+    beforeRequest: [
+      async (request) => {
+        try {
+          // console.log(`beforeRequest ${request.method} ${request.url}`)
+
+          const cacheKey = getCacheKeyForRequest(request)
+          // console.log({ cacheKey })
+          if (!cacheKey) {
+            return
+          }
+
+          if (!(await keyv.has(cacheKey))) {
+            return
+          }
+
+          const cachedResponse = await keyv.get(cacheKey)
+          // console.log({ cachedResponse })
+
+          if (!cachedResponse) {
+            return
+          }
+
+          return new Response(JSON.stringify(cachedResponse), {
+            status: 200,
+            headers: { 'Content-Type': 'application/json' }
+          })
+        } catch (err) {
+          console.error('ky beforeResponse cache error', err)
+        }
+      }
+    ],
+
+    afterResponse: [
+      async (request, _options, response) => {
+        try {
+          // console.log(
+          //   `afterRequest ${request.method} ${request.url} ⇒ ${response.status}`
+          // )
+
+          if (response.status < 200 || response.status >= 300) {
+            return
+          }
+
+          const cacheKey = getCacheKeyForRequest(request)
+          // console.log({ cacheKey })
+          if (!cacheKey) {
+            return
+          }
+
+          const responseBody = await response.json()
+          // console.log({ responseBody })
+
+          await keyv.set(cacheKey, responseBody)
+        } catch (err) {
+          console.error('ky afterResponse cache error', err)
+        }
+      }
+    ]
+  }
+})
+
 export class OpenAITestClient extends OpenAIClient {
   createChatCompletion = pMemoize(super.createChatCompletion, {
     cacheKey: (params) => getCacheKey('openai:chat', params),

test/services/bing.test.ts

@@ -2,7 +2,7 @@ import test from 'ava'
 
 import { BingWebSearchClient } from '@/services'
 
-import './_utils'
+import { ky } from '../_utils'
 
 test('BingWebSearchClient.search', async (t) => {
   if (!process.env.BING_API_KEY) {
@@ -10,9 +10,9 @@ test('BingWebSearchClient.search', async (t) => {
   }
 
   t.timeout(2 * 60 * 1000)
-  const client = new BingWebSearchClient()
+  const client = new BingWebSearchClient({ ky })
 
   const result = await client.search('coffee')
   // console.log(result)
-  t.truthy(result?.webPages)
+  t.true(result?.webPages?.value?.length > 0)
 })

test/services/diffbot.test.ts

@@ -0,0 +1,37 @@
+import test from 'ava'
+
+import { DiffbotClient } from '@/services'
+
+import { isCI, ky } from '../_utils'
+
+test('Diffbot.analyze', async (t) => {
+  if (!process.env.DIFFBOT_API_KEY || isCI) {
+    return t.pass()
+  }
+
+  t.timeout(2 * 60 * 1000)
+  const client = new DiffbotClient({ ky })
+
+  const result = await client.extractAnalyze({
+    url: 'https://transitivebullsh.it'
+  })
+  // console.log(result)
+  t.is(result.type, 'list')
+  t.is(result.objects?.length, 1)
+})
+
+test('Diffbot.article', async (t) => {
+  if (!process.env.DIFFBOT_API_KEY || isCI) {
+    return t.pass()
+  }
+
+  t.timeout(2 * 60 * 1000)
+  const client = new DiffbotClient({ ky })
+
+  const result = await client.extractArticle({
+    url: 'https://www.nytimes.com/2023/05/31/magazine/ai-start-up-accelerator-san-francisco.html'
+    // fields: ['meta']
+  })
+  // console.log(JSON.stringify(result, null, 2))
+  t.is(result.objects[0].type, 'article')
+})

test/services/novu.test.ts

@@ -2,7 +2,7 @@ import test from 'ava'
 
 import { NovuClient } from '@/services/novu'
 
-import './_utils'
+import { ky } from '../_utils'
 
 test('NovuClient.triggerEvent', async (t) => {
   if (!process.env.NOVU_API_KEY) {
@@ -10,7 +10,7 @@ test('NovuClient.triggerEvent', async (t) => {
   }
 
   t.timeout(2 * 60 * 1000)
-  const client = new NovuClient()
+  const client = new NovuClient({ ky })
 
   const result = await client.triggerEvent({
     name: 'send-email',

test/services/serpapi.test.ts

@@ -2,7 +2,7 @@ import test from 'ava'
 
 import { SerpAPIClient } from '@/services/serpapi'
 
-import './_utils'
+import { ky } from '../_utils'
 
 test('SerpAPIClient.search', async (t) => {
   if (!process.env.SERPAPI_API_KEY) {
@@ -10,7 +10,7 @@ test('SerpAPIClient.search', async (t) => {
   }
 
   t.timeout(2 * 60 * 1000)
-  const client = new SerpAPIClient()
+  const client = new SerpAPIClient({ ky })
 
   const result = await client.search('coffee')
   // console.log(result)

test/services/slack.test.ts

@@ -2,7 +2,7 @@ import test from 'ava'
 
 import { SlackClient } from '@/services/slack'
 
-import './_utils'
+import '../_utils'
 
 test('SlackClient.sendMessage', async (t) => {
   if (!process.env.SLACK_API_KEY || !process.env.SLACK_DEFAULT_CHANNEL) {

test/services/twilio-conversation.test.ts

@@ -2,7 +2,7 @@ import test from 'ava'
 
 import { TwilioConversationClient } from '@/services/twilio-conversation'
 
-import './_utils'
+import '../_utils'
 
 test.serial('TwilioConversationClient.createConversation', async (t) => {
   if (!process.env.TWILIO_ACCOUNT_SID || !process.env.TWILIO_AUTH_TOKEN) {

test/url-utils.test.ts

@@ -0,0 +1,27 @@
+import test from 'ava'
+
+import { normalizeUrl } from '@/url-utils'
+
+test('normalizeUrl', async (t) => {
+  t.is(normalizeUrl('https://www.google.com'), 'https://www.google.com')
+  t.is(normalizeUrl('//www.google.com'), 'https://www.google.com')
+  t.is(
+    normalizeUrl('https://www.google.com/foo?'),
+    'https://www.google.com/foo'
+  )
+  t.is(
+    normalizeUrl('https://www.google.com/?foo=bar&dog=cat'),
+    'https://www.google.com/?dog=cat&foo=bar'
+  )
+  t.is(
+    normalizeUrl('https://google.com/abc/123//'),
+    'https://google.com/abc/123'
+  )
+})
+
+test('normalizeUrl - invalid urls', async (t) => {
+  t.is(normalizeUrl('/foo'), null)
+  t.is(normalizeUrl('/foo/bar/baz'), null)
+  t.is(normalizeUrl('://foo.com'), null)
+  t.is(normalizeUrl('foo'), null)
+})

tsconfig.json

@@ -1,7 +1,7 @@
 {
   "compilerOptions": {
     "target": "es2020",
-    "lib": ["esnext", "es2022.error"],
+    "lib": ["esnext", "es2022.error", "DOM"],
     "allowJs": true,
     "skipLibCheck": true,
     "strict": true,