kopia lustrzana https://github.com/transitive-bullshit/chatgpt-api
feat: add diffbot; refactor ky and services; add optional ky caching for tests
Travis Fischer
rodzic
ab5321b641
commit
92ba811e93
|
|
@ -27,7 +27,12 @@
|
|||
"padding-line-between-statements": [
|
||||
"error",
|
||||
{ "blankLine": "always", "prev": ["block", "block-like"], "next": "*" },
|
||||
{ "blankLine": "always", "prev": "*", "next": ["block", "block-like"] }
|
||||
{ "blankLine": "always", "prev": "*", "next": ["block", "block-like"] },
|
||||
{
|
||||
"blankLine": "any",
|
||||
"prev": ["singleline-const", "singleline-let", "expression"],
|
||||
"next": ["block", "block-like"]
|
||||
}
|
||||
]
|
||||
},
|
||||
"overrides": [
|
||||
|
|
|
|||
12
package.json
12
package.json
|
|
@ -45,14 +45,16 @@
|
|||
"debug": "^4.3.4",
|
||||
"expr-eval": "^2.0.2",
|
||||
"handlebars": "^4.7.7",
|
||||
"is-relative-url": "^4.0.0",
|
||||
"js-tiktoken": "^1.0.6",
|
||||
"jsonrepair": "^3.1.0",
|
||||
"ky": "^0.33.3",
|
||||
"nanoid": "^4.0.2",
|
||||
"normalize-url": "^8.0.0",
|
||||
"openai-fetch": "^1.5.1",
|
||||
"p-map": "^6.0.0",
|
||||
"p-retry": "^5.1.2",
|
||||
"p-timeout": "^6.1.1",
|
||||
"p-timeout": "^6.1.2",
|
||||
"quick-lru": "^6.1.1",
|
||||
"ts-dedent": "^2.2.0",
|
||||
"uuid": "^9.0.0",
|
||||
|
|
@ -64,11 +66,11 @@
|
|||
"@keyv/redis": "^2.6.1",
|
||||
"@trivago/prettier-plugin-sort-imports": "^4.1.1",
|
||||
"@types/debug": "^4.1.8",
|
||||
"@types/node": "^20.3.0",
|
||||
"@types/node": "^20.3.1",
|
||||
"@types/sinon": "^10.0.15",
|
||||
"@types/uuid": "^9.0.2",
|
||||
"@typescript-eslint/eslint-plugin": "^5.59.9",
|
||||
"@typescript-eslint/parser": "^5.59.9",
|
||||
"@typescript-eslint/eslint-plugin": "^5.59.11",
|
||||
"@typescript-eslint/parser": "^5.59.11",
|
||||
"ava": "^5.3.0",
|
||||
"del-cli": "^5.0.0",
|
||||
"dotenv": "^16.1.4",
|
||||
|
|
@ -83,7 +85,7 @@
|
|||
"p-memoize": "^7.1.1",
|
||||
"prettier": "^2.8.8",
|
||||
"react": "^18.2.0",
|
||||
"sinon": "^15.1.0",
|
||||
"sinon": "^15.1.2",
|
||||
"tsup": "^6.7.0",
|
||||
"tsx": "^3.12.7",
|
||||
"type-fest": "^3.11.1",
|
||||
|
|
|
|||
137
pnpm-lock.yaml
137
pnpm-lock.yaml
|
|
@ -1,4 +1,4 @@
|
|||
lockfileVersion: '6.1'
|
||||
lockfileVersion: '6.0'
|
||||
|
||||
settings:
|
||||
autoInstallPeers: true
|
||||
|
|
@ -26,6 +26,9 @@ dependencies:
|
|||
handlebars:
|
||||
specifier: ^4.7.7
|
||||
version: 4.7.7
|
||||
is-relative-url:
|
||||
specifier: ^4.0.0
|
||||
version: 4.0.0
|
||||
js-tiktoken:
|
||||
specifier: ^1.0.6
|
||||
version: 1.0.6
|
||||
|
|
@ -38,6 +41,9 @@ dependencies:
|
|||
nanoid:
|
||||
specifier: ^4.0.2
|
||||
version: 4.0.2
|
||||
normalize-url:
|
||||
specifier: ^8.0.0
|
||||
version: 8.0.0
|
||||
openai-fetch:
|
||||
specifier: ^1.5.1
|
||||
version: 1.5.1
|
||||
|
|
@ -48,8 +54,8 @@ dependencies:
|
|||
specifier: ^5.1.2
|
||||
version: 5.1.2
|
||||
p-timeout:
|
||||
specifier: ^6.1.1
|
||||
version: 6.1.1
|
||||
specifier: ^6.1.2
|
||||
version: 6.1.2
|
||||
quick-lru:
|
||||
specifier: ^6.1.1
|
||||
version: 6.1.1
|
||||
|
|
@ -80,8 +86,8 @@ devDependencies:
|
|||
specifier: ^4.1.8
|
||||
version: 4.1.8
|
||||
'@types/node':
|
||||
specifier: ^20.3.0
|
||||
version: 20.3.0
|
||||
specifier: ^20.3.1
|
||||
version: 20.3.1
|
||||
'@types/sinon':
|
||||
specifier: ^10.0.15
|
||||
version: 10.0.15
|
||||
|
|
@ -89,11 +95,11 @@ devDependencies:
|
|||
specifier: ^9.0.2
|
||||
version: 9.0.2
|
||||
'@typescript-eslint/eslint-plugin':
|
||||
specifier: ^5.59.9
|
||||
version: 5.59.9(@typescript-eslint/parser@5.59.9)(eslint@8.42.0)(typescript@5.1.3)
|
||||
specifier: ^5.59.11
|
||||
version: 5.59.11(@typescript-eslint/parser@5.59.11)(eslint@8.42.0)(typescript@5.1.3)
|
||||
'@typescript-eslint/parser':
|
||||
specifier: ^5.59.9
|
||||
version: 5.59.9(eslint@8.42.0)(typescript@5.1.3)
|
||||
specifier: ^5.59.11
|
||||
version: 5.59.11(eslint@8.42.0)(typescript@5.1.3)
|
||||
ava:
|
||||
specifier: ^5.3.0
|
||||
version: 5.3.0
|
||||
|
|
@ -137,8 +143,8 @@ devDependencies:
|
|||
specifier: ^18.2.0
|
||||
version: 18.2.0
|
||||
sinon:
|
||||
specifier: ^15.1.0
|
||||
version: 15.1.0
|
||||
specifier: ^15.1.2
|
||||
version: 15.1.2
|
||||
tsup:
|
||||
specifier: ^6.7.0
|
||||
version: 6.7.0(typescript@5.1.3)
|
||||
|
|
@ -692,8 +698,8 @@ packages:
|
|||
type-detect: 4.0.8
|
||||
dev: true
|
||||
|
||||
/@sinonjs/fake-timers@10.2.0:
|
||||
resolution: {integrity: sha512-OPwQlEdg40HAj5KNF8WW6q2KG4Z+cBCZb3m4ninfTZKaBmbIJodviQsDBoYMPHkOyJJMHnOJo5j2+LKDOhOACg==}
|
||||
/@sinonjs/fake-timers@10.1.0:
|
||||
resolution: {integrity: sha512-w1qd368vtrwttm1PRJWPW1QHlbmHrVDGs1eBH/jZvRPUFS4MNXV9Q33EQdjOdeAxZ7O8+3wM7zxztm2nfUSyKw==}
|
||||
dependencies:
|
||||
'@sinonjs/commons': 3.0.0
|
||||
dev: true
|
||||
|
|
@ -748,8 +754,8 @@ packages:
|
|||
resolution: {integrity: sha512-iiUgKzV9AuaEkZqkOLDIvlQiL6ltuZd9tGcW3gwpnX8JbuiuhFlEGmmFXEXkN50Cvq7Os88IY2v0dkDqXYWVgA==}
|
||||
dev: true
|
||||
|
||||
/@types/node@20.3.0:
|
||||
resolution: {integrity: sha512-cumHmIAf6On83X7yP+LrsEyUOf/YlociZelmpRYaGFydoaPdxdt80MAbu6vWerQT2COCp2nPvHdsbD7tHn/YlQ==}
|
||||
/@types/node@20.3.1:
|
||||
resolution: {integrity: sha512-EhcH/wvidPy1WeML3TtYFGR83UzjxeWRen9V402T8aUGYsCHOmfoisV3ZSg03gAFIbLq8TnWOJ0f4cALtnSEUg==}
|
||||
dev: true
|
||||
|
||||
/@types/normalize-package-data@2.4.1:
|
||||
|
|
@ -778,8 +784,8 @@ packages:
|
|||
resolution: {integrity: sha512-kNnC1GFBLuhImSnV7w4njQkUiJi0ZXUycu1rUaouPqiKlXkh77JKgdRnTAp1x5eBwcIwbtI+3otwzuIDEuDoxQ==}
|
||||
dev: true
|
||||
|
||||
/@typescript-eslint/eslint-plugin@5.59.9(@typescript-eslint/parser@5.59.9)(eslint@8.42.0)(typescript@5.1.3):
|
||||
resolution: {integrity: sha512-4uQIBq1ffXd2YvF7MAvehWKW3zVv/w+mSfRAu+8cKbfj3nwzyqJLNcZJpQ/WZ1HLbJDiowwmQ6NO+63nCA+fqA==}
|
||||
/@typescript-eslint/eslint-plugin@5.59.11(@typescript-eslint/parser@5.59.11)(eslint@8.42.0)(typescript@5.1.3):
|
||||
resolution: {integrity: sha512-XxuOfTkCUiOSyBWIvHlUraLw/JT/6Io1365RO6ZuI88STKMavJZPNMU0lFcUTeQXEhHiv64CbxYxBNoDVSmghg==}
|
||||
engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0}
|
||||
peerDependencies:
|
||||
'@typescript-eslint/parser': ^5.0.0
|
||||
|
|
@ -790,10 +796,10 @@ packages:
|
|||
optional: true
|
||||
dependencies:
|
||||
'@eslint-community/regexpp': 4.5.1
|
||||
'@typescript-eslint/parser': 5.59.9(eslint@8.42.0)(typescript@5.1.3)
|
||||
'@typescript-eslint/scope-manager': 5.59.9
|
||||
'@typescript-eslint/type-utils': 5.59.9(eslint@8.42.0)(typescript@5.1.3)
|
||||
'@typescript-eslint/utils': 5.59.9(eslint@8.42.0)(typescript@5.1.3)
|
||||
'@typescript-eslint/parser': 5.59.11(eslint@8.42.0)(typescript@5.1.3)
|
||||
'@typescript-eslint/scope-manager': 5.59.11
|
||||
'@typescript-eslint/type-utils': 5.59.11(eslint@8.42.0)(typescript@5.1.3)
|
||||
'@typescript-eslint/utils': 5.59.11(eslint@8.42.0)(typescript@5.1.3)
|
||||
debug: 4.3.4
|
||||
eslint: 8.42.0
|
||||
grapheme-splitter: 1.0.4
|
||||
|
|
@ -806,8 +812,8 @@ packages:
|
|||
- supports-color
|
||||
dev: true
|
||||
|
||||
/@typescript-eslint/parser@5.59.9(eslint@8.42.0)(typescript@5.1.3):
|
||||
resolution: {integrity: sha512-FsPkRvBtcLQ/eVK1ivDiNYBjn3TGJdXy2fhXX+rc7czWl4ARwnpArwbihSOHI2Peg9WbtGHrbThfBUkZZGTtvQ==}
|
||||
/@typescript-eslint/parser@5.59.11(eslint@8.42.0)(typescript@5.1.3):
|
||||
resolution: {integrity: sha512-s9ZF3M+Nym6CAZEkJJeO2TFHHDsKAM3ecNkLuH4i4s8/RCPnF5JRip2GyviYkeEAcwGMJxkqG9h2dAsnA1nZpA==}
|
||||
engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0}
|
||||
peerDependencies:
|
||||
eslint: ^6.0.0 || ^7.0.0 || ^8.0.0
|
||||
|
|
@ -816,9 +822,9 @@ packages:
|
|||
typescript:
|
||||
optional: true
|
||||
dependencies:
|
||||
'@typescript-eslint/scope-manager': 5.59.9
|
||||
'@typescript-eslint/types': 5.59.9
|
||||
'@typescript-eslint/typescript-estree': 5.59.9(typescript@5.1.3)
|
||||
'@typescript-eslint/scope-manager': 5.59.11
|
||||
'@typescript-eslint/types': 5.59.11
|
||||
'@typescript-eslint/typescript-estree': 5.59.11(typescript@5.1.3)
|
||||
debug: 4.3.4
|
||||
eslint: 8.42.0
|
||||
typescript: 5.1.3
|
||||
|
|
@ -826,16 +832,16 @@ packages:
|
|||
- supports-color
|
||||
dev: true
|
||||
|
||||
/@typescript-eslint/scope-manager@5.59.9:
|
||||
resolution: {integrity: sha512-8RA+E+w78z1+2dzvK/tGZ2cpGigBZ58VMEHDZtpE1v+LLjzrYGc8mMaTONSxKyEkz3IuXFM0IqYiGHlCsmlZxQ==}
|
||||
/@typescript-eslint/scope-manager@5.59.11:
|
||||
resolution: {integrity: sha512-dHFOsxoLFtrIcSj5h0QoBT/89hxQONwmn3FOQ0GOQcLOOXm+MIrS8zEAhs4tWl5MraxCY3ZJpaXQQdFMc2Tu+Q==}
|
||||
engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0}
|
||||
dependencies:
|
||||
'@typescript-eslint/types': 5.59.9
|
||||
'@typescript-eslint/visitor-keys': 5.59.9
|
||||
'@typescript-eslint/types': 5.59.11
|
||||
'@typescript-eslint/visitor-keys': 5.59.11
|
||||
dev: true
|
||||
|
||||
/@typescript-eslint/type-utils@5.59.9(eslint@8.42.0)(typescript@5.1.3):
|
||||
resolution: {integrity: sha512-ksEsT0/mEHg9e3qZu98AlSrONAQtrSTljL3ow9CGej8eRo7pe+yaC/mvTjptp23Xo/xIf2mLZKC6KPv4Sji26Q==}
|
||||
/@typescript-eslint/type-utils@5.59.11(eslint@8.42.0)(typescript@5.1.3):
|
||||
resolution: {integrity: sha512-LZqVY8hMiVRF2a7/swmkStMYSoXMFlzL6sXV6U/2gL5cwnLWQgLEG8tjWPpaE4rMIdZ6VKWwcffPlo1jPfk43g==}
|
||||
engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0}
|
||||
peerDependencies:
|
||||
eslint: '*'
|
||||
|
|
@ -844,8 +850,8 @@ packages:
|
|||
typescript:
|
||||
optional: true
|
||||
dependencies:
|
||||
'@typescript-eslint/typescript-estree': 5.59.9(typescript@5.1.3)
|
||||
'@typescript-eslint/utils': 5.59.9(eslint@8.42.0)(typescript@5.1.3)
|
||||
'@typescript-eslint/typescript-estree': 5.59.11(typescript@5.1.3)
|
||||
'@typescript-eslint/utils': 5.59.11(eslint@8.42.0)(typescript@5.1.3)
|
||||
debug: 4.3.4
|
||||
eslint: 8.42.0
|
||||
tsutils: 3.21.0(typescript@5.1.3)
|
||||
|
|
@ -854,13 +860,13 @@ packages:
|
|||
- supports-color
|
||||
dev: true
|
||||
|
||||
/@typescript-eslint/types@5.59.9:
|
||||
resolution: {integrity: sha512-uW8H5NRgTVneSVTfiCVffBb8AbwWSKg7qcA4Ot3JI3MPCJGsB4Db4BhvAODIIYE5mNj7Q+VJkK7JxmRhk2Lyjw==}
|
||||
/@typescript-eslint/types@5.59.11:
|
||||
resolution: {integrity: sha512-epoN6R6tkvBYSc+cllrz+c2sOFWkbisJZWkOE+y3xHtvYaOE6Wk6B8e114McRJwFRjGvYdJwLXQH5c9osME/AA==}
|
||||
engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0}
|
||||
dev: true
|
||||
|
||||
/@typescript-eslint/typescript-estree@5.59.9(typescript@5.1.3):
|
||||
resolution: {integrity: sha512-pmM0/VQ7kUhd1QyIxgS+aRvMgw+ZljB3eDb+jYyp6d2bC0mQWLzUDF+DLwCTkQ3tlNyVsvZRXjFyV0LkU/aXjA==}
|
||||
/@typescript-eslint/typescript-estree@5.59.11(typescript@5.1.3):
|
||||
resolution: {integrity: sha512-YupOpot5hJO0maupJXixi6l5ETdrITxeo5eBOeuV7RSKgYdU3G5cxO49/9WRnJq9EMrB7AuTSLH/bqOsXi7wPA==}
|
||||
engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0}
|
||||
peerDependencies:
|
||||
typescript: '*'
|
||||
|
|
@ -868,8 +874,8 @@ packages:
|
|||
typescript:
|
||||
optional: true
|
||||
dependencies:
|
||||
'@typescript-eslint/types': 5.59.9
|
||||
'@typescript-eslint/visitor-keys': 5.59.9
|
||||
'@typescript-eslint/types': 5.59.11
|
||||
'@typescript-eslint/visitor-keys': 5.59.11
|
||||
debug: 4.3.4
|
||||
globby: 11.1.0
|
||||
is-glob: 4.0.3
|
||||
|
|
@ -880,8 +886,8 @@ packages:
|
|||
- supports-color
|
||||
dev: true
|
||||
|
||||
/@typescript-eslint/utils@5.59.9(eslint@8.42.0)(typescript@5.1.3):
|
||||
resolution: {integrity: sha512-1PuMYsju/38I5Ggblaeb98TOoUvjhRvLpLa1DoTOFaLWqaXl/1iQ1eGurTXgBY58NUdtfTXKP5xBq7q9NDaLKg==}
|
||||
/@typescript-eslint/utils@5.59.11(eslint@8.42.0)(typescript@5.1.3):
|
||||
resolution: {integrity: sha512-didu2rHSOMUdJThLk4aZ1Or8IcO3HzCw/ZvEjTTIfjIrcdd5cvSIwwDy2AOlE7htSNp7QIZ10fLMyRCveesMLg==}
|
||||
engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0}
|
||||
peerDependencies:
|
||||
eslint: ^6.0.0 || ^7.0.0 || ^8.0.0
|
||||
|
|
@ -889,9 +895,9 @@ packages:
|
|||
'@eslint-community/eslint-utils': 4.4.0(eslint@8.42.0)
|
||||
'@types/json-schema': 7.0.12
|
||||
'@types/semver': 7.5.0
|
||||
'@typescript-eslint/scope-manager': 5.59.9
|
||||
'@typescript-eslint/types': 5.59.9
|
||||
'@typescript-eslint/typescript-estree': 5.59.9(typescript@5.1.3)
|
||||
'@typescript-eslint/scope-manager': 5.59.11
|
||||
'@typescript-eslint/types': 5.59.11
|
||||
'@typescript-eslint/typescript-estree': 5.59.11(typescript@5.1.3)
|
||||
eslint: 8.42.0
|
||||
eslint-scope: 5.1.1
|
||||
semver: 7.5.1
|
||||
|
|
@ -900,11 +906,11 @@ packages:
|
|||
- typescript
|
||||
dev: true
|
||||
|
||||
/@typescript-eslint/visitor-keys@5.59.9:
|
||||
resolution: {integrity: sha512-bT7s0td97KMaLwpEBckbzj/YohnvXtqbe2XgqNvTl6RJVakY5mvENOTPvw5u66nljfZxthESpDozs86U+oLY8Q==}
|
||||
/@typescript-eslint/visitor-keys@5.59.11:
|
||||
resolution: {integrity: sha512-KGYniTGG3AMTuKF9QBD7EIrvufkB6O6uX3knP73xbKLMpH+QRPcgnCxjWXSHjMRuOxFLovljqQgQpR0c7GvjoA==}
|
||||
engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0}
|
||||
dependencies:
|
||||
'@typescript-eslint/types': 5.59.9
|
||||
'@typescript-eslint/types': 5.59.11
|
||||
eslint-visitor-keys: 3.4.1
|
||||
dev: true
|
||||
|
||||
|
|
@ -2243,6 +2249,11 @@ packages:
|
|||
engines: {node: '>=8'}
|
||||
dev: true
|
||||
|
||||
/is-absolute-url@4.0.1:
|
||||
resolution: {integrity: sha512-/51/TKE88Lmm7Gc4/8btclNXWS+g50wXhYJq8HWIBAGUBnoAdRu1aXeh364t/O7wXDAcTJDP8PNuNKWUDWie+A==}
|
||||
engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}
|
||||
dev: false
|
||||
|
||||
/is-array-buffer@3.0.2:
|
||||
resolution: {integrity: sha512-y+FyyR/w8vfIRq4eQcM1EYgSTnmHXPqaF+IgzgraytCFq5Xh8lllDVmAZolPJiZttZLeFSINPYMaEJ7/vWUa1w==}
|
||||
dependencies:
|
||||
|
|
@ -2383,6 +2394,13 @@ packages:
|
|||
has-tostringtag: 1.0.0
|
||||
dev: true
|
||||
|
||||
/is-relative-url@4.0.0:
|
||||
resolution: {integrity: sha512-PkzoL1qKAYXNFct5IKdKRH/iBQou/oCC85QhXj6WKtUQBliZ4Yfd3Zk27RHu9KQG8r6zgvAA2AQKC9p+rqTszg==}
|
||||
engines: {node: '>=14.16'}
|
||||
dependencies:
|
||||
is-absolute-url: 4.0.1
|
||||
dev: false
|
||||
|
||||
/is-shared-array-buffer@1.0.2:
|
||||
resolution: {integrity: sha512-sqN2UDu1/0y6uvXyStCOzyhAjCSlHceFoMKJW8W9EU9cvic/QdsZ0kEU93HEy3IUEFZIiH/3w+AH/UQbPHNdhA==}
|
||||
dependencies:
|
||||
|
|
@ -2833,7 +2851,7 @@ packages:
|
|||
resolution: {integrity: sha512-8+Ib8rRJ4L0o3kfmyVCL7gzrohyDe0cMFTBa2d364yIrEGMEoetznKJx899YxjybU6bL9SQkYPSBBs1gyYs8Xg==}
|
||||
dependencies:
|
||||
'@sinonjs/commons': 2.0.0
|
||||
'@sinonjs/fake-timers': 10.2.0
|
||||
'@sinonjs/fake-timers': 10.1.0
|
||||
'@sinonjs/text-encoding': 0.7.2
|
||||
just-extend: 4.2.1
|
||||
path-to-regexp: 1.8.0
|
||||
|
|
@ -2880,6 +2898,11 @@ packages:
|
|||
engines: {node: '>=0.10.0'}
|
||||
dev: true
|
||||
|
||||
/normalize-url@8.0.0:
|
||||
resolution: {integrity: sha512-uVFpKhj5MheNBJRTiMZ9pE/7hD1QTeEvugSJW/OmLzAp78PB5O6adfMNTvmfKhXBkvCzC+rqifWcVYpGFwTjnw==}
|
||||
engines: {node: '>=14.16'}
|
||||
dev: false
|
||||
|
||||
/npm-run-all@4.1.5:
|
||||
resolution: {integrity: sha512-Oo82gJDAVcaMdi3nuoKFavkIHBRVqQ1qvMb+9LHk/cF4P6B2m8aP04hGf7oL6wZ9BuGwX1onlLhpuoofSyoQDQ==}
|
||||
engines: {node: '>= 4'}
|
||||
|
|
@ -3058,8 +3081,8 @@ packages:
|
|||
engines: {node: '>=12'}
|
||||
dev: true
|
||||
|
||||
/p-timeout@6.1.1:
|
||||
resolution: {integrity: sha512-yqz2Wi4fiFRpMmK0L2pGAU49naSUaP23fFIQL2Y6YT+qDGPoFwpvgQM/wzc6F8JoenUkIlAFa4Ql7NguXBxI7w==}
|
||||
/p-timeout@6.1.2:
|
||||
resolution: {integrity: sha512-UbD77BuZ9Bc9aABo74gfXhNvzC9Tx7SxtHSh1fxvx3jTLLYvmVhiQZZrJzqqU0jKbN32kb5VOKiLEQI/3bIjgQ==}
|
||||
engines: {node: '>=14.16'}
|
||||
dev: false
|
||||
|
||||
|
|
@ -3375,8 +3398,8 @@ packages:
|
|||
glob: 7.2.3
|
||||
dev: true
|
||||
|
||||
/rollup@3.24.1:
|
||||
resolution: {integrity: sha512-REHe5dx30ERBRFS0iENPHy+t6wtSEYkjrhwNsLyh3qpRaZ1+aylvMUdMBUHWUD/RjjLmLzEvY8Z9XRlpcdIkHA==}
|
||||
/rollup@3.25.1:
|
||||
resolution: {integrity: sha512-tywOR+rwIt5m2ZAWSe5AIJcTat8vGlnPFAv15ycCrw33t6iFsXZ6mzHVFh2psSjxQPmI+xgzMZZizUAukBI4aQ==}
|
||||
engines: {node: '>=14.18.0', npm: '>=8.0.0'}
|
||||
hasBin: true
|
||||
optionalDependencies:
|
||||
|
|
@ -3477,11 +3500,11 @@ packages:
|
|||
engines: {node: '>=14'}
|
||||
dev: true
|
||||
|
||||
/sinon@15.1.0:
|
||||
resolution: {integrity: sha512-cS5FgpDdE9/zx7no8bxROHymSlPLZzq0ChbbLk1DrxBfc+eTeBK3y8nIL+nu/0QeYydhhbLIr7ecHJpywjQaoQ==}
|
||||
/sinon@15.1.2:
|
||||
resolution: {integrity: sha512-uG1pU54Fis4EfYOPoEi13fmRHgZNg/u+3aReSEzHsN52Bpf+bMVfsBQS5MjouI+rTuG6UBIINlpuuO2Epr7SiA==}
|
||||
dependencies:
|
||||
'@sinonjs/commons': 3.0.0
|
||||
'@sinonjs/fake-timers': 10.2.0
|
||||
'@sinonjs/fake-timers': 10.1.0
|
||||
'@sinonjs/samsam': 8.0.0
|
||||
diff: 5.1.0
|
||||
nise: 5.1.4
|
||||
|
|
@ -3842,7 +3865,7 @@ packages:
|
|||
joycon: 3.1.1
|
||||
postcss-load-config: 3.1.4
|
||||
resolve-from: 5.0.0
|
||||
rollup: 3.24.1
|
||||
rollup: 3.25.1
|
||||
source-map: 0.8.0-beta.0
|
||||
sucrase: 3.32.0
|
||||
tree-kill: 1.2.2
|
||||
|
|
|
|||
|
|
@ -0,0 +1,13 @@
|
|||
import { ApifyClient } from 'apify-client'
|
||||
import 'dotenv/config'
|
||||
|
||||
async function main() {
|
||||
const apify = new ApifyClient({
|
||||
token: process.env.APIFY_API_KEY
|
||||
})
|
||||
|
||||
const actor = await apify.actor('apify/website-content-crawler').get()
|
||||
console.log(actor)
|
||||
}
|
||||
|
||||
main()
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
import ky from 'ky'
|
||||
import defaultKy from 'ky'
|
||||
|
||||
export const BING_BASE_URL = 'https://api.bing.microsoft.com'
|
||||
export const BING_API_BASE_URL = 'https://api.bing.microsoft.com'
|
||||
|
||||
export interface BingWebSearchQuery {
|
||||
q: string
|
||||
|
|
@ -235,22 +235,30 @@ interface FluffyContractualRule {
|
|||
}
|
||||
|
||||
export class BingWebSearchClient {
|
||||
api: typeof defaultKy
|
||||
|
||||
apiKey: string
|
||||
baseUrl: string
|
||||
apiBaseUrl: string
|
||||
|
||||
constructor({
|
||||
apiKey = process.env.BING_API_KEY,
|
||||
baseUrl = BING_BASE_URL
|
||||
apiBaseUrl = BING_API_BASE_URL,
|
||||
ky = defaultKy
|
||||
}: {
|
||||
apiKey?: string
|
||||
baseUrl?: string
|
||||
apiBaseUrl?: string
|
||||
ky?: typeof defaultKy
|
||||
} = {}) {
|
||||
if (!apiKey) {
|
||||
throw new Error(`Error BingWebSearchClient missing required "apiKey"`)
|
||||
}
|
||||
|
||||
this.apiKey = apiKey
|
||||
this.baseUrl = baseUrl
|
||||
this.apiBaseUrl = apiBaseUrl
|
||||
|
||||
this.api = ky.extend({
|
||||
prefixUrl: this.apiBaseUrl
|
||||
})
|
||||
}
|
||||
|
||||
async search(queryOrOpts: string | BingWebSearchQuery) {
|
||||
|
|
@ -269,9 +277,9 @@ export class BingWebSearchClient {
|
|||
...queryOrOpts
|
||||
}
|
||||
|
||||
console.log(searchParams)
|
||||
return ky
|
||||
.get(`${this.baseUrl}/v7.0/search`, {
|
||||
// console.log(searchParams)
|
||||
return this.api
|
||||
.get('v7.0/search', {
|
||||
headers: {
|
||||
'Ocp-Apim-Subscription-Key': this.apiKey
|
||||
},
|
||||
|
|
|
|||
|
|
@ -0,0 +1,303 @@
|
|||
import defaultKy from 'ky'
|
||||
|
||||
export const DIFFBOT_API_BASE_URL = 'https://api.diffbot.com'
|
||||
export const DIFFBOT_KNOWLEDGE_GRAPH_API_BASE_URL = 'https://kg.diffbot.com'
|
||||
|
||||
export interface DiffbotExtractOptions {
|
||||
/** Specify optional fields to be returned from any fully-extracted pages, e.g.: &fields=querystring,links. See available fields within each API's individual documentation pages.
|
||||
* @see https://docs.diffbot.com/reference/extract-optional-fields
|
||||
*/
|
||||
fields?: string[]
|
||||
|
||||
/** (*Undocumented*) Pass paging=false to disable automatic concatenation of multiple-page articles. (By default, Diffbot will concatenate up to 20 pages of a single article.) */
|
||||
paging?: boolean
|
||||
|
||||
/** Pass discussion=false to disable automatic extraction of comments or reviews from pages identified as articles or products. This will not affect pages identified as discussions. */
|
||||
discussion?: boolean
|
||||
|
||||
/** Sets a value in milliseconds to wait for the retrieval/fetch of content from the requested URL. The default timeout for the third-party response is 30 seconds (30000). */
|
||||
timeout?: number
|
||||
|
||||
/** Used to specify the IP address of a custom proxy that will be used to fetch the target page, instead of Diffbot's default IPs/proxies. (Ex: &proxy=168.212.226.204) */
|
||||
proxy?: string
|
||||
|
||||
/** Used to specify the authentication parameters that will be used with the proxy specified in the &proxy parameter. (Ex: &proxyAuth=username:password) */
|
||||
proxyAuth?: string
|
||||
|
||||
/** `none` will instruct Extract to not use proxies, even if proxies have been enabled for this particular URL globally. */
|
||||
useProxy?: string
|
||||
|
||||
/** @see https://docs.diffbot.com/reference/extract-custom-javascript */
|
||||
customJs?: string
|
||||
|
||||
/** @see https://docs.diffbot.com/reference/extract-custom-headers */
|
||||
customHeaders?: Record<string, string>
|
||||
}
|
||||
|
||||
export interface DiffbotExtractAnalyzeOptions extends DiffbotExtractOptions {
|
||||
/** Web page URL of the analyze to process */
|
||||
url: string
|
||||
|
||||
/** By default the Analyze API will fully extract all pages that match an existing Automatic API -- articles, products or image pages. Set mode to a specific page-type (e.g., mode=article) to extract content only from that specific page-type. All other pages will simply return the default Analyze fields. */
|
||||
mode?: string
|
||||
|
||||
/** Force any non-extracted pages (those with a type of "other") through a specific API. For example, to route all "other" pages through the Article API, pass &fallback=article. Pages that utilize this functionality will return a fallbackType field at the top-level of the response and a originalType field within each extracted object, both of which will indicate the fallback API used. */
|
||||
fallback?: string
|
||||
}
|
||||
|
||||
export interface DiffbotExtractArticleOptions extends DiffbotExtractOptions {
|
||||
/** Web page URL of the analyze to process */
|
||||
url: string
|
||||
|
||||
/** Set the maximum number of automatically-generated tags to return. By default a maximum of ten tags will be returned. */
|
||||
maxTags?: number
|
||||
|
||||
/** Set the minimum relevance score of tags to return, between 0.0 and 1.0. By default only tags with a score equal to or above 0.5 will be returned. */
|
||||
tagConfidence?: number
|
||||
|
||||
/** Used to request the output of the Diffbot Natural Language API in the field naturalLanguage. Example: &naturalLanguage=entities,facts,categories,sentiment. */
|
||||
naturalLanguage?: string[]
|
||||
}
|
||||
|
||||
export interface DiffbotExtractResponse {
|
||||
request: DiffbotRequest
|
||||
objects: DiffbotObject[]
|
||||
}
|
||||
|
||||
export type DiffbotExtractArticleResponse = DiffbotExtractResponse
|
||||
|
||||
export interface DiffbotExtractAnalyzeResponse extends DiffbotExtractResponse {
|
||||
type: string
|
||||
title: string
|
||||
humanLanguage: string
|
||||
}
|
||||
|
||||
export interface DiffbotObject {
|
||||
date: string
|
||||
sentiment: number
|
||||
images: DiffbotImage[]
|
||||
author: string
|
||||
estimatedDate: string
|
||||
publisherRegion: string
|
||||
icon: string
|
||||
diffbotUri: string
|
||||
siteName: string
|
||||
type: string
|
||||
title: string
|
||||
tags: DiffbotTag[]
|
||||
publisherCountry: string
|
||||
humanLanguage: string
|
||||
authorUrl: string
|
||||
pageUrl: string
|
||||
html: string
|
||||
text: string
|
||||
categories?: DiffbotCategory[]
|
||||
authors: DiffbotAuthor[]
|
||||
breadcrumb?: DiffbotBreadcrumb[]
|
||||
meta?: any
|
||||
}
|
||||
|
||||
interface DiffbotAuthor {
|
||||
name: string
|
||||
link: string
|
||||
}
|
||||
|
||||
interface DiffbotCategory {
|
||||
score: number
|
||||
name: string
|
||||
id: string
|
||||
}
|
||||
|
||||
export interface DiffbotBreadcrumb {
|
||||
link: string
|
||||
name: string
|
||||
}
|
||||
|
||||
interface DiffbotImage {
|
||||
url: string
|
||||
diffbotUri: string
|
||||
|
||||
naturalWidth: number
|
||||
naturalHeight: number
|
||||
width: number
|
||||
height: number
|
||||
|
||||
isCached?: boolean
|
||||
primary?: boolean
|
||||
}
|
||||
|
||||
interface DiffbotTag {
|
||||
score: number
|
||||
sentiment: number
|
||||
count: number
|
||||
label: string
|
||||
uri: string
|
||||
rdfTypes: string[]
|
||||
}
|
||||
|
||||
interface DiffbotRequest {
|
||||
pageUrl: string
|
||||
api: string
|
||||
version: number
|
||||
}
|
||||
|
||||
export interface Image {
|
||||
naturalHeight: number
|
||||
diffbotUri: string
|
||||
url: string
|
||||
naturalWidth: number
|
||||
primary: boolean
|
||||
}
|
||||
|
||||
export interface Tag {
|
||||
score: number
|
||||
sentiment: number
|
||||
count: number
|
||||
label: string
|
||||
uri: string
|
||||
rdfTypes: string[]
|
||||
}
|
||||
|
||||
export interface Request {
|
||||
pageUrl: string
|
||||
api: string
|
||||
version: number
|
||||
}
|
||||
|
||||
export interface DiffbotSearchKnowledgeGraphOptions {
|
||||
type?: 'query' | 'text' | 'queryTextFallback' | 'crawl'
|
||||
query: string
|
||||
col?: string
|
||||
from?: number
|
||||
size?: number
|
||||
|
||||
// NOTE: we only support `json`, so these options are not needed
|
||||
// We can always convert from json to another format if needed.
|
||||
// format?: 'json' | 'jsonl' | 'csv' | 'xls' | 'xlsx'
|
||||
// exportspec?: string
|
||||
// exportseparator?: string
|
||||
// exportfile?: string
|
||||
|
||||
filter?: string
|
||||
jsonmode?: 'extended' | 'id'
|
||||
nonCanonicalFacts?: boolean
|
||||
noDedupArticles?: boolean
|
||||
cluster?: 'all' | 'best' | 'dedupe'
|
||||
report?: boolean
|
||||
}
|
||||
|
||||
export interface DiffbotSearchKnowledgeGraphResponse {
|
||||
version: number
|
||||
hits: number
|
||||
results: number
|
||||
kgversion: string
|
||||
diffbot_type: string
|
||||
facet: boolean
|
||||
data: DiffbotKnowledgeGraphNode[]
|
||||
}
|
||||
|
||||
export interface DiffbotKnowledgeGraphNode {
|
||||
score: number
|
||||
entity: DiffbotKnowledgeGraphEntity
|
||||
entity_ctx: any
|
||||
errors: string[]
|
||||
callbackQuery: string
|
||||
upperBound: number
|
||||
lowerBound: number
|
||||
count: number
|
||||
value: string
|
||||
uri: string
|
||||
}
|
||||
|
||||
export interface DiffbotKnowledgeGraphEntity {
|
||||
id: string
|
||||
images: DiffbotImage[]
|
||||
diffbotUri: string
|
||||
name: string
|
||||
origins: string[]
|
||||
}
|
||||
|
||||
export class DiffbotClient {
|
||||
api: typeof defaultKy
|
||||
apiKnowledgeGraph: typeof defaultKy
|
||||
|
||||
apiKey: string
|
||||
apiBaseUrl: string
|
||||
apiKnowledgeGraphBaseUrl: string
|
||||
|
||||
constructor({
|
||||
apiKey = process.env.DIFFBOT_API_KEY,
|
||||
apiBaseUrl = DIFFBOT_API_BASE_URL,
|
||||
apiKnowledgeGraphBaseUrl = DIFFBOT_KNOWLEDGE_GRAPH_API_BASE_URL,
|
||||
timeoutMs = 60_000,
|
||||
ky = defaultKy
|
||||
}: {
|
||||
apiKey?: string
|
||||
apiBaseUrl?: string
|
||||
apiKnowledgeGraphBaseUrl?: string
|
||||
timeoutMs?: number
|
||||
ky?: typeof defaultKy
|
||||
} = {}) {
|
||||
if (!apiKey) {
|
||||
throw new Error(`Error DiffbotClient missing required "apiKey"`)
|
||||
}
|
||||
|
||||
this.apiKey = apiKey
|
||||
this.apiBaseUrl = apiBaseUrl
|
||||
this.apiKnowledgeGraphBaseUrl = apiKnowledgeGraphBaseUrl
|
||||
|
||||
this.api = ky.extend({ prefixUrl: apiBaseUrl, timeout: timeoutMs })
|
||||
this.apiKnowledgeGraph = ky.extend({
|
||||
prefixUrl: apiKnowledgeGraphBaseUrl,
|
||||
timeout: timeoutMs
|
||||
})
|
||||
}
|
||||
|
||||
protected async _extract<
|
||||
T extends DiffbotExtractResponse = DiffbotExtractResponse
|
||||
>(endpoint: string, options: DiffbotExtractOptions): Promise<T> {
|
||||
const { customJs, customHeaders, ...rest } = options
|
||||
const searchParams: Record<string, any> = {
|
||||
...rest,
|
||||
token: this.apiKey
|
||||
}
|
||||
const headers = {
|
||||
...Object.fromEntries(
|
||||
[['X-Forward-X-Evaluate', customJs]].filter(([, value]) => value)
|
||||
),
|
||||
...customHeaders
|
||||
}
|
||||
|
||||
for (const [key, value] of Object.entries(rest)) {
|
||||
if (Array.isArray(value)) {
|
||||
searchParams[key] = value.join(',')
|
||||
}
|
||||
}
|
||||
|
||||
return this.api
|
||||
.get(endpoint, {
|
||||
searchParams,
|
||||
headers
|
||||
})
|
||||
.json<T>()
|
||||
}
|
||||
|
||||
async extractAnalyze(options: DiffbotExtractAnalyzeOptions) {
|
||||
return this._extract<DiffbotExtractAnalyzeResponse>('v3/analyze', options)
|
||||
}
|
||||
|
||||
async extractArticle(options: DiffbotExtractArticleOptions) {
|
||||
return this._extract<DiffbotExtractArticleResponse>('v3/article', options)
|
||||
}
|
||||
|
||||
async searchKnowledgeGraph(options: DiffbotSearchKnowledgeGraphOptions) {
|
||||
return this.apiKnowledgeGraph
|
||||
.get('kg/v3/dql', {
|
||||
searchParams: {
|
||||
...options,
|
||||
token: this.apiKey
|
||||
}
|
||||
})
|
||||
.json<DiffbotSearchKnowledgeGraphResponse>()
|
||||
}
|
||||
}
|
||||
|
|
@ -1,4 +1,5 @@
|
|||
export * from './bing-web-search'
|
||||
export * from './diffbot'
|
||||
export * from './metaphor'
|
||||
export * from './novu'
|
||||
export * from './serpapi'
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import ky from 'ky'
|
||||
import defaultKy from 'ky'
|
||||
import { z } from 'zod'
|
||||
|
||||
export const METAPHOR_BASE_URL = 'https://api.metaphor.systems'
|
||||
export const METAPHOR_API_BASE_URL = 'https://api.metaphor.systems'
|
||||
|
||||
// https://metaphorapi.readme.io/reference/search
|
||||
export const MetaphorSearchInputSchema = z.object({
|
||||
|
|
@ -33,27 +33,35 @@ export const MetaphorSearchOutputSchema = z.object({
|
|||
export type MetaphorSearchOutput = z.infer<typeof MetaphorSearchOutputSchema>
|
||||
|
||||
export class MetaphorClient {
|
||||
api: typeof defaultKy
|
||||
|
||||
apiKey: string
|
||||
baseUrl: string
|
||||
apiBaseUrl: string
|
||||
|
||||
constructor({
|
||||
apiKey = process.env.METAPHOR_API_KEY,
|
||||
baseUrl = METAPHOR_BASE_URL
|
||||
apiBaseUrl = METAPHOR_API_BASE_URL,
|
||||
ky = defaultKy
|
||||
}: {
|
||||
apiKey?: string
|
||||
baseUrl?: string
|
||||
apiBaseUrl?: string
|
||||
ky?: typeof defaultKy
|
||||
} = {}) {
|
||||
if (!apiKey) {
|
||||
throw new Error(`Error MetaphorClient missing required "apiKey"`)
|
||||
}
|
||||
|
||||
this.apiKey = apiKey
|
||||
this.baseUrl = baseUrl
|
||||
this.apiBaseUrl = apiBaseUrl
|
||||
|
||||
this.api = ky.extend({
|
||||
prefixUrl: this.apiBaseUrl
|
||||
})
|
||||
}
|
||||
|
||||
async search(params: MetaphorSearchInput) {
|
||||
return ky
|
||||
.post(`${this.baseUrl}/search`, {
|
||||
return this.api
|
||||
.post('search', {
|
||||
headers: {
|
||||
'x-api-key': this.apiKey
|
||||
},
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import ky from 'ky'
|
||||
import defaultKy from 'ky'
|
||||
|
||||
export const NOVU_BASE_URL = 'https://api.novu.co/v1'
|
||||
export const NOVU_API_BASE_URL = 'https://api.novu.co/v1'
|
||||
|
||||
export type NovuSubscriber = {
|
||||
subscriberId: string
|
||||
|
|
@ -19,22 +19,30 @@ export type NovuTriggerEventResponse = {
|
|||
}
|
||||
|
||||
export class NovuClient {
|
||||
api: typeof defaultKy
|
||||
|
||||
apiKey: string
|
||||
baseUrl: string
|
||||
apiBaseUrl: string
|
||||
|
||||
constructor({
|
||||
apiKey = process.env.NOVU_API_KEY,
|
||||
baseUrl = NOVU_BASE_URL
|
||||
apiBaseUrl = NOVU_API_BASE_URL,
|
||||
ky = defaultKy
|
||||
}: {
|
||||
apiKey?: string
|
||||
baseUrl?: string
|
||||
apiBaseUrl?: string
|
||||
ky?: typeof defaultKy
|
||||
} = {}) {
|
||||
if (!apiKey) {
|
||||
throw new Error(`Error NovuClient missing required "apiKey"`)
|
||||
}
|
||||
|
||||
this.apiKey = apiKey
|
||||
this.baseUrl = baseUrl
|
||||
this.apiBaseUrl = apiBaseUrl
|
||||
|
||||
this.api = ky.extend({
|
||||
prefixUrl: this.apiBaseUrl
|
||||
})
|
||||
}
|
||||
|
||||
async triggerEvent({
|
||||
|
|
@ -46,19 +54,18 @@ export class NovuClient {
|
|||
payload: Record<string, unknown>
|
||||
to: NovuSubscriber[]
|
||||
}) {
|
||||
const url = `${this.baseUrl}/events/trigger`
|
||||
const headers = {
|
||||
Authorization: `ApiKey ${this.apiKey}`,
|
||||
'Content-Type': 'application/json'
|
||||
}
|
||||
const response = await ky.post(url, {
|
||||
headers,
|
||||
json: {
|
||||
name,
|
||||
payload,
|
||||
to
|
||||
}
|
||||
})
|
||||
return response.json<NovuTriggerEventResponse>()
|
||||
return this.api
|
||||
.post('events/trigger', {
|
||||
headers: {
|
||||
Authorization: `ApiKey ${this.apiKey}`,
|
||||
'Content-Type': 'application/json'
|
||||
},
|
||||
json: {
|
||||
name,
|
||||
payload,
|
||||
to
|
||||
}
|
||||
})
|
||||
.json<NovuTriggerEventResponse>()
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import ky from 'ky'
|
||||
import defaultKy from 'ky'
|
||||
|
||||
/**
|
||||
* All types have been exported from the `serpapi` package, which we're
|
||||
|
|
@ -350,7 +350,8 @@ export type SerpAPISearchResponse = BaseResponse<GoogleParameters>
|
|||
|
||||
export interface SerpAPIClientOptions extends Partial<SerpAPIParams> {
|
||||
apiKey?: string
|
||||
baseUrl?: string
|
||||
apiBaseUrl?: string
|
||||
ky?: typeof defaultKy
|
||||
}
|
||||
|
||||
export const SERPAPI_BASE_URL = 'https://serpapi.com'
|
||||
|
|
@ -361,13 +362,16 @@ export const SERPAPI_BASE_URL = 'https://serpapi.com'
|
|||
* @see https://serpapi.com/search-api
|
||||
*/
|
||||
export class SerpAPIClient {
|
||||
api: typeof defaultKy
|
||||
|
||||
apiKey: string
|
||||
baseUrl: string
|
||||
apiBaseUrl: string
|
||||
params: Partial<SerpAPIParams>
|
||||
|
||||
constructor({
|
||||
apiKey = process.env.SERPAPI_API_KEY ?? process.env.SERP_API_KEY,
|
||||
baseUrl = SERPAPI_BASE_URL,
|
||||
apiBaseUrl = SERPAPI_BASE_URL,
|
||||
ky = defaultKy,
|
||||
...params
|
||||
}: SerpAPIClientOptions = {}) {
|
||||
if (!apiKey) {
|
||||
|
|
@ -375,8 +379,12 @@ export class SerpAPIClient {
|
|||
}
|
||||
|
||||
this.apiKey = apiKey
|
||||
this.baseUrl = baseUrl
|
||||
this.apiBaseUrl = apiBaseUrl
|
||||
this.params = params
|
||||
|
||||
this.api = ky.extend({
|
||||
prefixUrl: this.apiBaseUrl
|
||||
})
|
||||
}
|
||||
|
||||
async search(queryOrOpts: string | { query: string }) {
|
||||
|
|
@ -384,8 +392,8 @@ export class SerpAPIClient {
|
|||
typeof queryOrOpts === 'string' ? queryOrOpts : queryOrOpts.query
|
||||
const { timeout, ...rest } = this.params
|
||||
|
||||
return ky
|
||||
.get(`${this.baseUrl}/search`, {
|
||||
return this.api
|
||||
.get('search', {
|
||||
searchParams: {
|
||||
...rest,
|
||||
engine: 'google',
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import ky from 'ky'
|
||||
import defaultKy from 'ky'
|
||||
|
||||
import { sleep } from '@/utils'
|
||||
|
||||
|
|
@ -256,30 +256,33 @@ export type SlackSendAndWaitOptions = {
|
|||
}
|
||||
|
||||
export class SlackClient {
|
||||
private api: typeof ky
|
||||
protected api: typeof defaultKy
|
||||
|
||||
protected defaultChannel?: string
|
||||
|
||||
constructor({
|
||||
apiKey = process.env.SLACK_API_KEY,
|
||||
baseUrl = SLACK_API_BASE_URL,
|
||||
defaultChannel = process.env.SLACK_DEFAULT_CHANNEL
|
||||
defaultChannel = process.env.SLACK_DEFAULT_CHANNEL,
|
||||
ky = defaultKy
|
||||
}: {
|
||||
apiKey?: string
|
||||
baseUrl?: string
|
||||
defaultChannel?: string
|
||||
ky?: typeof defaultKy
|
||||
} = {}) {
|
||||
if (!apiKey) {
|
||||
throw new Error(`Error SlackClient missing required "apiKey"`)
|
||||
}
|
||||
|
||||
this.api = ky.create({
|
||||
this.defaultChannel = defaultChannel
|
||||
|
||||
this.api = ky.extend({
|
||||
prefixUrl: baseUrl,
|
||||
headers: {
|
||||
Authorization: `Bearer ${apiKey}`
|
||||
}
|
||||
})
|
||||
this.defaultChannel = defaultChannel
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -328,10 +331,11 @@ export class SlackClient {
|
|||
* Returns a list of messages that were sent in a channel after a given timestamp both directly and in threads.
|
||||
*/
|
||||
private async fetchCandidates(channel: string, ts: string) {
|
||||
let candidates: SlackMessage[] = []
|
||||
const history = await this.fetchConversationHistory({ channel })
|
||||
const directReplies = await this.fetchReplies({ channel, ts })
|
||||
|
||||
let candidates: SlackMessage[] = []
|
||||
|
||||
if (directReplies.ok) {
|
||||
candidates = candidates.concat(directReplies.messages)
|
||||
}
|
||||
|
|
@ -349,6 +353,7 @@ export class SlackClient {
|
|||
candidates.sort((a, b) => {
|
||||
return parseFloat(b.ts) - parseFloat(a.ts)
|
||||
})
|
||||
|
||||
return candidates
|
||||
}
|
||||
|
||||
|
|
@ -357,7 +362,7 @@ export class SlackClient {
|
|||
*
|
||||
* ### Notes
|
||||
*
|
||||
* - The implementation will poll for replies to the message until the timeout is reached. This is not ideal, but it is the only way to retrieve replies to a message in Slack without spinning up a local server to receive webhook events.
|
||||
* - The implementation will poll for replies to the message until the timeout is reached. This is not ideal, but it is the only way to retrieve replies to a message in Slack without spinning up a local server to receive webhook events.
|
||||
*/
|
||||
public async sendAndWaitForReply({
|
||||
text,
|
||||
|
|
|
|||
|
|
@ -1,9 +1,9 @@
|
|||
import ky, { type KyResponse } from 'ky'
|
||||
import defaultKy from 'ky'
|
||||
|
||||
import { DEFAULT_BOT_NAME } from '@/constants'
|
||||
import { sleep } from '@/utils'
|
||||
|
||||
export const TWILIO_CONVERSATION_BASE_URL =
|
||||
export const TWILIO_CONVERSATION_API_BASE_URL =
|
||||
'https://conversations.twilio.com/v1'
|
||||
|
||||
export const DEFAULT_TWILIO_TIMEOUT_MS = 120_000
|
||||
|
|
@ -129,7 +129,8 @@ export type TwilioSendAndWaitOptions = {
|
|||
* @see {@link https://www.twilio.com/docs/conversations/api}
|
||||
*/
|
||||
export class TwilioConversationClient {
|
||||
api: typeof ky
|
||||
api: typeof defaultKy
|
||||
|
||||
phoneNumber: string
|
||||
botName: string
|
||||
|
||||
|
|
@ -137,18 +138,26 @@ export class TwilioConversationClient {
|
|||
accountSid = process.env.TWILIO_ACCOUNT_SID,
|
||||
authToken = process.env.TWILIO_AUTH_TOKEN,
|
||||
phoneNumber = process.env.TWILIO_PHONE_NUMBER,
|
||||
baseUrl = TWILIO_CONVERSATION_BASE_URL,
|
||||
botName = DEFAULT_BOT_NAME
|
||||
apiBaseUrl = TWILIO_CONVERSATION_API_BASE_URL,
|
||||
botName = DEFAULT_BOT_NAME,
|
||||
ky = defaultKy
|
||||
}: {
|
||||
accountSid?: string
|
||||
authToken?: string
|
||||
phoneNumber?: string
|
||||
baseUrl?: string
|
||||
apiBaseUrl?: string
|
||||
botName?: string
|
||||
ky?: typeof defaultKy
|
||||
} = {}) {
|
||||
if (!accountSid || !authToken) {
|
||||
if (!accountSid) {
|
||||
throw new Error(
|
||||
`Error TwilioConversationClient missing required "accountSid" and/or "authToken"`
|
||||
`Error TwilioConversationClient missing required "accountSid"`
|
||||
)
|
||||
}
|
||||
|
||||
if (!authToken) {
|
||||
throw new Error(
|
||||
`Error TwilioConversationClient missing required "authToken"`
|
||||
)
|
||||
}
|
||||
|
||||
|
|
@ -161,8 +170,8 @@ export class TwilioConversationClient {
|
|||
this.botName = botName
|
||||
this.phoneNumber = phoneNumber
|
||||
|
||||
this.api = ky.create({
|
||||
prefixUrl: baseUrl,
|
||||
this.api = ky.extend({
|
||||
prefixUrl: apiBaseUrl,
|
||||
headers: {
|
||||
Authorization:
|
||||
'Basic ' +
|
||||
|
|
@ -175,7 +184,7 @@ export class TwilioConversationClient {
|
|||
/**
|
||||
* Deletes a conversation and all its messages.
|
||||
*/
|
||||
async deleteConversation(conversationSid: string): Promise<KyResponse> {
|
||||
async deleteConversation(conversationSid: string) {
|
||||
return this.api.delete(`Conversations/${conversationSid}`)
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,44 @@
|
|||
import isRelativeUrl from 'is-relative-url'
|
||||
import normalizeUrlImpl from 'normalize-url'
|
||||
import QuickLRU from 'quick-lru'
|
||||
|
||||
// const protocolAllowList = new Set(['https:', 'http:'])
|
||||
const normalizedUrlCache = new QuickLRU<string, string | null>({
|
||||
maxSize: 4000
|
||||
})
|
||||
|
||||
export function normalizeUrl(url: string): string | null {
|
||||
let normalizedUrl: string | null | undefined
|
||||
|
||||
try {
|
||||
if (!url || (isRelativeUrl(url) && !url.startsWith('//'))) {
|
||||
return null
|
||||
}
|
||||
|
||||
normalizedUrl = normalizedUrlCache.get(url)
|
||||
|
||||
if (normalizedUrl !== undefined) {
|
||||
return normalizedUrl
|
||||
}
|
||||
|
||||
normalizedUrl = normalizeUrlImpl(url, {
|
||||
stripWWW: false,
|
||||
defaultProtocol: 'https',
|
||||
normalizeProtocol: true,
|
||||
forceHttps: false,
|
||||
stripHash: false,
|
||||
stripTextFragment: true,
|
||||
removeQueryParameters: [/^utm_\w+/i, 'ref'],
|
||||
removeTrailingSlash: true,
|
||||
removeSingleSlash: true,
|
||||
removeExplicitPort: true,
|
||||
sortQueryParameters: true
|
||||
})
|
||||
} catch (err) {
|
||||
// ignore invalid urls
|
||||
normalizedUrl = null
|
||||
}
|
||||
|
||||
normalizedUrlCache.set(url, normalizedUrl!)
|
||||
return normalizedUrl
|
||||
}
|
||||
|
|
@ -4,10 +4,12 @@ import 'dotenv/config'
|
|||
import hashObject from 'hash-obj'
|
||||
import Redis from 'ioredis'
|
||||
import Keyv from 'keyv'
|
||||
import defaultKy from 'ky'
|
||||
import { OpenAIClient } from 'openai-fetch'
|
||||
import pMemoize from 'p-memoize'
|
||||
|
||||
import { Agentic } from '@/agentic'
|
||||
import { normalizeUrl } from '@/url-utils'
|
||||
|
||||
export const fakeOpenAIAPIKey = 'fake-openai-api-key'
|
||||
export const fakeAnthropicAPIKey = 'fake-anthropic-api-key'
|
||||
|
|
@ -39,6 +41,97 @@ keyv.has = async (key, ...rest) => {
|
|||
return res
|
||||
}
|
||||
|
||||
function getCacheKeyForRequest(request: Request): string | null {
|
||||
const method = request.method.toLowerCase()
|
||||
|
||||
if (method === 'get') {
|
||||
const url = normalizeUrl(request.url)
|
||||
|
||||
if (url) {
|
||||
const cacheParams = {
|
||||
// TODO: request.headers isn't a normal JS object...
|
||||
headers: { ...request.headers }
|
||||
}
|
||||
|
||||
// console.log('getCacheKeyForRequest', { url, cacheParams })
|
||||
const cacheKey = getCacheKey(`http:${method} ${url}`, cacheParams)
|
||||
return cacheKey
|
||||
}
|
||||
}
|
||||
|
||||
return null
|
||||
}
|
||||
|
||||
/**
|
||||
* Custom `ky` instance that caches GET JSON requests.
|
||||
*
|
||||
* TODO:
|
||||
* - support non-GET requests
|
||||
* - support non-JSON responses
|
||||
*/
|
||||
export const ky = defaultKy.extend({
|
||||
hooks: {
|
||||
beforeRequest: [
|
||||
async (request) => {
|
||||
try {
|
||||
// console.log(`beforeRequest ${request.method} ${request.url}`)
|
||||
|
||||
const cacheKey = getCacheKeyForRequest(request)
|
||||
// console.log({ cacheKey })
|
||||
if (!cacheKey) {
|
||||
return
|
||||
}
|
||||
|
||||
if (!(await keyv.has(cacheKey))) {
|
||||
return
|
||||
}
|
||||
|
||||
const cachedResponse = await keyv.get(cacheKey)
|
||||
// console.log({ cachedResponse })
|
||||
|
||||
if (!cachedResponse) {
|
||||
return
|
||||
}
|
||||
|
||||
return new Response(JSON.stringify(cachedResponse), {
|
||||
status: 200,
|
||||
headers: { 'Content-Type': 'application/json' }
|
||||
})
|
||||
} catch (err) {
|
||||
console.error('ky beforeResponse cache error', err)
|
||||
}
|
||||
}
|
||||
],
|
||||
|
||||
afterResponse: [
|
||||
async (request, _options, response) => {
|
||||
try {
|
||||
// console.log(
|
||||
// `afterRequest ${request.method} ${request.url} ⇒ ${response.status}`
|
||||
// )
|
||||
|
||||
if (response.status < 200 || response.status >= 300) {
|
||||
return
|
||||
}
|
||||
|
||||
const cacheKey = getCacheKeyForRequest(request)
|
||||
// console.log({ cacheKey })
|
||||
if (!cacheKey) {
|
||||
return
|
||||
}
|
||||
|
||||
const responseBody = await response.json()
|
||||
// console.log({ responseBody })
|
||||
|
||||
await keyv.set(cacheKey, responseBody)
|
||||
} catch (err) {
|
||||
console.error('ky afterResponse cache error', err)
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
})
|
||||
|
||||
export class OpenAITestClient extends OpenAIClient {
|
||||
createChatCompletion = pMemoize(super.createChatCompletion, {
|
||||
cacheKey: (params) => getCacheKey('openai:chat', params),
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import test from 'ava'
|
|||
|
||||
import { BingWebSearchClient } from '@/services'
|
||||
|
||||
import './_utils'
|
||||
import { ky } from '../_utils'
|
||||
|
||||
test('BingWebSearchClient.search', async (t) => {
|
||||
if (!process.env.BING_API_KEY) {
|
||||
|
|
@ -10,9 +10,9 @@ test('BingWebSearchClient.search', async (t) => {
|
|||
}
|
||||
|
||||
t.timeout(2 * 60 * 1000)
|
||||
const client = new BingWebSearchClient()
|
||||
const client = new BingWebSearchClient({ ky })
|
||||
|
||||
const result = await client.search('coffee')
|
||||
// console.log(result)
|
||||
t.truthy(result?.webPages)
|
||||
t.true(result?.webPages?.value?.length > 0)
|
||||
})
|
||||
|
|
@ -0,0 +1,37 @@
|
|||
import test from 'ava'
|
||||
|
||||
import { DiffbotClient } from '@/services'
|
||||
|
||||
import { isCI, ky } from '../_utils'
|
||||
|
||||
test('Diffbot.analyze', async (t) => {
|
||||
if (!process.env.DIFFBOT_API_KEY || isCI) {
|
||||
return t.pass()
|
||||
}
|
||||
|
||||
t.timeout(2 * 60 * 1000)
|
||||
const client = new DiffbotClient({ ky })
|
||||
|
||||
const result = await client.extractAnalyze({
|
||||
url: 'https://transitivebullsh.it'
|
||||
})
|
||||
// console.log(result)
|
||||
t.is(result.type, 'list')
|
||||
t.is(result.objects?.length, 1)
|
||||
})
|
||||
|
||||
test('Diffbot.article', async (t) => {
|
||||
if (!process.env.DIFFBOT_API_KEY || isCI) {
|
||||
return t.pass()
|
||||
}
|
||||
|
||||
t.timeout(2 * 60 * 1000)
|
||||
const client = new DiffbotClient({ ky })
|
||||
|
||||
const result = await client.extractArticle({
|
||||
url: 'https://www.nytimes.com/2023/05/31/magazine/ai-start-up-accelerator-san-francisco.html'
|
||||
// fields: ['meta']
|
||||
})
|
||||
// console.log(JSON.stringify(result, null, 2))
|
||||
t.is(result.objects[0].type, 'article')
|
||||
})
|
||||
|
|
@ -2,7 +2,7 @@ import test from 'ava'
|
|||
|
||||
import { NovuClient } from '@/services/novu'
|
||||
|
||||
import './_utils'
|
||||
import { ky } from '../_utils'
|
||||
|
||||
test('NovuClient.triggerEvent', async (t) => {
|
||||
if (!process.env.NOVU_API_KEY) {
|
||||
|
|
@ -10,7 +10,7 @@ test('NovuClient.triggerEvent', async (t) => {
|
|||
}
|
||||
|
||||
t.timeout(2 * 60 * 1000)
|
||||
const client = new NovuClient()
|
||||
const client = new NovuClient({ ky })
|
||||
|
||||
const result = await client.triggerEvent({
|
||||
name: 'send-email',
|
||||
|
|
@ -2,7 +2,7 @@ import test from 'ava'
|
|||
|
||||
import { SerpAPIClient } from '@/services/serpapi'
|
||||
|
||||
import './_utils'
|
||||
import { ky } from '../_utils'
|
||||
|
||||
test('SerpAPIClient.search', async (t) => {
|
||||
if (!process.env.SERPAPI_API_KEY) {
|
||||
|
|
@ -10,7 +10,7 @@ test('SerpAPIClient.search', async (t) => {
|
|||
}
|
||||
|
||||
t.timeout(2 * 60 * 1000)
|
||||
const client = new SerpAPIClient()
|
||||
const client = new SerpAPIClient({ ky })
|
||||
|
||||
const result = await client.search('coffee')
|
||||
// console.log(result)
|
||||
|
|
@ -2,7 +2,7 @@ import test from 'ava'
|
|||
|
||||
import { SlackClient } from '@/services/slack'
|
||||
|
||||
import './_utils'
|
||||
import '../_utils'
|
||||
|
||||
test('SlackClient.sendMessage', async (t) => {
|
||||
if (!process.env.SLACK_API_KEY || !process.env.SLACK_DEFAULT_CHANNEL) {
|
||||
|
|
@ -2,7 +2,7 @@ import test from 'ava'
|
|||
|
||||
import { TwilioConversationClient } from '@/services/twilio-conversation'
|
||||
|
||||
import './_utils'
|
||||
import '../_utils'
|
||||
|
||||
test.serial('TwilioConversationClient.createConversation', async (t) => {
|
||||
if (!process.env.TWILIO_ACCOUNT_SID || !process.env.TWILIO_AUTH_TOKEN) {
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
import test from 'ava'
|
||||
|
||||
import { normalizeUrl } from '@/url-utils'
|
||||
|
||||
test('normalizeUrl', async (t) => {
|
||||
t.is(normalizeUrl('https://www.google.com'), 'https://www.google.com')
|
||||
t.is(normalizeUrl('//www.google.com'), 'https://www.google.com')
|
||||
t.is(
|
||||
normalizeUrl('https://www.google.com/foo?'),
|
||||
'https://www.google.com/foo'
|
||||
)
|
||||
t.is(
|
||||
normalizeUrl('https://www.google.com/?foo=bar&dog=cat'),
|
||||
'https://www.google.com/?dog=cat&foo=bar'
|
||||
)
|
||||
t.is(
|
||||
normalizeUrl('https://google.com/abc/123//'),
|
||||
'https://google.com/abc/123'
|
||||
)
|
||||
})
|
||||
|
||||
test('normalizeUrl - invalid urls', async (t) => {
|
||||
t.is(normalizeUrl('/foo'), null)
|
||||
t.is(normalizeUrl('/foo/bar/baz'), null)
|
||||
t.is(normalizeUrl('://foo.com'), null)
|
||||
t.is(normalizeUrl('foo'), null)
|
||||
})
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
{
|
||||
"compilerOptions": {
|
||||
"target": "es2020",
|
||||
"lib": ["esnext", "es2022.error"],
|
||||
"lib": ["esnext", "es2022.error", "DOM"],
|
||||
"allowJs": true,
|
||||
"skipLibCheck": true,
|
||||
"strict": true,
|
||||
|
|
|
|||
Ładowanie…
Reference in New Issue