kopia lustrzana https://github.com/transitive-bullshit/chatgpt-api
feat: add diffbot; refactor ky and services; add optional ky caching for tests
rodzic
ab5321b641
commit
92ba811e93
|
@ -27,7 +27,12 @@
|
||||||
"padding-line-between-statements": [
|
"padding-line-between-statements": [
|
||||||
"error",
|
"error",
|
||||||
{ "blankLine": "always", "prev": ["block", "block-like"], "next": "*" },
|
{ "blankLine": "always", "prev": ["block", "block-like"], "next": "*" },
|
||||||
{ "blankLine": "always", "prev": "*", "next": ["block", "block-like"] }
|
{ "blankLine": "always", "prev": "*", "next": ["block", "block-like"] },
|
||||||
|
{
|
||||||
|
"blankLine": "any",
|
||||||
|
"prev": ["singleline-const", "singleline-let", "expression"],
|
||||||
|
"next": ["block", "block-like"]
|
||||||
|
}
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
"overrides": [
|
"overrides": [
|
||||||
|
|
12
package.json
12
package.json
|
@ -45,14 +45,16 @@
|
||||||
"debug": "^4.3.4",
|
"debug": "^4.3.4",
|
||||||
"expr-eval": "^2.0.2",
|
"expr-eval": "^2.0.2",
|
||||||
"handlebars": "^4.7.7",
|
"handlebars": "^4.7.7",
|
||||||
|
"is-relative-url": "^4.0.0",
|
||||||
"js-tiktoken": "^1.0.6",
|
"js-tiktoken": "^1.0.6",
|
||||||
"jsonrepair": "^3.1.0",
|
"jsonrepair": "^3.1.0",
|
||||||
"ky": "^0.33.3",
|
"ky": "^0.33.3",
|
||||||
"nanoid": "^4.0.2",
|
"nanoid": "^4.0.2",
|
||||||
|
"normalize-url": "^8.0.0",
|
||||||
"openai-fetch": "^1.5.1",
|
"openai-fetch": "^1.5.1",
|
||||||
"p-map": "^6.0.0",
|
"p-map": "^6.0.0",
|
||||||
"p-retry": "^5.1.2",
|
"p-retry": "^5.1.2",
|
||||||
"p-timeout": "^6.1.1",
|
"p-timeout": "^6.1.2",
|
||||||
"quick-lru": "^6.1.1",
|
"quick-lru": "^6.1.1",
|
||||||
"ts-dedent": "^2.2.0",
|
"ts-dedent": "^2.2.0",
|
||||||
"uuid": "^9.0.0",
|
"uuid": "^9.0.0",
|
||||||
|
@ -64,11 +66,11 @@
|
||||||
"@keyv/redis": "^2.6.1",
|
"@keyv/redis": "^2.6.1",
|
||||||
"@trivago/prettier-plugin-sort-imports": "^4.1.1",
|
"@trivago/prettier-plugin-sort-imports": "^4.1.1",
|
||||||
"@types/debug": "^4.1.8",
|
"@types/debug": "^4.1.8",
|
||||||
"@types/node": "^20.3.0",
|
"@types/node": "^20.3.1",
|
||||||
"@types/sinon": "^10.0.15",
|
"@types/sinon": "^10.0.15",
|
||||||
"@types/uuid": "^9.0.2",
|
"@types/uuid": "^9.0.2",
|
||||||
"@typescript-eslint/eslint-plugin": "^5.59.9",
|
"@typescript-eslint/eslint-plugin": "^5.59.11",
|
||||||
"@typescript-eslint/parser": "^5.59.9",
|
"@typescript-eslint/parser": "^5.59.11",
|
||||||
"ava": "^5.3.0",
|
"ava": "^5.3.0",
|
||||||
"del-cli": "^5.0.0",
|
"del-cli": "^5.0.0",
|
||||||
"dotenv": "^16.1.4",
|
"dotenv": "^16.1.4",
|
||||||
|
@ -83,7 +85,7 @@
|
||||||
"p-memoize": "^7.1.1",
|
"p-memoize": "^7.1.1",
|
||||||
"prettier": "^2.8.8",
|
"prettier": "^2.8.8",
|
||||||
"react": "^18.2.0",
|
"react": "^18.2.0",
|
||||||
"sinon": "^15.1.0",
|
"sinon": "^15.1.2",
|
||||||
"tsup": "^6.7.0",
|
"tsup": "^6.7.0",
|
||||||
"tsx": "^3.12.7",
|
"tsx": "^3.12.7",
|
||||||
"type-fest": "^3.11.1",
|
"type-fest": "^3.11.1",
|
||||||
|
|
137
pnpm-lock.yaml
137
pnpm-lock.yaml
|
@ -1,4 +1,4 @@
|
||||||
lockfileVersion: '6.1'
|
lockfileVersion: '6.0'
|
||||||
|
|
||||||
settings:
|
settings:
|
||||||
autoInstallPeers: true
|
autoInstallPeers: true
|
||||||
|
@ -26,6 +26,9 @@ dependencies:
|
||||||
handlebars:
|
handlebars:
|
||||||
specifier: ^4.7.7
|
specifier: ^4.7.7
|
||||||
version: 4.7.7
|
version: 4.7.7
|
||||||
|
is-relative-url:
|
||||||
|
specifier: ^4.0.0
|
||||||
|
version: 4.0.0
|
||||||
js-tiktoken:
|
js-tiktoken:
|
||||||
specifier: ^1.0.6
|
specifier: ^1.0.6
|
||||||
version: 1.0.6
|
version: 1.0.6
|
||||||
|
@ -38,6 +41,9 @@ dependencies:
|
||||||
nanoid:
|
nanoid:
|
||||||
specifier: ^4.0.2
|
specifier: ^4.0.2
|
||||||
version: 4.0.2
|
version: 4.0.2
|
||||||
|
normalize-url:
|
||||||
|
specifier: ^8.0.0
|
||||||
|
version: 8.0.0
|
||||||
openai-fetch:
|
openai-fetch:
|
||||||
specifier: ^1.5.1
|
specifier: ^1.5.1
|
||||||
version: 1.5.1
|
version: 1.5.1
|
||||||
|
@ -48,8 +54,8 @@ dependencies:
|
||||||
specifier: ^5.1.2
|
specifier: ^5.1.2
|
||||||
version: 5.1.2
|
version: 5.1.2
|
||||||
p-timeout:
|
p-timeout:
|
||||||
specifier: ^6.1.1
|
specifier: ^6.1.2
|
||||||
version: 6.1.1
|
version: 6.1.2
|
||||||
quick-lru:
|
quick-lru:
|
||||||
specifier: ^6.1.1
|
specifier: ^6.1.1
|
||||||
version: 6.1.1
|
version: 6.1.1
|
||||||
|
@ -80,8 +86,8 @@ devDependencies:
|
||||||
specifier: ^4.1.8
|
specifier: ^4.1.8
|
||||||
version: 4.1.8
|
version: 4.1.8
|
||||||
'@types/node':
|
'@types/node':
|
||||||
specifier: ^20.3.0
|
specifier: ^20.3.1
|
||||||
version: 20.3.0
|
version: 20.3.1
|
||||||
'@types/sinon':
|
'@types/sinon':
|
||||||
specifier: ^10.0.15
|
specifier: ^10.0.15
|
||||||
version: 10.0.15
|
version: 10.0.15
|
||||||
|
@ -89,11 +95,11 @@ devDependencies:
|
||||||
specifier: ^9.0.2
|
specifier: ^9.0.2
|
||||||
version: 9.0.2
|
version: 9.0.2
|
||||||
'@typescript-eslint/eslint-plugin':
|
'@typescript-eslint/eslint-plugin':
|
||||||
specifier: ^5.59.9
|
specifier: ^5.59.11
|
||||||
version: 5.59.9(@typescript-eslint/parser@5.59.9)(eslint@8.42.0)(typescript@5.1.3)
|
version: 5.59.11(@typescript-eslint/parser@5.59.11)(eslint@8.42.0)(typescript@5.1.3)
|
||||||
'@typescript-eslint/parser':
|
'@typescript-eslint/parser':
|
||||||
specifier: ^5.59.9
|
specifier: ^5.59.11
|
||||||
version: 5.59.9(eslint@8.42.0)(typescript@5.1.3)
|
version: 5.59.11(eslint@8.42.0)(typescript@5.1.3)
|
||||||
ava:
|
ava:
|
||||||
specifier: ^5.3.0
|
specifier: ^5.3.0
|
||||||
version: 5.3.0
|
version: 5.3.0
|
||||||
|
@ -137,8 +143,8 @@ devDependencies:
|
||||||
specifier: ^18.2.0
|
specifier: ^18.2.0
|
||||||
version: 18.2.0
|
version: 18.2.0
|
||||||
sinon:
|
sinon:
|
||||||
specifier: ^15.1.0
|
specifier: ^15.1.2
|
||||||
version: 15.1.0
|
version: 15.1.2
|
||||||
tsup:
|
tsup:
|
||||||
specifier: ^6.7.0
|
specifier: ^6.7.0
|
||||||
version: 6.7.0(typescript@5.1.3)
|
version: 6.7.0(typescript@5.1.3)
|
||||||
|
@ -692,8 +698,8 @@ packages:
|
||||||
type-detect: 4.0.8
|
type-detect: 4.0.8
|
||||||
dev: true
|
dev: true
|
||||||
|
|
||||||
/@sinonjs/fake-timers@10.2.0:
|
/@sinonjs/fake-timers@10.1.0:
|
||||||
resolution: {integrity: sha512-OPwQlEdg40HAj5KNF8WW6q2KG4Z+cBCZb3m4ninfTZKaBmbIJodviQsDBoYMPHkOyJJMHnOJo5j2+LKDOhOACg==}
|
resolution: {integrity: sha512-w1qd368vtrwttm1PRJWPW1QHlbmHrVDGs1eBH/jZvRPUFS4MNXV9Q33EQdjOdeAxZ7O8+3wM7zxztm2nfUSyKw==}
|
||||||
dependencies:
|
dependencies:
|
||||||
'@sinonjs/commons': 3.0.0
|
'@sinonjs/commons': 3.0.0
|
||||||
dev: true
|
dev: true
|
||||||
|
@ -748,8 +754,8 @@ packages:
|
||||||
resolution: {integrity: sha512-iiUgKzV9AuaEkZqkOLDIvlQiL6ltuZd9tGcW3gwpnX8JbuiuhFlEGmmFXEXkN50Cvq7Os88IY2v0dkDqXYWVgA==}
|
resolution: {integrity: sha512-iiUgKzV9AuaEkZqkOLDIvlQiL6ltuZd9tGcW3gwpnX8JbuiuhFlEGmmFXEXkN50Cvq7Os88IY2v0dkDqXYWVgA==}
|
||||||
dev: true
|
dev: true
|
||||||
|
|
||||||
/@types/node@20.3.0:
|
/@types/node@20.3.1:
|
||||||
resolution: {integrity: sha512-cumHmIAf6On83X7yP+LrsEyUOf/YlociZelmpRYaGFydoaPdxdt80MAbu6vWerQT2COCp2nPvHdsbD7tHn/YlQ==}
|
resolution: {integrity: sha512-EhcH/wvidPy1WeML3TtYFGR83UzjxeWRen9V402T8aUGYsCHOmfoisV3ZSg03gAFIbLq8TnWOJ0f4cALtnSEUg==}
|
||||||
dev: true
|
dev: true
|
||||||
|
|
||||||
/@types/normalize-package-data@2.4.1:
|
/@types/normalize-package-data@2.4.1:
|
||||||
|
@ -778,8 +784,8 @@ packages:
|
||||||
resolution: {integrity: sha512-kNnC1GFBLuhImSnV7w4njQkUiJi0ZXUycu1rUaouPqiKlXkh77JKgdRnTAp1x5eBwcIwbtI+3otwzuIDEuDoxQ==}
|
resolution: {integrity: sha512-kNnC1GFBLuhImSnV7w4njQkUiJi0ZXUycu1rUaouPqiKlXkh77JKgdRnTAp1x5eBwcIwbtI+3otwzuIDEuDoxQ==}
|
||||||
dev: true
|
dev: true
|
||||||
|
|
||||||
/@typescript-eslint/eslint-plugin@5.59.9(@typescript-eslint/parser@5.59.9)(eslint@8.42.0)(typescript@5.1.3):
|
/@typescript-eslint/eslint-plugin@5.59.11(@typescript-eslint/parser@5.59.11)(eslint@8.42.0)(typescript@5.1.3):
|
||||||
resolution: {integrity: sha512-4uQIBq1ffXd2YvF7MAvehWKW3zVv/w+mSfRAu+8cKbfj3nwzyqJLNcZJpQ/WZ1HLbJDiowwmQ6NO+63nCA+fqA==}
|
resolution: {integrity: sha512-XxuOfTkCUiOSyBWIvHlUraLw/JT/6Io1365RO6ZuI88STKMavJZPNMU0lFcUTeQXEhHiv64CbxYxBNoDVSmghg==}
|
||||||
engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0}
|
engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0}
|
||||||
peerDependencies:
|
peerDependencies:
|
||||||
'@typescript-eslint/parser': ^5.0.0
|
'@typescript-eslint/parser': ^5.0.0
|
||||||
|
@ -790,10 +796,10 @@ packages:
|
||||||
optional: true
|
optional: true
|
||||||
dependencies:
|
dependencies:
|
||||||
'@eslint-community/regexpp': 4.5.1
|
'@eslint-community/regexpp': 4.5.1
|
||||||
'@typescript-eslint/parser': 5.59.9(eslint@8.42.0)(typescript@5.1.3)
|
'@typescript-eslint/parser': 5.59.11(eslint@8.42.0)(typescript@5.1.3)
|
||||||
'@typescript-eslint/scope-manager': 5.59.9
|
'@typescript-eslint/scope-manager': 5.59.11
|
||||||
'@typescript-eslint/type-utils': 5.59.9(eslint@8.42.0)(typescript@5.1.3)
|
'@typescript-eslint/type-utils': 5.59.11(eslint@8.42.0)(typescript@5.1.3)
|
||||||
'@typescript-eslint/utils': 5.59.9(eslint@8.42.0)(typescript@5.1.3)
|
'@typescript-eslint/utils': 5.59.11(eslint@8.42.0)(typescript@5.1.3)
|
||||||
debug: 4.3.4
|
debug: 4.3.4
|
||||||
eslint: 8.42.0
|
eslint: 8.42.0
|
||||||
grapheme-splitter: 1.0.4
|
grapheme-splitter: 1.0.4
|
||||||
|
@ -806,8 +812,8 @@ packages:
|
||||||
- supports-color
|
- supports-color
|
||||||
dev: true
|
dev: true
|
||||||
|
|
||||||
/@typescript-eslint/parser@5.59.9(eslint@8.42.0)(typescript@5.1.3):
|
/@typescript-eslint/parser@5.59.11(eslint@8.42.0)(typescript@5.1.3):
|
||||||
resolution: {integrity: sha512-FsPkRvBtcLQ/eVK1ivDiNYBjn3TGJdXy2fhXX+rc7czWl4ARwnpArwbihSOHI2Peg9WbtGHrbThfBUkZZGTtvQ==}
|
resolution: {integrity: sha512-s9ZF3M+Nym6CAZEkJJeO2TFHHDsKAM3ecNkLuH4i4s8/RCPnF5JRip2GyviYkeEAcwGMJxkqG9h2dAsnA1nZpA==}
|
||||||
engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0}
|
engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0}
|
||||||
peerDependencies:
|
peerDependencies:
|
||||||
eslint: ^6.0.0 || ^7.0.0 || ^8.0.0
|
eslint: ^6.0.0 || ^7.0.0 || ^8.0.0
|
||||||
|
@ -816,9 +822,9 @@ packages:
|
||||||
typescript:
|
typescript:
|
||||||
optional: true
|
optional: true
|
||||||
dependencies:
|
dependencies:
|
||||||
'@typescript-eslint/scope-manager': 5.59.9
|
'@typescript-eslint/scope-manager': 5.59.11
|
||||||
'@typescript-eslint/types': 5.59.9
|
'@typescript-eslint/types': 5.59.11
|
||||||
'@typescript-eslint/typescript-estree': 5.59.9(typescript@5.1.3)
|
'@typescript-eslint/typescript-estree': 5.59.11(typescript@5.1.3)
|
||||||
debug: 4.3.4
|
debug: 4.3.4
|
||||||
eslint: 8.42.0
|
eslint: 8.42.0
|
||||||
typescript: 5.1.3
|
typescript: 5.1.3
|
||||||
|
@ -826,16 +832,16 @@ packages:
|
||||||
- supports-color
|
- supports-color
|
||||||
dev: true
|
dev: true
|
||||||
|
|
||||||
/@typescript-eslint/scope-manager@5.59.9:
|
/@typescript-eslint/scope-manager@5.59.11:
|
||||||
resolution: {integrity: sha512-8RA+E+w78z1+2dzvK/tGZ2cpGigBZ58VMEHDZtpE1v+LLjzrYGc8mMaTONSxKyEkz3IuXFM0IqYiGHlCsmlZxQ==}
|
resolution: {integrity: sha512-dHFOsxoLFtrIcSj5h0QoBT/89hxQONwmn3FOQ0GOQcLOOXm+MIrS8zEAhs4tWl5MraxCY3ZJpaXQQdFMc2Tu+Q==}
|
||||||
engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0}
|
engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0}
|
||||||
dependencies:
|
dependencies:
|
||||||
'@typescript-eslint/types': 5.59.9
|
'@typescript-eslint/types': 5.59.11
|
||||||
'@typescript-eslint/visitor-keys': 5.59.9
|
'@typescript-eslint/visitor-keys': 5.59.11
|
||||||
dev: true
|
dev: true
|
||||||
|
|
||||||
/@typescript-eslint/type-utils@5.59.9(eslint@8.42.0)(typescript@5.1.3):
|
/@typescript-eslint/type-utils@5.59.11(eslint@8.42.0)(typescript@5.1.3):
|
||||||
resolution: {integrity: sha512-ksEsT0/mEHg9e3qZu98AlSrONAQtrSTljL3ow9CGej8eRo7pe+yaC/mvTjptp23Xo/xIf2mLZKC6KPv4Sji26Q==}
|
resolution: {integrity: sha512-LZqVY8hMiVRF2a7/swmkStMYSoXMFlzL6sXV6U/2gL5cwnLWQgLEG8tjWPpaE4rMIdZ6VKWwcffPlo1jPfk43g==}
|
||||||
engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0}
|
engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0}
|
||||||
peerDependencies:
|
peerDependencies:
|
||||||
eslint: '*'
|
eslint: '*'
|
||||||
|
@ -844,8 +850,8 @@ packages:
|
||||||
typescript:
|
typescript:
|
||||||
optional: true
|
optional: true
|
||||||
dependencies:
|
dependencies:
|
||||||
'@typescript-eslint/typescript-estree': 5.59.9(typescript@5.1.3)
|
'@typescript-eslint/typescript-estree': 5.59.11(typescript@5.1.3)
|
||||||
'@typescript-eslint/utils': 5.59.9(eslint@8.42.0)(typescript@5.1.3)
|
'@typescript-eslint/utils': 5.59.11(eslint@8.42.0)(typescript@5.1.3)
|
||||||
debug: 4.3.4
|
debug: 4.3.4
|
||||||
eslint: 8.42.0
|
eslint: 8.42.0
|
||||||
tsutils: 3.21.0(typescript@5.1.3)
|
tsutils: 3.21.0(typescript@5.1.3)
|
||||||
|
@ -854,13 +860,13 @@ packages:
|
||||||
- supports-color
|
- supports-color
|
||||||
dev: true
|
dev: true
|
||||||
|
|
||||||
/@typescript-eslint/types@5.59.9:
|
/@typescript-eslint/types@5.59.11:
|
||||||
resolution: {integrity: sha512-uW8H5NRgTVneSVTfiCVffBb8AbwWSKg7qcA4Ot3JI3MPCJGsB4Db4BhvAODIIYE5mNj7Q+VJkK7JxmRhk2Lyjw==}
|
resolution: {integrity: sha512-epoN6R6tkvBYSc+cllrz+c2sOFWkbisJZWkOE+y3xHtvYaOE6Wk6B8e114McRJwFRjGvYdJwLXQH5c9osME/AA==}
|
||||||
engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0}
|
engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0}
|
||||||
dev: true
|
dev: true
|
||||||
|
|
||||||
/@typescript-eslint/typescript-estree@5.59.9(typescript@5.1.3):
|
/@typescript-eslint/typescript-estree@5.59.11(typescript@5.1.3):
|
||||||
resolution: {integrity: sha512-pmM0/VQ7kUhd1QyIxgS+aRvMgw+ZljB3eDb+jYyp6d2bC0mQWLzUDF+DLwCTkQ3tlNyVsvZRXjFyV0LkU/aXjA==}
|
resolution: {integrity: sha512-YupOpot5hJO0maupJXixi6l5ETdrITxeo5eBOeuV7RSKgYdU3G5cxO49/9WRnJq9EMrB7AuTSLH/bqOsXi7wPA==}
|
||||||
engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0}
|
engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0}
|
||||||
peerDependencies:
|
peerDependencies:
|
||||||
typescript: '*'
|
typescript: '*'
|
||||||
|
@ -868,8 +874,8 @@ packages:
|
||||||
typescript:
|
typescript:
|
||||||
optional: true
|
optional: true
|
||||||
dependencies:
|
dependencies:
|
||||||
'@typescript-eslint/types': 5.59.9
|
'@typescript-eslint/types': 5.59.11
|
||||||
'@typescript-eslint/visitor-keys': 5.59.9
|
'@typescript-eslint/visitor-keys': 5.59.11
|
||||||
debug: 4.3.4
|
debug: 4.3.4
|
||||||
globby: 11.1.0
|
globby: 11.1.0
|
||||||
is-glob: 4.0.3
|
is-glob: 4.0.3
|
||||||
|
@ -880,8 +886,8 @@ packages:
|
||||||
- supports-color
|
- supports-color
|
||||||
dev: true
|
dev: true
|
||||||
|
|
||||||
/@typescript-eslint/utils@5.59.9(eslint@8.42.0)(typescript@5.1.3):
|
/@typescript-eslint/utils@5.59.11(eslint@8.42.0)(typescript@5.1.3):
|
||||||
resolution: {integrity: sha512-1PuMYsju/38I5Ggblaeb98TOoUvjhRvLpLa1DoTOFaLWqaXl/1iQ1eGurTXgBY58NUdtfTXKP5xBq7q9NDaLKg==}
|
resolution: {integrity: sha512-didu2rHSOMUdJThLk4aZ1Or8IcO3HzCw/ZvEjTTIfjIrcdd5cvSIwwDy2AOlE7htSNp7QIZ10fLMyRCveesMLg==}
|
||||||
engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0}
|
engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0}
|
||||||
peerDependencies:
|
peerDependencies:
|
||||||
eslint: ^6.0.0 || ^7.0.0 || ^8.0.0
|
eslint: ^6.0.0 || ^7.0.0 || ^8.0.0
|
||||||
|
@ -889,9 +895,9 @@ packages:
|
||||||
'@eslint-community/eslint-utils': 4.4.0(eslint@8.42.0)
|
'@eslint-community/eslint-utils': 4.4.0(eslint@8.42.0)
|
||||||
'@types/json-schema': 7.0.12
|
'@types/json-schema': 7.0.12
|
||||||
'@types/semver': 7.5.0
|
'@types/semver': 7.5.0
|
||||||
'@typescript-eslint/scope-manager': 5.59.9
|
'@typescript-eslint/scope-manager': 5.59.11
|
||||||
'@typescript-eslint/types': 5.59.9
|
'@typescript-eslint/types': 5.59.11
|
||||||
'@typescript-eslint/typescript-estree': 5.59.9(typescript@5.1.3)
|
'@typescript-eslint/typescript-estree': 5.59.11(typescript@5.1.3)
|
||||||
eslint: 8.42.0
|
eslint: 8.42.0
|
||||||
eslint-scope: 5.1.1
|
eslint-scope: 5.1.1
|
||||||
semver: 7.5.1
|
semver: 7.5.1
|
||||||
|
@ -900,11 +906,11 @@ packages:
|
||||||
- typescript
|
- typescript
|
||||||
dev: true
|
dev: true
|
||||||
|
|
||||||
/@typescript-eslint/visitor-keys@5.59.9:
|
/@typescript-eslint/visitor-keys@5.59.11:
|
||||||
resolution: {integrity: sha512-bT7s0td97KMaLwpEBckbzj/YohnvXtqbe2XgqNvTl6RJVakY5mvENOTPvw5u66nljfZxthESpDozs86U+oLY8Q==}
|
resolution: {integrity: sha512-KGYniTGG3AMTuKF9QBD7EIrvufkB6O6uX3knP73xbKLMpH+QRPcgnCxjWXSHjMRuOxFLovljqQgQpR0c7GvjoA==}
|
||||||
engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0}
|
engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0}
|
||||||
dependencies:
|
dependencies:
|
||||||
'@typescript-eslint/types': 5.59.9
|
'@typescript-eslint/types': 5.59.11
|
||||||
eslint-visitor-keys: 3.4.1
|
eslint-visitor-keys: 3.4.1
|
||||||
dev: true
|
dev: true
|
||||||
|
|
||||||
|
@ -2243,6 +2249,11 @@ packages:
|
||||||
engines: {node: '>=8'}
|
engines: {node: '>=8'}
|
||||||
dev: true
|
dev: true
|
||||||
|
|
||||||
|
/is-absolute-url@4.0.1:
|
||||||
|
resolution: {integrity: sha512-/51/TKE88Lmm7Gc4/8btclNXWS+g50wXhYJq8HWIBAGUBnoAdRu1aXeh364t/O7wXDAcTJDP8PNuNKWUDWie+A==}
|
||||||
|
engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}
|
||||||
|
dev: false
|
||||||
|
|
||||||
/is-array-buffer@3.0.2:
|
/is-array-buffer@3.0.2:
|
||||||
resolution: {integrity: sha512-y+FyyR/w8vfIRq4eQcM1EYgSTnmHXPqaF+IgzgraytCFq5Xh8lllDVmAZolPJiZttZLeFSINPYMaEJ7/vWUa1w==}
|
resolution: {integrity: sha512-y+FyyR/w8vfIRq4eQcM1EYgSTnmHXPqaF+IgzgraytCFq5Xh8lllDVmAZolPJiZttZLeFSINPYMaEJ7/vWUa1w==}
|
||||||
dependencies:
|
dependencies:
|
||||||
|
@ -2383,6 +2394,13 @@ packages:
|
||||||
has-tostringtag: 1.0.0
|
has-tostringtag: 1.0.0
|
||||||
dev: true
|
dev: true
|
||||||
|
|
||||||
|
/is-relative-url@4.0.0:
|
||||||
|
resolution: {integrity: sha512-PkzoL1qKAYXNFct5IKdKRH/iBQou/oCC85QhXj6WKtUQBliZ4Yfd3Zk27RHu9KQG8r6zgvAA2AQKC9p+rqTszg==}
|
||||||
|
engines: {node: '>=14.16'}
|
||||||
|
dependencies:
|
||||||
|
is-absolute-url: 4.0.1
|
||||||
|
dev: false
|
||||||
|
|
||||||
/is-shared-array-buffer@1.0.2:
|
/is-shared-array-buffer@1.0.2:
|
||||||
resolution: {integrity: sha512-sqN2UDu1/0y6uvXyStCOzyhAjCSlHceFoMKJW8W9EU9cvic/QdsZ0kEU93HEy3IUEFZIiH/3w+AH/UQbPHNdhA==}
|
resolution: {integrity: sha512-sqN2UDu1/0y6uvXyStCOzyhAjCSlHceFoMKJW8W9EU9cvic/QdsZ0kEU93HEy3IUEFZIiH/3w+AH/UQbPHNdhA==}
|
||||||
dependencies:
|
dependencies:
|
||||||
|
@ -2833,7 +2851,7 @@ packages:
|
||||||
resolution: {integrity: sha512-8+Ib8rRJ4L0o3kfmyVCL7gzrohyDe0cMFTBa2d364yIrEGMEoetznKJx899YxjybU6bL9SQkYPSBBs1gyYs8Xg==}
|
resolution: {integrity: sha512-8+Ib8rRJ4L0o3kfmyVCL7gzrohyDe0cMFTBa2d364yIrEGMEoetznKJx899YxjybU6bL9SQkYPSBBs1gyYs8Xg==}
|
||||||
dependencies:
|
dependencies:
|
||||||
'@sinonjs/commons': 2.0.0
|
'@sinonjs/commons': 2.0.0
|
||||||
'@sinonjs/fake-timers': 10.2.0
|
'@sinonjs/fake-timers': 10.1.0
|
||||||
'@sinonjs/text-encoding': 0.7.2
|
'@sinonjs/text-encoding': 0.7.2
|
||||||
just-extend: 4.2.1
|
just-extend: 4.2.1
|
||||||
path-to-regexp: 1.8.0
|
path-to-regexp: 1.8.0
|
||||||
|
@ -2880,6 +2898,11 @@ packages:
|
||||||
engines: {node: '>=0.10.0'}
|
engines: {node: '>=0.10.0'}
|
||||||
dev: true
|
dev: true
|
||||||
|
|
||||||
|
/normalize-url@8.0.0:
|
||||||
|
resolution: {integrity: sha512-uVFpKhj5MheNBJRTiMZ9pE/7hD1QTeEvugSJW/OmLzAp78PB5O6adfMNTvmfKhXBkvCzC+rqifWcVYpGFwTjnw==}
|
||||||
|
engines: {node: '>=14.16'}
|
||||||
|
dev: false
|
||||||
|
|
||||||
/npm-run-all@4.1.5:
|
/npm-run-all@4.1.5:
|
||||||
resolution: {integrity: sha512-Oo82gJDAVcaMdi3nuoKFavkIHBRVqQ1qvMb+9LHk/cF4P6B2m8aP04hGf7oL6wZ9BuGwX1onlLhpuoofSyoQDQ==}
|
resolution: {integrity: sha512-Oo82gJDAVcaMdi3nuoKFavkIHBRVqQ1qvMb+9LHk/cF4P6B2m8aP04hGf7oL6wZ9BuGwX1onlLhpuoofSyoQDQ==}
|
||||||
engines: {node: '>= 4'}
|
engines: {node: '>= 4'}
|
||||||
|
@ -3058,8 +3081,8 @@ packages:
|
||||||
engines: {node: '>=12'}
|
engines: {node: '>=12'}
|
||||||
dev: true
|
dev: true
|
||||||
|
|
||||||
/p-timeout@6.1.1:
|
/p-timeout@6.1.2:
|
||||||
resolution: {integrity: sha512-yqz2Wi4fiFRpMmK0L2pGAU49naSUaP23fFIQL2Y6YT+qDGPoFwpvgQM/wzc6F8JoenUkIlAFa4Ql7NguXBxI7w==}
|
resolution: {integrity: sha512-UbD77BuZ9Bc9aABo74gfXhNvzC9Tx7SxtHSh1fxvx3jTLLYvmVhiQZZrJzqqU0jKbN32kb5VOKiLEQI/3bIjgQ==}
|
||||||
engines: {node: '>=14.16'}
|
engines: {node: '>=14.16'}
|
||||||
dev: false
|
dev: false
|
||||||
|
|
||||||
|
@ -3375,8 +3398,8 @@ packages:
|
||||||
glob: 7.2.3
|
glob: 7.2.3
|
||||||
dev: true
|
dev: true
|
||||||
|
|
||||||
/rollup@3.24.1:
|
/rollup@3.25.1:
|
||||||
resolution: {integrity: sha512-REHe5dx30ERBRFS0iENPHy+t6wtSEYkjrhwNsLyh3qpRaZ1+aylvMUdMBUHWUD/RjjLmLzEvY8Z9XRlpcdIkHA==}
|
resolution: {integrity: sha512-tywOR+rwIt5m2ZAWSe5AIJcTat8vGlnPFAv15ycCrw33t6iFsXZ6mzHVFh2psSjxQPmI+xgzMZZizUAukBI4aQ==}
|
||||||
engines: {node: '>=14.18.0', npm: '>=8.0.0'}
|
engines: {node: '>=14.18.0', npm: '>=8.0.0'}
|
||||||
hasBin: true
|
hasBin: true
|
||||||
optionalDependencies:
|
optionalDependencies:
|
||||||
|
@ -3477,11 +3500,11 @@ packages:
|
||||||
engines: {node: '>=14'}
|
engines: {node: '>=14'}
|
||||||
dev: true
|
dev: true
|
||||||
|
|
||||||
/sinon@15.1.0:
|
/sinon@15.1.2:
|
||||||
resolution: {integrity: sha512-cS5FgpDdE9/zx7no8bxROHymSlPLZzq0ChbbLk1DrxBfc+eTeBK3y8nIL+nu/0QeYydhhbLIr7ecHJpywjQaoQ==}
|
resolution: {integrity: sha512-uG1pU54Fis4EfYOPoEi13fmRHgZNg/u+3aReSEzHsN52Bpf+bMVfsBQS5MjouI+rTuG6UBIINlpuuO2Epr7SiA==}
|
||||||
dependencies:
|
dependencies:
|
||||||
'@sinonjs/commons': 3.0.0
|
'@sinonjs/commons': 3.0.0
|
||||||
'@sinonjs/fake-timers': 10.2.0
|
'@sinonjs/fake-timers': 10.1.0
|
||||||
'@sinonjs/samsam': 8.0.0
|
'@sinonjs/samsam': 8.0.0
|
||||||
diff: 5.1.0
|
diff: 5.1.0
|
||||||
nise: 5.1.4
|
nise: 5.1.4
|
||||||
|
@ -3842,7 +3865,7 @@ packages:
|
||||||
joycon: 3.1.1
|
joycon: 3.1.1
|
||||||
postcss-load-config: 3.1.4
|
postcss-load-config: 3.1.4
|
||||||
resolve-from: 5.0.0
|
resolve-from: 5.0.0
|
||||||
rollup: 3.24.1
|
rollup: 3.25.1
|
||||||
source-map: 0.8.0-beta.0
|
source-map: 0.8.0-beta.0
|
||||||
sucrase: 3.32.0
|
sucrase: 3.32.0
|
||||||
tree-kill: 1.2.2
|
tree-kill: 1.2.2
|
||||||
|
|
|
@ -0,0 +1,13 @@
|
||||||
|
import { ApifyClient } from 'apify-client'
|
||||||
|
import 'dotenv/config'
|
||||||
|
|
||||||
|
async function main() {
|
||||||
|
const apify = new ApifyClient({
|
||||||
|
token: process.env.APIFY_API_KEY
|
||||||
|
})
|
||||||
|
|
||||||
|
const actor = await apify.actor('apify/website-content-crawler').get()
|
||||||
|
console.log(actor)
|
||||||
|
}
|
||||||
|
|
||||||
|
main()
|
|
@ -1,6 +1,6 @@
|
||||||
import ky from 'ky'
|
import defaultKy from 'ky'
|
||||||
|
|
||||||
export const BING_BASE_URL = 'https://api.bing.microsoft.com'
|
export const BING_API_BASE_URL = 'https://api.bing.microsoft.com'
|
||||||
|
|
||||||
export interface BingWebSearchQuery {
|
export interface BingWebSearchQuery {
|
||||||
q: string
|
q: string
|
||||||
|
@ -235,22 +235,30 @@ interface FluffyContractualRule {
|
||||||
}
|
}
|
||||||
|
|
||||||
export class BingWebSearchClient {
|
export class BingWebSearchClient {
|
||||||
|
api: typeof defaultKy
|
||||||
|
|
||||||
apiKey: string
|
apiKey: string
|
||||||
baseUrl: string
|
apiBaseUrl: string
|
||||||
|
|
||||||
constructor({
|
constructor({
|
||||||
apiKey = process.env.BING_API_KEY,
|
apiKey = process.env.BING_API_KEY,
|
||||||
baseUrl = BING_BASE_URL
|
apiBaseUrl = BING_API_BASE_URL,
|
||||||
|
ky = defaultKy
|
||||||
}: {
|
}: {
|
||||||
apiKey?: string
|
apiKey?: string
|
||||||
baseUrl?: string
|
apiBaseUrl?: string
|
||||||
|
ky?: typeof defaultKy
|
||||||
} = {}) {
|
} = {}) {
|
||||||
if (!apiKey) {
|
if (!apiKey) {
|
||||||
throw new Error(`Error BingWebSearchClient missing required "apiKey"`)
|
throw new Error(`Error BingWebSearchClient missing required "apiKey"`)
|
||||||
}
|
}
|
||||||
|
|
||||||
this.apiKey = apiKey
|
this.apiKey = apiKey
|
||||||
this.baseUrl = baseUrl
|
this.apiBaseUrl = apiBaseUrl
|
||||||
|
|
||||||
|
this.api = ky.extend({
|
||||||
|
prefixUrl: this.apiBaseUrl
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
async search(queryOrOpts: string | BingWebSearchQuery) {
|
async search(queryOrOpts: string | BingWebSearchQuery) {
|
||||||
|
@ -269,9 +277,9 @@ export class BingWebSearchClient {
|
||||||
...queryOrOpts
|
...queryOrOpts
|
||||||
}
|
}
|
||||||
|
|
||||||
console.log(searchParams)
|
// console.log(searchParams)
|
||||||
return ky
|
return this.api
|
||||||
.get(`${this.baseUrl}/v7.0/search`, {
|
.get('v7.0/search', {
|
||||||
headers: {
|
headers: {
|
||||||
'Ocp-Apim-Subscription-Key': this.apiKey
|
'Ocp-Apim-Subscription-Key': this.apiKey
|
||||||
},
|
},
|
||||||
|
|
|
@ -0,0 +1,303 @@
|
||||||
|
import defaultKy from 'ky'
|
||||||
|
|
||||||
|
export const DIFFBOT_API_BASE_URL = 'https://api.diffbot.com'
|
||||||
|
export const DIFFBOT_KNOWLEDGE_GRAPH_API_BASE_URL = 'https://kg.diffbot.com'
|
||||||
|
|
||||||
|
export interface DiffbotExtractOptions {
|
||||||
|
/** Specify optional fields to be returned from any fully-extracted pages, e.g.: &fields=querystring,links. See available fields within each API's individual documentation pages.
|
||||||
|
* @see https://docs.diffbot.com/reference/extract-optional-fields
|
||||||
|
*/
|
||||||
|
fields?: string[]
|
||||||
|
|
||||||
|
/** (*Undocumented*) Pass paging=false to disable automatic concatenation of multiple-page articles. (By default, Diffbot will concatenate up to 20 pages of a single article.) */
|
||||||
|
paging?: boolean
|
||||||
|
|
||||||
|
/** Pass discussion=false to disable automatic extraction of comments or reviews from pages identified as articles or products. This will not affect pages identified as discussions. */
|
||||||
|
discussion?: boolean
|
||||||
|
|
||||||
|
/** Sets a value in milliseconds to wait for the retrieval/fetch of content from the requested URL. The default timeout for the third-party response is 30 seconds (30000). */
|
||||||
|
timeout?: number
|
||||||
|
|
||||||
|
/** Used to specify the IP address of a custom proxy that will be used to fetch the target page, instead of Diffbot's default IPs/proxies. (Ex: &proxy=168.212.226.204) */
|
||||||
|
proxy?: string
|
||||||
|
|
||||||
|
/** Used to specify the authentication parameters that will be used with the proxy specified in the &proxy parameter. (Ex: &proxyAuth=username:password) */
|
||||||
|
proxyAuth?: string
|
||||||
|
|
||||||
|
/** `none` will instruct Extract to not use proxies, even if proxies have been enabled for this particular URL globally. */
|
||||||
|
useProxy?: string
|
||||||
|
|
||||||
|
/** @see https://docs.diffbot.com/reference/extract-custom-javascript */
|
||||||
|
customJs?: string
|
||||||
|
|
||||||
|
/** @see https://docs.diffbot.com/reference/extract-custom-headers */
|
||||||
|
customHeaders?: Record<string, string>
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface DiffbotExtractAnalyzeOptions extends DiffbotExtractOptions {
|
||||||
|
/** Web page URL of the analyze to process */
|
||||||
|
url: string
|
||||||
|
|
||||||
|
/** By default the Analyze API will fully extract all pages that match an existing Automatic API -- articles, products or image pages. Set mode to a specific page-type (e.g., mode=article) to extract content only from that specific page-type. All other pages will simply return the default Analyze fields. */
|
||||||
|
mode?: string
|
||||||
|
|
||||||
|
/** Force any non-extracted pages (those with a type of "other") through a specific API. For example, to route all "other" pages through the Article API, pass &fallback=article. Pages that utilize this functionality will return a fallbackType field at the top-level of the response and a originalType field within each extracted object, both of which will indicate the fallback API used. */
|
||||||
|
fallback?: string
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface DiffbotExtractArticleOptions extends DiffbotExtractOptions {
|
||||||
|
/** Web page URL of the analyze to process */
|
||||||
|
url: string
|
||||||
|
|
||||||
|
/** Set the maximum number of automatically-generated tags to return. By default a maximum of ten tags will be returned. */
|
||||||
|
maxTags?: number
|
||||||
|
|
||||||
|
/** Set the minimum relevance score of tags to return, between 0.0 and 1.0. By default only tags with a score equal to or above 0.5 will be returned. */
|
||||||
|
tagConfidence?: number
|
||||||
|
|
||||||
|
/** Used to request the output of the Diffbot Natural Language API in the field naturalLanguage. Example: &naturalLanguage=entities,facts,categories,sentiment. */
|
||||||
|
naturalLanguage?: string[]
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface DiffbotExtractResponse {
|
||||||
|
request: DiffbotRequest
|
||||||
|
objects: DiffbotObject[]
|
||||||
|
}
|
||||||
|
|
||||||
|
export type DiffbotExtractArticleResponse = DiffbotExtractResponse
|
||||||
|
|
||||||
|
export interface DiffbotExtractAnalyzeResponse extends DiffbotExtractResponse {
|
||||||
|
type: string
|
||||||
|
title: string
|
||||||
|
humanLanguage: string
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface DiffbotObject {
|
||||||
|
date: string
|
||||||
|
sentiment: number
|
||||||
|
images: DiffbotImage[]
|
||||||
|
author: string
|
||||||
|
estimatedDate: string
|
||||||
|
publisherRegion: string
|
||||||
|
icon: string
|
||||||
|
diffbotUri: string
|
||||||
|
siteName: string
|
||||||
|
type: string
|
||||||
|
title: string
|
||||||
|
tags: DiffbotTag[]
|
||||||
|
publisherCountry: string
|
||||||
|
humanLanguage: string
|
||||||
|
authorUrl: string
|
||||||
|
pageUrl: string
|
||||||
|
html: string
|
||||||
|
text: string
|
||||||
|
categories?: DiffbotCategory[]
|
||||||
|
authors: DiffbotAuthor[]
|
||||||
|
breadcrumb?: DiffbotBreadcrumb[]
|
||||||
|
meta?: any
|
||||||
|
}
|
||||||
|
|
||||||
|
interface DiffbotAuthor {
|
||||||
|
name: string
|
||||||
|
link: string
|
||||||
|
}
|
||||||
|
|
||||||
|
interface DiffbotCategory {
|
||||||
|
score: number
|
||||||
|
name: string
|
||||||
|
id: string
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface DiffbotBreadcrumb {
|
||||||
|
link: string
|
||||||
|
name: string
|
||||||
|
}
|
||||||
|
|
||||||
|
interface DiffbotImage {
|
||||||
|
url: string
|
||||||
|
diffbotUri: string
|
||||||
|
|
||||||
|
naturalWidth: number
|
||||||
|
naturalHeight: number
|
||||||
|
width: number
|
||||||
|
height: number
|
||||||
|
|
||||||
|
isCached?: boolean
|
||||||
|
primary?: boolean
|
||||||
|
}
|
||||||
|
|
||||||
|
interface DiffbotTag {
|
||||||
|
score: number
|
||||||
|
sentiment: number
|
||||||
|
count: number
|
||||||
|
label: string
|
||||||
|
uri: string
|
||||||
|
rdfTypes: string[]
|
||||||
|
}
|
||||||
|
|
||||||
|
interface DiffbotRequest {
|
||||||
|
pageUrl: string
|
||||||
|
api: string
|
||||||
|
version: number
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface Image {
|
||||||
|
naturalHeight: number
|
||||||
|
diffbotUri: string
|
||||||
|
url: string
|
||||||
|
naturalWidth: number
|
||||||
|
primary: boolean
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface Tag {
|
||||||
|
score: number
|
||||||
|
sentiment: number
|
||||||
|
count: number
|
||||||
|
label: string
|
||||||
|
uri: string
|
||||||
|
rdfTypes: string[]
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface Request {
|
||||||
|
pageUrl: string
|
||||||
|
api: string
|
||||||
|
version: number
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface DiffbotSearchKnowledgeGraphOptions {
|
||||||
|
type?: 'query' | 'text' | 'queryTextFallback' | 'crawl'
|
||||||
|
query: string
|
||||||
|
col?: string
|
||||||
|
from?: number
|
||||||
|
size?: number
|
||||||
|
|
||||||
|
// NOTE: we only support `json`, so these options are not needed
|
||||||
|
// We can always convert from json to another format if needed.
|
||||||
|
// format?: 'json' | 'jsonl' | 'csv' | 'xls' | 'xlsx'
|
||||||
|
// exportspec?: string
|
||||||
|
// exportseparator?: string
|
||||||
|
// exportfile?: string
|
||||||
|
|
||||||
|
filter?: string
|
||||||
|
jsonmode?: 'extended' | 'id'
|
||||||
|
nonCanonicalFacts?: boolean
|
||||||
|
noDedupArticles?: boolean
|
||||||
|
cluster?: 'all' | 'best' | 'dedupe'
|
||||||
|
report?: boolean
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface DiffbotSearchKnowledgeGraphResponse {
|
||||||
|
version: number
|
||||||
|
hits: number
|
||||||
|
results: number
|
||||||
|
kgversion: string
|
||||||
|
diffbot_type: string
|
||||||
|
facet: boolean
|
||||||
|
data: DiffbotKnowledgeGraphNode[]
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface DiffbotKnowledgeGraphNode {
|
||||||
|
score: number
|
||||||
|
entity: DiffbotKnowledgeGraphEntity
|
||||||
|
entity_ctx: any
|
||||||
|
errors: string[]
|
||||||
|
callbackQuery: string
|
||||||
|
upperBound: number
|
||||||
|
lowerBound: number
|
||||||
|
count: number
|
||||||
|
value: string
|
||||||
|
uri: string
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface DiffbotKnowledgeGraphEntity {
|
||||||
|
id: string
|
||||||
|
images: DiffbotImage[]
|
||||||
|
diffbotUri: string
|
||||||
|
name: string
|
||||||
|
origins: string[]
|
||||||
|
}
|
||||||
|
|
||||||
|
export class DiffbotClient {
|
||||||
|
api: typeof defaultKy
|
||||||
|
apiKnowledgeGraph: typeof defaultKy
|
||||||
|
|
||||||
|
apiKey: string
|
||||||
|
apiBaseUrl: string
|
||||||
|
apiKnowledgeGraphBaseUrl: string
|
||||||
|
|
||||||
|
constructor({
|
||||||
|
apiKey = process.env.DIFFBOT_API_KEY,
|
||||||
|
apiBaseUrl = DIFFBOT_API_BASE_URL,
|
||||||
|
apiKnowledgeGraphBaseUrl = DIFFBOT_KNOWLEDGE_GRAPH_API_BASE_URL,
|
||||||
|
timeoutMs = 60_000,
|
||||||
|
ky = defaultKy
|
||||||
|
}: {
|
||||||
|
apiKey?: string
|
||||||
|
apiBaseUrl?: string
|
||||||
|
apiKnowledgeGraphBaseUrl?: string
|
||||||
|
timeoutMs?: number
|
||||||
|
ky?: typeof defaultKy
|
||||||
|
} = {}) {
|
||||||
|
if (!apiKey) {
|
||||||
|
throw new Error(`Error DiffbotClient missing required "apiKey"`)
|
||||||
|
}
|
||||||
|
|
||||||
|
this.apiKey = apiKey
|
||||||
|
this.apiBaseUrl = apiBaseUrl
|
||||||
|
this.apiKnowledgeGraphBaseUrl = apiKnowledgeGraphBaseUrl
|
||||||
|
|
||||||
|
this.api = ky.extend({ prefixUrl: apiBaseUrl, timeout: timeoutMs })
|
||||||
|
this.apiKnowledgeGraph = ky.extend({
|
||||||
|
prefixUrl: apiKnowledgeGraphBaseUrl,
|
||||||
|
timeout: timeoutMs
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
protected async _extract<
|
||||||
|
T extends DiffbotExtractResponse = DiffbotExtractResponse
|
||||||
|
>(endpoint: string, options: DiffbotExtractOptions): Promise<T> {
|
||||||
|
const { customJs, customHeaders, ...rest } = options
|
||||||
|
const searchParams: Record<string, any> = {
|
||||||
|
...rest,
|
||||||
|
token: this.apiKey
|
||||||
|
}
|
||||||
|
const headers = {
|
||||||
|
...Object.fromEntries(
|
||||||
|
[['X-Forward-X-Evaluate', customJs]].filter(([, value]) => value)
|
||||||
|
),
|
||||||
|
...customHeaders
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const [key, value] of Object.entries(rest)) {
|
||||||
|
if (Array.isArray(value)) {
|
||||||
|
searchParams[key] = value.join(',')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return this.api
|
||||||
|
.get(endpoint, {
|
||||||
|
searchParams,
|
||||||
|
headers
|
||||||
|
})
|
||||||
|
.json<T>()
|
||||||
|
}
|
||||||
|
|
||||||
|
async extractAnalyze(options: DiffbotExtractAnalyzeOptions) {
|
||||||
|
return this._extract<DiffbotExtractAnalyzeResponse>('v3/analyze', options)
|
||||||
|
}
|
||||||
|
|
||||||
|
async extractArticle(options: DiffbotExtractArticleOptions) {
|
||||||
|
return this._extract<DiffbotExtractArticleResponse>('v3/article', options)
|
||||||
|
}
|
||||||
|
|
||||||
|
async searchKnowledgeGraph(options: DiffbotSearchKnowledgeGraphOptions) {
|
||||||
|
return this.apiKnowledgeGraph
|
||||||
|
.get('kg/v3/dql', {
|
||||||
|
searchParams: {
|
||||||
|
...options,
|
||||||
|
token: this.apiKey
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.json<DiffbotSearchKnowledgeGraphResponse>()
|
||||||
|
}
|
||||||
|
}
|
|
@ -1,4 +1,5 @@
|
||||||
export * from './bing-web-search'
|
export * from './bing-web-search'
|
||||||
|
export * from './diffbot'
|
||||||
export * from './metaphor'
|
export * from './metaphor'
|
||||||
export * from './novu'
|
export * from './novu'
|
||||||
export * from './serpapi'
|
export * from './serpapi'
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
import ky from 'ky'
|
import defaultKy from 'ky'
|
||||||
import { z } from 'zod'
|
import { z } from 'zod'
|
||||||
|
|
||||||
export const METAPHOR_BASE_URL = 'https://api.metaphor.systems'
|
export const METAPHOR_API_BASE_URL = 'https://api.metaphor.systems'
|
||||||
|
|
||||||
// https://metaphorapi.readme.io/reference/search
|
// https://metaphorapi.readme.io/reference/search
|
||||||
export const MetaphorSearchInputSchema = z.object({
|
export const MetaphorSearchInputSchema = z.object({
|
||||||
|
@ -33,27 +33,35 @@ export const MetaphorSearchOutputSchema = z.object({
|
||||||
export type MetaphorSearchOutput = z.infer<typeof MetaphorSearchOutputSchema>
|
export type MetaphorSearchOutput = z.infer<typeof MetaphorSearchOutputSchema>
|
||||||
|
|
||||||
export class MetaphorClient {
|
export class MetaphorClient {
|
||||||
|
api: typeof defaultKy
|
||||||
|
|
||||||
apiKey: string
|
apiKey: string
|
||||||
baseUrl: string
|
apiBaseUrl: string
|
||||||
|
|
||||||
constructor({
|
constructor({
|
||||||
apiKey = process.env.METAPHOR_API_KEY,
|
apiKey = process.env.METAPHOR_API_KEY,
|
||||||
baseUrl = METAPHOR_BASE_URL
|
apiBaseUrl = METAPHOR_API_BASE_URL,
|
||||||
|
ky = defaultKy
|
||||||
}: {
|
}: {
|
||||||
apiKey?: string
|
apiKey?: string
|
||||||
baseUrl?: string
|
apiBaseUrl?: string
|
||||||
|
ky?: typeof defaultKy
|
||||||
} = {}) {
|
} = {}) {
|
||||||
if (!apiKey) {
|
if (!apiKey) {
|
||||||
throw new Error(`Error MetaphorClient missing required "apiKey"`)
|
throw new Error(`Error MetaphorClient missing required "apiKey"`)
|
||||||
}
|
}
|
||||||
|
|
||||||
this.apiKey = apiKey
|
this.apiKey = apiKey
|
||||||
this.baseUrl = baseUrl
|
this.apiBaseUrl = apiBaseUrl
|
||||||
|
|
||||||
|
this.api = ky.extend({
|
||||||
|
prefixUrl: this.apiBaseUrl
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
async search(params: MetaphorSearchInput) {
|
async search(params: MetaphorSearchInput) {
|
||||||
return ky
|
return this.api
|
||||||
.post(`${this.baseUrl}/search`, {
|
.post('search', {
|
||||||
headers: {
|
headers: {
|
||||||
'x-api-key': this.apiKey
|
'x-api-key': this.apiKey
|
||||||
},
|
},
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
import ky from 'ky'
|
import defaultKy from 'ky'
|
||||||
|
|
||||||
export const NOVU_BASE_URL = 'https://api.novu.co/v1'
|
export const NOVU_API_BASE_URL = 'https://api.novu.co/v1'
|
||||||
|
|
||||||
export type NovuSubscriber = {
|
export type NovuSubscriber = {
|
||||||
subscriberId: string
|
subscriberId: string
|
||||||
|
@ -19,22 +19,30 @@ export type NovuTriggerEventResponse = {
|
||||||
}
|
}
|
||||||
|
|
||||||
export class NovuClient {
|
export class NovuClient {
|
||||||
|
api: typeof defaultKy
|
||||||
|
|
||||||
apiKey: string
|
apiKey: string
|
||||||
baseUrl: string
|
apiBaseUrl: string
|
||||||
|
|
||||||
constructor({
|
constructor({
|
||||||
apiKey = process.env.NOVU_API_KEY,
|
apiKey = process.env.NOVU_API_KEY,
|
||||||
baseUrl = NOVU_BASE_URL
|
apiBaseUrl = NOVU_API_BASE_URL,
|
||||||
|
ky = defaultKy
|
||||||
}: {
|
}: {
|
||||||
apiKey?: string
|
apiKey?: string
|
||||||
baseUrl?: string
|
apiBaseUrl?: string
|
||||||
|
ky?: typeof defaultKy
|
||||||
} = {}) {
|
} = {}) {
|
||||||
if (!apiKey) {
|
if (!apiKey) {
|
||||||
throw new Error(`Error NovuClient missing required "apiKey"`)
|
throw new Error(`Error NovuClient missing required "apiKey"`)
|
||||||
}
|
}
|
||||||
|
|
||||||
this.apiKey = apiKey
|
this.apiKey = apiKey
|
||||||
this.baseUrl = baseUrl
|
this.apiBaseUrl = apiBaseUrl
|
||||||
|
|
||||||
|
this.api = ky.extend({
|
||||||
|
prefixUrl: this.apiBaseUrl
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
async triggerEvent({
|
async triggerEvent({
|
||||||
|
@ -46,19 +54,18 @@ export class NovuClient {
|
||||||
payload: Record<string, unknown>
|
payload: Record<string, unknown>
|
||||||
to: NovuSubscriber[]
|
to: NovuSubscriber[]
|
||||||
}) {
|
}) {
|
||||||
const url = `${this.baseUrl}/events/trigger`
|
return this.api
|
||||||
const headers = {
|
.post('events/trigger', {
|
||||||
Authorization: `ApiKey ${this.apiKey}`,
|
headers: {
|
||||||
'Content-Type': 'application/json'
|
Authorization: `ApiKey ${this.apiKey}`,
|
||||||
}
|
'Content-Type': 'application/json'
|
||||||
const response = await ky.post(url, {
|
},
|
||||||
headers,
|
json: {
|
||||||
json: {
|
name,
|
||||||
name,
|
payload,
|
||||||
payload,
|
to
|
||||||
to
|
}
|
||||||
}
|
})
|
||||||
})
|
.json<NovuTriggerEventResponse>()
|
||||||
return response.json<NovuTriggerEventResponse>()
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
import ky from 'ky'
|
import defaultKy from 'ky'
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* All types have been exported from the `serpapi` package, which we're
|
* All types have been exported from the `serpapi` package, which we're
|
||||||
|
@ -350,7 +350,8 @@ export type SerpAPISearchResponse = BaseResponse<GoogleParameters>
|
||||||
|
|
||||||
export interface SerpAPIClientOptions extends Partial<SerpAPIParams> {
|
export interface SerpAPIClientOptions extends Partial<SerpAPIParams> {
|
||||||
apiKey?: string
|
apiKey?: string
|
||||||
baseUrl?: string
|
apiBaseUrl?: string
|
||||||
|
ky?: typeof defaultKy
|
||||||
}
|
}
|
||||||
|
|
||||||
export const SERPAPI_BASE_URL = 'https://serpapi.com'
|
export const SERPAPI_BASE_URL = 'https://serpapi.com'
|
||||||
|
@ -361,13 +362,16 @@ export const SERPAPI_BASE_URL = 'https://serpapi.com'
|
||||||
* @see https://serpapi.com/search-api
|
* @see https://serpapi.com/search-api
|
||||||
*/
|
*/
|
||||||
export class SerpAPIClient {
|
export class SerpAPIClient {
|
||||||
|
api: typeof defaultKy
|
||||||
|
|
||||||
apiKey: string
|
apiKey: string
|
||||||
baseUrl: string
|
apiBaseUrl: string
|
||||||
params: Partial<SerpAPIParams>
|
params: Partial<SerpAPIParams>
|
||||||
|
|
||||||
constructor({
|
constructor({
|
||||||
apiKey = process.env.SERPAPI_API_KEY ?? process.env.SERP_API_KEY,
|
apiKey = process.env.SERPAPI_API_KEY ?? process.env.SERP_API_KEY,
|
||||||
baseUrl = SERPAPI_BASE_URL,
|
apiBaseUrl = SERPAPI_BASE_URL,
|
||||||
|
ky = defaultKy,
|
||||||
...params
|
...params
|
||||||
}: SerpAPIClientOptions = {}) {
|
}: SerpAPIClientOptions = {}) {
|
||||||
if (!apiKey) {
|
if (!apiKey) {
|
||||||
|
@ -375,8 +379,12 @@ export class SerpAPIClient {
|
||||||
}
|
}
|
||||||
|
|
||||||
this.apiKey = apiKey
|
this.apiKey = apiKey
|
||||||
this.baseUrl = baseUrl
|
this.apiBaseUrl = apiBaseUrl
|
||||||
this.params = params
|
this.params = params
|
||||||
|
|
||||||
|
this.api = ky.extend({
|
||||||
|
prefixUrl: this.apiBaseUrl
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
async search(queryOrOpts: string | { query: string }) {
|
async search(queryOrOpts: string | { query: string }) {
|
||||||
|
@ -384,8 +392,8 @@ export class SerpAPIClient {
|
||||||
typeof queryOrOpts === 'string' ? queryOrOpts : queryOrOpts.query
|
typeof queryOrOpts === 'string' ? queryOrOpts : queryOrOpts.query
|
||||||
const { timeout, ...rest } = this.params
|
const { timeout, ...rest } = this.params
|
||||||
|
|
||||||
return ky
|
return this.api
|
||||||
.get(`${this.baseUrl}/search`, {
|
.get('search', {
|
||||||
searchParams: {
|
searchParams: {
|
||||||
...rest,
|
...rest,
|
||||||
engine: 'google',
|
engine: 'google',
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
import ky from 'ky'
|
import defaultKy from 'ky'
|
||||||
|
|
||||||
import { sleep } from '@/utils'
|
import { sleep } from '@/utils'
|
||||||
|
|
||||||
|
@ -256,30 +256,33 @@ export type SlackSendAndWaitOptions = {
|
||||||
}
|
}
|
||||||
|
|
||||||
export class SlackClient {
|
export class SlackClient {
|
||||||
private api: typeof ky
|
protected api: typeof defaultKy
|
||||||
|
|
||||||
protected defaultChannel?: string
|
protected defaultChannel?: string
|
||||||
|
|
||||||
constructor({
|
constructor({
|
||||||
apiKey = process.env.SLACK_API_KEY,
|
apiKey = process.env.SLACK_API_KEY,
|
||||||
baseUrl = SLACK_API_BASE_URL,
|
baseUrl = SLACK_API_BASE_URL,
|
||||||
defaultChannel = process.env.SLACK_DEFAULT_CHANNEL
|
defaultChannel = process.env.SLACK_DEFAULT_CHANNEL,
|
||||||
|
ky = defaultKy
|
||||||
}: {
|
}: {
|
||||||
apiKey?: string
|
apiKey?: string
|
||||||
baseUrl?: string
|
baseUrl?: string
|
||||||
defaultChannel?: string
|
defaultChannel?: string
|
||||||
|
ky?: typeof defaultKy
|
||||||
} = {}) {
|
} = {}) {
|
||||||
if (!apiKey) {
|
if (!apiKey) {
|
||||||
throw new Error(`Error SlackClient missing required "apiKey"`)
|
throw new Error(`Error SlackClient missing required "apiKey"`)
|
||||||
}
|
}
|
||||||
|
|
||||||
this.api = ky.create({
|
this.defaultChannel = defaultChannel
|
||||||
|
|
||||||
|
this.api = ky.extend({
|
||||||
prefixUrl: baseUrl,
|
prefixUrl: baseUrl,
|
||||||
headers: {
|
headers: {
|
||||||
Authorization: `Bearer ${apiKey}`
|
Authorization: `Bearer ${apiKey}`
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
this.defaultChannel = defaultChannel
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -328,10 +331,11 @@ export class SlackClient {
|
||||||
* Returns a list of messages that were sent in a channel after a given timestamp both directly and in threads.
|
* Returns a list of messages that were sent in a channel after a given timestamp both directly and in threads.
|
||||||
*/
|
*/
|
||||||
private async fetchCandidates(channel: string, ts: string) {
|
private async fetchCandidates(channel: string, ts: string) {
|
||||||
let candidates: SlackMessage[] = []
|
|
||||||
const history = await this.fetchConversationHistory({ channel })
|
const history = await this.fetchConversationHistory({ channel })
|
||||||
const directReplies = await this.fetchReplies({ channel, ts })
|
const directReplies = await this.fetchReplies({ channel, ts })
|
||||||
|
|
||||||
|
let candidates: SlackMessage[] = []
|
||||||
|
|
||||||
if (directReplies.ok) {
|
if (directReplies.ok) {
|
||||||
candidates = candidates.concat(directReplies.messages)
|
candidates = candidates.concat(directReplies.messages)
|
||||||
}
|
}
|
||||||
|
@ -349,6 +353,7 @@ export class SlackClient {
|
||||||
candidates.sort((a, b) => {
|
candidates.sort((a, b) => {
|
||||||
return parseFloat(b.ts) - parseFloat(a.ts)
|
return parseFloat(b.ts) - parseFloat(a.ts)
|
||||||
})
|
})
|
||||||
|
|
||||||
return candidates
|
return candidates
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -357,7 +362,7 @@ export class SlackClient {
|
||||||
*
|
*
|
||||||
* ### Notes
|
* ### Notes
|
||||||
*
|
*
|
||||||
* - The implementation will poll for replies to the message until the timeout is reached. This is not ideal, but it is the only way to retrieve replies to a message in Slack without spinning up a local server to receive webhook events.
|
* - The implementation will poll for replies to the message until the timeout is reached. This is not ideal, but it is the only way to retrieve replies to a message in Slack without spinning up a local server to receive webhook events.
|
||||||
*/
|
*/
|
||||||
public async sendAndWaitForReply({
|
public async sendAndWaitForReply({
|
||||||
text,
|
text,
|
||||||
|
|
|
@ -1,9 +1,9 @@
|
||||||
import ky, { type KyResponse } from 'ky'
|
import defaultKy from 'ky'
|
||||||
|
|
||||||
import { DEFAULT_BOT_NAME } from '@/constants'
|
import { DEFAULT_BOT_NAME } from '@/constants'
|
||||||
import { sleep } from '@/utils'
|
import { sleep } from '@/utils'
|
||||||
|
|
||||||
export const TWILIO_CONVERSATION_BASE_URL =
|
export const TWILIO_CONVERSATION_API_BASE_URL =
|
||||||
'https://conversations.twilio.com/v1'
|
'https://conversations.twilio.com/v1'
|
||||||
|
|
||||||
export const DEFAULT_TWILIO_TIMEOUT_MS = 120_000
|
export const DEFAULT_TWILIO_TIMEOUT_MS = 120_000
|
||||||
|
@ -129,7 +129,8 @@ export type TwilioSendAndWaitOptions = {
|
||||||
* @see {@link https://www.twilio.com/docs/conversations/api}
|
* @see {@link https://www.twilio.com/docs/conversations/api}
|
||||||
*/
|
*/
|
||||||
export class TwilioConversationClient {
|
export class TwilioConversationClient {
|
||||||
api: typeof ky
|
api: typeof defaultKy
|
||||||
|
|
||||||
phoneNumber: string
|
phoneNumber: string
|
||||||
botName: string
|
botName: string
|
||||||
|
|
||||||
|
@ -137,18 +138,26 @@ export class TwilioConversationClient {
|
||||||
accountSid = process.env.TWILIO_ACCOUNT_SID,
|
accountSid = process.env.TWILIO_ACCOUNT_SID,
|
||||||
authToken = process.env.TWILIO_AUTH_TOKEN,
|
authToken = process.env.TWILIO_AUTH_TOKEN,
|
||||||
phoneNumber = process.env.TWILIO_PHONE_NUMBER,
|
phoneNumber = process.env.TWILIO_PHONE_NUMBER,
|
||||||
baseUrl = TWILIO_CONVERSATION_BASE_URL,
|
apiBaseUrl = TWILIO_CONVERSATION_API_BASE_URL,
|
||||||
botName = DEFAULT_BOT_NAME
|
botName = DEFAULT_BOT_NAME,
|
||||||
|
ky = defaultKy
|
||||||
}: {
|
}: {
|
||||||
accountSid?: string
|
accountSid?: string
|
||||||
authToken?: string
|
authToken?: string
|
||||||
phoneNumber?: string
|
phoneNumber?: string
|
||||||
baseUrl?: string
|
apiBaseUrl?: string
|
||||||
botName?: string
|
botName?: string
|
||||||
|
ky?: typeof defaultKy
|
||||||
} = {}) {
|
} = {}) {
|
||||||
if (!accountSid || !authToken) {
|
if (!accountSid) {
|
||||||
throw new Error(
|
throw new Error(
|
||||||
`Error TwilioConversationClient missing required "accountSid" and/or "authToken"`
|
`Error TwilioConversationClient missing required "accountSid"`
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!authToken) {
|
||||||
|
throw new Error(
|
||||||
|
`Error TwilioConversationClient missing required "authToken"`
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -161,8 +170,8 @@ export class TwilioConversationClient {
|
||||||
this.botName = botName
|
this.botName = botName
|
||||||
this.phoneNumber = phoneNumber
|
this.phoneNumber = phoneNumber
|
||||||
|
|
||||||
this.api = ky.create({
|
this.api = ky.extend({
|
||||||
prefixUrl: baseUrl,
|
prefixUrl: apiBaseUrl,
|
||||||
headers: {
|
headers: {
|
||||||
Authorization:
|
Authorization:
|
||||||
'Basic ' +
|
'Basic ' +
|
||||||
|
@ -175,7 +184,7 @@ export class TwilioConversationClient {
|
||||||
/**
|
/**
|
||||||
* Deletes a conversation and all its messages.
|
* Deletes a conversation and all its messages.
|
||||||
*/
|
*/
|
||||||
async deleteConversation(conversationSid: string): Promise<KyResponse> {
|
async deleteConversation(conversationSid: string) {
|
||||||
return this.api.delete(`Conversations/${conversationSid}`)
|
return this.api.delete(`Conversations/${conversationSid}`)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,44 @@
|
||||||
|
import isRelativeUrl from 'is-relative-url'
|
||||||
|
import normalizeUrlImpl from 'normalize-url'
|
||||||
|
import QuickLRU from 'quick-lru'
|
||||||
|
|
||||||
|
// const protocolAllowList = new Set(['https:', 'http:'])
|
||||||
|
const normalizedUrlCache = new QuickLRU<string, string | null>({
|
||||||
|
maxSize: 4000
|
||||||
|
})
|
||||||
|
|
||||||
|
export function normalizeUrl(url: string): string | null {
|
||||||
|
let normalizedUrl: string | null | undefined
|
||||||
|
|
||||||
|
try {
|
||||||
|
if (!url || (isRelativeUrl(url) && !url.startsWith('//'))) {
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
|
||||||
|
normalizedUrl = normalizedUrlCache.get(url)
|
||||||
|
|
||||||
|
if (normalizedUrl !== undefined) {
|
||||||
|
return normalizedUrl
|
||||||
|
}
|
||||||
|
|
||||||
|
normalizedUrl = normalizeUrlImpl(url, {
|
||||||
|
stripWWW: false,
|
||||||
|
defaultProtocol: 'https',
|
||||||
|
normalizeProtocol: true,
|
||||||
|
forceHttps: false,
|
||||||
|
stripHash: false,
|
||||||
|
stripTextFragment: true,
|
||||||
|
removeQueryParameters: [/^utm_\w+/i, 'ref'],
|
||||||
|
removeTrailingSlash: true,
|
||||||
|
removeSingleSlash: true,
|
||||||
|
removeExplicitPort: true,
|
||||||
|
sortQueryParameters: true
|
||||||
|
})
|
||||||
|
} catch (err) {
|
||||||
|
// ignore invalid urls
|
||||||
|
normalizedUrl = null
|
||||||
|
}
|
||||||
|
|
||||||
|
normalizedUrlCache.set(url, normalizedUrl!)
|
||||||
|
return normalizedUrl
|
||||||
|
}
|
|
@ -4,10 +4,12 @@ import 'dotenv/config'
|
||||||
import hashObject from 'hash-obj'
|
import hashObject from 'hash-obj'
|
||||||
import Redis from 'ioredis'
|
import Redis from 'ioredis'
|
||||||
import Keyv from 'keyv'
|
import Keyv from 'keyv'
|
||||||
|
import defaultKy from 'ky'
|
||||||
import { OpenAIClient } from 'openai-fetch'
|
import { OpenAIClient } from 'openai-fetch'
|
||||||
import pMemoize from 'p-memoize'
|
import pMemoize from 'p-memoize'
|
||||||
|
|
||||||
import { Agentic } from '@/agentic'
|
import { Agentic } from '@/agentic'
|
||||||
|
import { normalizeUrl } from '@/url-utils'
|
||||||
|
|
||||||
export const fakeOpenAIAPIKey = 'fake-openai-api-key'
|
export const fakeOpenAIAPIKey = 'fake-openai-api-key'
|
||||||
export const fakeAnthropicAPIKey = 'fake-anthropic-api-key'
|
export const fakeAnthropicAPIKey = 'fake-anthropic-api-key'
|
||||||
|
@ -39,6 +41,97 @@ keyv.has = async (key, ...rest) => {
|
||||||
return res
|
return res
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function getCacheKeyForRequest(request: Request): string | null {
|
||||||
|
const method = request.method.toLowerCase()
|
||||||
|
|
||||||
|
if (method === 'get') {
|
||||||
|
const url = normalizeUrl(request.url)
|
||||||
|
|
||||||
|
if (url) {
|
||||||
|
const cacheParams = {
|
||||||
|
// TODO: request.headers isn't a normal JS object...
|
||||||
|
headers: { ...request.headers }
|
||||||
|
}
|
||||||
|
|
||||||
|
// console.log('getCacheKeyForRequest', { url, cacheParams })
|
||||||
|
const cacheKey = getCacheKey(`http:${method} ${url}`, cacheParams)
|
||||||
|
return cacheKey
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Custom `ky` instance that caches GET JSON requests.
|
||||||
|
*
|
||||||
|
* TODO:
|
||||||
|
* - support non-GET requests
|
||||||
|
* - support non-JSON responses
|
||||||
|
*/
|
||||||
|
export const ky = defaultKy.extend({
|
||||||
|
hooks: {
|
||||||
|
beforeRequest: [
|
||||||
|
async (request) => {
|
||||||
|
try {
|
||||||
|
// console.log(`beforeRequest ${request.method} ${request.url}`)
|
||||||
|
|
||||||
|
const cacheKey = getCacheKeyForRequest(request)
|
||||||
|
// console.log({ cacheKey })
|
||||||
|
if (!cacheKey) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!(await keyv.has(cacheKey))) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
const cachedResponse = await keyv.get(cacheKey)
|
||||||
|
// console.log({ cachedResponse })
|
||||||
|
|
||||||
|
if (!cachedResponse) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
return new Response(JSON.stringify(cachedResponse), {
|
||||||
|
status: 200,
|
||||||
|
headers: { 'Content-Type': 'application/json' }
|
||||||
|
})
|
||||||
|
} catch (err) {
|
||||||
|
console.error('ky beforeResponse cache error', err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
|
||||||
|
afterResponse: [
|
||||||
|
async (request, _options, response) => {
|
||||||
|
try {
|
||||||
|
// console.log(
|
||||||
|
// `afterRequest ${request.method} ${request.url} ⇒ ${response.status}`
|
||||||
|
// )
|
||||||
|
|
||||||
|
if (response.status < 200 || response.status >= 300) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
const cacheKey = getCacheKeyForRequest(request)
|
||||||
|
// console.log({ cacheKey })
|
||||||
|
if (!cacheKey) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
const responseBody = await response.json()
|
||||||
|
// console.log({ responseBody })
|
||||||
|
|
||||||
|
await keyv.set(cacheKey, responseBody)
|
||||||
|
} catch (err) {
|
||||||
|
console.error('ky afterResponse cache error', err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
export class OpenAITestClient extends OpenAIClient {
|
export class OpenAITestClient extends OpenAIClient {
|
||||||
createChatCompletion = pMemoize(super.createChatCompletion, {
|
createChatCompletion = pMemoize(super.createChatCompletion, {
|
||||||
cacheKey: (params) => getCacheKey('openai:chat', params),
|
cacheKey: (params) => getCacheKey('openai:chat', params),
|
||||||
|
|
|
@ -2,7 +2,7 @@ import test from 'ava'
|
||||||
|
|
||||||
import { BingWebSearchClient } from '@/services'
|
import { BingWebSearchClient } from '@/services'
|
||||||
|
|
||||||
import './_utils'
|
import { ky } from '../_utils'
|
||||||
|
|
||||||
test('BingWebSearchClient.search', async (t) => {
|
test('BingWebSearchClient.search', async (t) => {
|
||||||
if (!process.env.BING_API_KEY) {
|
if (!process.env.BING_API_KEY) {
|
||||||
|
@ -10,9 +10,9 @@ test('BingWebSearchClient.search', async (t) => {
|
||||||
}
|
}
|
||||||
|
|
||||||
t.timeout(2 * 60 * 1000)
|
t.timeout(2 * 60 * 1000)
|
||||||
const client = new BingWebSearchClient()
|
const client = new BingWebSearchClient({ ky })
|
||||||
|
|
||||||
const result = await client.search('coffee')
|
const result = await client.search('coffee')
|
||||||
// console.log(result)
|
// console.log(result)
|
||||||
t.truthy(result?.webPages)
|
t.true(result?.webPages?.value?.length > 0)
|
||||||
})
|
})
|
|
@ -0,0 +1,37 @@
|
||||||
|
import test from 'ava'
|
||||||
|
|
||||||
|
import { DiffbotClient } from '@/services'
|
||||||
|
|
||||||
|
import { isCI, ky } from '../_utils'
|
||||||
|
|
||||||
|
test('Diffbot.analyze', async (t) => {
|
||||||
|
if (!process.env.DIFFBOT_API_KEY || isCI) {
|
||||||
|
return t.pass()
|
||||||
|
}
|
||||||
|
|
||||||
|
t.timeout(2 * 60 * 1000)
|
||||||
|
const client = new DiffbotClient({ ky })
|
||||||
|
|
||||||
|
const result = await client.extractAnalyze({
|
||||||
|
url: 'https://transitivebullsh.it'
|
||||||
|
})
|
||||||
|
// console.log(result)
|
||||||
|
t.is(result.type, 'list')
|
||||||
|
t.is(result.objects?.length, 1)
|
||||||
|
})
|
||||||
|
|
||||||
|
test('Diffbot.article', async (t) => {
|
||||||
|
if (!process.env.DIFFBOT_API_KEY || isCI) {
|
||||||
|
return t.pass()
|
||||||
|
}
|
||||||
|
|
||||||
|
t.timeout(2 * 60 * 1000)
|
||||||
|
const client = new DiffbotClient({ ky })
|
||||||
|
|
||||||
|
const result = await client.extractArticle({
|
||||||
|
url: 'https://www.nytimes.com/2023/05/31/magazine/ai-start-up-accelerator-san-francisco.html'
|
||||||
|
// fields: ['meta']
|
||||||
|
})
|
||||||
|
// console.log(JSON.stringify(result, null, 2))
|
||||||
|
t.is(result.objects[0].type, 'article')
|
||||||
|
})
|
|
@ -2,7 +2,7 @@ import test from 'ava'
|
||||||
|
|
||||||
import { NovuClient } from '@/services/novu'
|
import { NovuClient } from '@/services/novu'
|
||||||
|
|
||||||
import './_utils'
|
import { ky } from '../_utils'
|
||||||
|
|
||||||
test('NovuClient.triggerEvent', async (t) => {
|
test('NovuClient.triggerEvent', async (t) => {
|
||||||
if (!process.env.NOVU_API_KEY) {
|
if (!process.env.NOVU_API_KEY) {
|
||||||
|
@ -10,7 +10,7 @@ test('NovuClient.triggerEvent', async (t) => {
|
||||||
}
|
}
|
||||||
|
|
||||||
t.timeout(2 * 60 * 1000)
|
t.timeout(2 * 60 * 1000)
|
||||||
const client = new NovuClient()
|
const client = new NovuClient({ ky })
|
||||||
|
|
||||||
const result = await client.triggerEvent({
|
const result = await client.triggerEvent({
|
||||||
name: 'send-email',
|
name: 'send-email',
|
|
@ -2,7 +2,7 @@ import test from 'ava'
|
||||||
|
|
||||||
import { SerpAPIClient } from '@/services/serpapi'
|
import { SerpAPIClient } from '@/services/serpapi'
|
||||||
|
|
||||||
import './_utils'
|
import { ky } from '../_utils'
|
||||||
|
|
||||||
test('SerpAPIClient.search', async (t) => {
|
test('SerpAPIClient.search', async (t) => {
|
||||||
if (!process.env.SERPAPI_API_KEY) {
|
if (!process.env.SERPAPI_API_KEY) {
|
||||||
|
@ -10,7 +10,7 @@ test('SerpAPIClient.search', async (t) => {
|
||||||
}
|
}
|
||||||
|
|
||||||
t.timeout(2 * 60 * 1000)
|
t.timeout(2 * 60 * 1000)
|
||||||
const client = new SerpAPIClient()
|
const client = new SerpAPIClient({ ky })
|
||||||
|
|
||||||
const result = await client.search('coffee')
|
const result = await client.search('coffee')
|
||||||
// console.log(result)
|
// console.log(result)
|
|
@ -2,7 +2,7 @@ import test from 'ava'
|
||||||
|
|
||||||
import { SlackClient } from '@/services/slack'
|
import { SlackClient } from '@/services/slack'
|
||||||
|
|
||||||
import './_utils'
|
import '../_utils'
|
||||||
|
|
||||||
test('SlackClient.sendMessage', async (t) => {
|
test('SlackClient.sendMessage', async (t) => {
|
||||||
if (!process.env.SLACK_API_KEY || !process.env.SLACK_DEFAULT_CHANNEL) {
|
if (!process.env.SLACK_API_KEY || !process.env.SLACK_DEFAULT_CHANNEL) {
|
|
@ -2,7 +2,7 @@ import test from 'ava'
|
||||||
|
|
||||||
import { TwilioConversationClient } from '@/services/twilio-conversation'
|
import { TwilioConversationClient } from '@/services/twilio-conversation'
|
||||||
|
|
||||||
import './_utils'
|
import '../_utils'
|
||||||
|
|
||||||
test.serial('TwilioConversationClient.createConversation', async (t) => {
|
test.serial('TwilioConversationClient.createConversation', async (t) => {
|
||||||
if (!process.env.TWILIO_ACCOUNT_SID || !process.env.TWILIO_AUTH_TOKEN) {
|
if (!process.env.TWILIO_ACCOUNT_SID || !process.env.TWILIO_AUTH_TOKEN) {
|
|
@ -0,0 +1,27 @@
|
||||||
|
import test from 'ava'
|
||||||
|
|
||||||
|
import { normalizeUrl } from '@/url-utils'
|
||||||
|
|
||||||
|
test('normalizeUrl', async (t) => {
|
||||||
|
t.is(normalizeUrl('https://www.google.com'), 'https://www.google.com')
|
||||||
|
t.is(normalizeUrl('//www.google.com'), 'https://www.google.com')
|
||||||
|
t.is(
|
||||||
|
normalizeUrl('https://www.google.com/foo?'),
|
||||||
|
'https://www.google.com/foo'
|
||||||
|
)
|
||||||
|
t.is(
|
||||||
|
normalizeUrl('https://www.google.com/?foo=bar&dog=cat'),
|
||||||
|
'https://www.google.com/?dog=cat&foo=bar'
|
||||||
|
)
|
||||||
|
t.is(
|
||||||
|
normalizeUrl('https://google.com/abc/123//'),
|
||||||
|
'https://google.com/abc/123'
|
||||||
|
)
|
||||||
|
})
|
||||||
|
|
||||||
|
test('normalizeUrl - invalid urls', async (t) => {
|
||||||
|
t.is(normalizeUrl('/foo'), null)
|
||||||
|
t.is(normalizeUrl('/foo/bar/baz'), null)
|
||||||
|
t.is(normalizeUrl('://foo.com'), null)
|
||||||
|
t.is(normalizeUrl('foo'), null)
|
||||||
|
})
|
|
@ -1,7 +1,7 @@
|
||||||
{
|
{
|
||||||
"compilerOptions": {
|
"compilerOptions": {
|
||||||
"target": "es2020",
|
"target": "es2020",
|
||||||
"lib": ["esnext", "es2022.error"],
|
"lib": ["esnext", "es2022.error", "DOM"],
|
||||||
"allowJs": true,
|
"allowJs": true,
|
||||||
"skipLibCheck": true,
|
"skipLibCheck": true,
|
||||||
"strict": true,
|
"strict": true,
|
||||||
|
|
Ładowanie…
Reference in New Issue