From 92ba811e937f1f15a0925ecf5ebb002315b76229 Mon Sep 17 00:00:00 2001 From: Travis Fischer Date: Tue, 13 Jun 2023 01:09:26 -0700 Subject: [PATCH] feat: add diffbot; refactor ky and services; add optional ky caching for tests --- .eslintrc | 7 +- package.json | 12 +- pnpm-lock.yaml | 137 ++++---- scratch/apify.ts | 13 + src/services/bing-web-search.ts | 26 +- src/services/diffbot.ts | 303 ++++++++++++++++++ src/services/index.ts | 1 + src/services/metaphor.ts | 24 +- src/services/novu.ts | 47 +-- src/services/serpapi.ts | 22 +- src/services/slack.ts | 19 +- src/services/twilio-conversation.ts | 31 +- src/url-utils.ts | 44 +++ test/_utils.ts | 93 ++++++ test/{ => services}/bing.test.ts | 6 +- test/services/diffbot.test.ts | 37 +++ test/{ => services}/novu.test.ts | 4 +- test/{ => services}/serpapi.test.ts | 4 +- test/{ => services}/slack.test.ts | 2 +- .../twilio-conversation.test.ts | 2 +- test/url-utils.test.ts | 27 ++ tsconfig.json | 2 +- 22 files changed, 728 insertions(+), 135 deletions(-) create mode 100644 scratch/apify.ts create mode 100644 src/services/diffbot.ts create mode 100644 src/url-utils.ts rename test/{ => services}/bing.test.ts (69%) create mode 100644 test/services/diffbot.test.ts rename test/{ => services}/novu.test.ts (86%) rename test/{ => services}/serpapi.test.ts (80%) rename test/{ => services}/slack.test.ts (98%) rename test/{ => services}/twilio-conversation.test.ts (99%) create mode 100644 test/url-utils.test.ts diff --git a/.eslintrc b/.eslintrc index 5ff729c8..3da80307 100644 --- a/.eslintrc +++ b/.eslintrc @@ -27,7 +27,12 @@ "padding-line-between-statements": [ "error", { "blankLine": "always", "prev": ["block", "block-like"], "next": "*" }, - { "blankLine": "always", "prev": "*", "next": ["block", "block-like"] } + { "blankLine": "always", "prev": "*", "next": ["block", "block-like"] }, + { + "blankLine": "any", + "prev": ["singleline-const", "singleline-let", "expression"], + "next": ["block", "block-like"] + } ] }, "overrides": [ diff --git a/package.json b/package.json index 8bda68f6..6bb69f27 100644 --- a/package.json +++ b/package.json @@ -45,14 +45,16 @@ "debug": "^4.3.4", "expr-eval": "^2.0.2", "handlebars": "^4.7.7", + "is-relative-url": "^4.0.0", "js-tiktoken": "^1.0.6", "jsonrepair": "^3.1.0", "ky": "^0.33.3", "nanoid": "^4.0.2", + "normalize-url": "^8.0.0", "openai-fetch": "^1.5.1", "p-map": "^6.0.0", "p-retry": "^5.1.2", - "p-timeout": "^6.1.1", + "p-timeout": "^6.1.2", "quick-lru": "^6.1.1", "ts-dedent": "^2.2.0", "uuid": "^9.0.0", @@ -64,11 +66,11 @@ "@keyv/redis": "^2.6.1", "@trivago/prettier-plugin-sort-imports": "^4.1.1", "@types/debug": "^4.1.8", - "@types/node": "^20.3.0", + "@types/node": "^20.3.1", "@types/sinon": "^10.0.15", "@types/uuid": "^9.0.2", - "@typescript-eslint/eslint-plugin": "^5.59.9", - "@typescript-eslint/parser": "^5.59.9", + "@typescript-eslint/eslint-plugin": "^5.59.11", + "@typescript-eslint/parser": "^5.59.11", "ava": "^5.3.0", "del-cli": "^5.0.0", "dotenv": "^16.1.4", @@ -83,7 +85,7 @@ "p-memoize": "^7.1.1", "prettier": "^2.8.8", "react": "^18.2.0", - "sinon": "^15.1.0", + "sinon": "^15.1.2", "tsup": "^6.7.0", "tsx": "^3.12.7", "type-fest": "^3.11.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2d8dd621..f657ff4e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1,4 +1,4 @@ -lockfileVersion: '6.1' +lockfileVersion: '6.0' settings: autoInstallPeers: true @@ -26,6 +26,9 @@ dependencies: handlebars: specifier: ^4.7.7 version: 4.7.7 + is-relative-url: + specifier: ^4.0.0 + version: 4.0.0 js-tiktoken: specifier: ^1.0.6 version: 1.0.6 @@ -38,6 +41,9 @@ dependencies: nanoid: specifier: ^4.0.2 version: 4.0.2 + normalize-url: + specifier: ^8.0.0 + version: 8.0.0 openai-fetch: specifier: ^1.5.1 version: 1.5.1 @@ -48,8 +54,8 @@ dependencies: specifier: ^5.1.2 version: 5.1.2 p-timeout: - specifier: ^6.1.1 - version: 6.1.1 + specifier: ^6.1.2 + version: 6.1.2 quick-lru: specifier: ^6.1.1 version: 6.1.1 @@ -80,8 +86,8 @@ devDependencies: specifier: ^4.1.8 version: 4.1.8 '@types/node': - specifier: ^20.3.0 - version: 20.3.0 + specifier: ^20.3.1 + version: 20.3.1 '@types/sinon': specifier: ^10.0.15 version: 10.0.15 @@ -89,11 +95,11 @@ devDependencies: specifier: ^9.0.2 version: 9.0.2 '@typescript-eslint/eslint-plugin': - specifier: ^5.59.9 - version: 5.59.9(@typescript-eslint/parser@5.59.9)(eslint@8.42.0)(typescript@5.1.3) + specifier: ^5.59.11 + version: 5.59.11(@typescript-eslint/parser@5.59.11)(eslint@8.42.0)(typescript@5.1.3) '@typescript-eslint/parser': - specifier: ^5.59.9 - version: 5.59.9(eslint@8.42.0)(typescript@5.1.3) + specifier: ^5.59.11 + version: 5.59.11(eslint@8.42.0)(typescript@5.1.3) ava: specifier: ^5.3.0 version: 5.3.0 @@ -137,8 +143,8 @@ devDependencies: specifier: ^18.2.0 version: 18.2.0 sinon: - specifier: ^15.1.0 - version: 15.1.0 + specifier: ^15.1.2 + version: 15.1.2 tsup: specifier: ^6.7.0 version: 6.7.0(typescript@5.1.3) @@ -692,8 +698,8 @@ packages: type-detect: 4.0.8 dev: true - /@sinonjs/fake-timers@10.2.0: - resolution: {integrity: sha512-OPwQlEdg40HAj5KNF8WW6q2KG4Z+cBCZb3m4ninfTZKaBmbIJodviQsDBoYMPHkOyJJMHnOJo5j2+LKDOhOACg==} + /@sinonjs/fake-timers@10.1.0: + resolution: {integrity: sha512-w1qd368vtrwttm1PRJWPW1QHlbmHrVDGs1eBH/jZvRPUFS4MNXV9Q33EQdjOdeAxZ7O8+3wM7zxztm2nfUSyKw==} dependencies: '@sinonjs/commons': 3.0.0 dev: true @@ -748,8 +754,8 @@ packages: resolution: {integrity: sha512-iiUgKzV9AuaEkZqkOLDIvlQiL6ltuZd9tGcW3gwpnX8JbuiuhFlEGmmFXEXkN50Cvq7Os88IY2v0dkDqXYWVgA==} dev: true - /@types/node@20.3.0: - resolution: {integrity: sha512-cumHmIAf6On83X7yP+LrsEyUOf/YlociZelmpRYaGFydoaPdxdt80MAbu6vWerQT2COCp2nPvHdsbD7tHn/YlQ==} + /@types/node@20.3.1: + resolution: {integrity: sha512-EhcH/wvidPy1WeML3TtYFGR83UzjxeWRen9V402T8aUGYsCHOmfoisV3ZSg03gAFIbLq8TnWOJ0f4cALtnSEUg==} dev: true /@types/normalize-package-data@2.4.1: @@ -778,8 +784,8 @@ packages: resolution: {integrity: sha512-kNnC1GFBLuhImSnV7w4njQkUiJi0ZXUycu1rUaouPqiKlXkh77JKgdRnTAp1x5eBwcIwbtI+3otwzuIDEuDoxQ==} dev: true - /@typescript-eslint/eslint-plugin@5.59.9(@typescript-eslint/parser@5.59.9)(eslint@8.42.0)(typescript@5.1.3): - resolution: {integrity: sha512-4uQIBq1ffXd2YvF7MAvehWKW3zVv/w+mSfRAu+8cKbfj3nwzyqJLNcZJpQ/WZ1HLbJDiowwmQ6NO+63nCA+fqA==} + /@typescript-eslint/eslint-plugin@5.59.11(@typescript-eslint/parser@5.59.11)(eslint@8.42.0)(typescript@5.1.3): + resolution: {integrity: sha512-XxuOfTkCUiOSyBWIvHlUraLw/JT/6Io1365RO6ZuI88STKMavJZPNMU0lFcUTeQXEhHiv64CbxYxBNoDVSmghg==} engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} peerDependencies: '@typescript-eslint/parser': ^5.0.0 @@ -790,10 +796,10 @@ packages: optional: true dependencies: '@eslint-community/regexpp': 4.5.1 - '@typescript-eslint/parser': 5.59.9(eslint@8.42.0)(typescript@5.1.3) - '@typescript-eslint/scope-manager': 5.59.9 - '@typescript-eslint/type-utils': 5.59.9(eslint@8.42.0)(typescript@5.1.3) - '@typescript-eslint/utils': 5.59.9(eslint@8.42.0)(typescript@5.1.3) + '@typescript-eslint/parser': 5.59.11(eslint@8.42.0)(typescript@5.1.3) + '@typescript-eslint/scope-manager': 5.59.11 + '@typescript-eslint/type-utils': 5.59.11(eslint@8.42.0)(typescript@5.1.3) + '@typescript-eslint/utils': 5.59.11(eslint@8.42.0)(typescript@5.1.3) debug: 4.3.4 eslint: 8.42.0 grapheme-splitter: 1.0.4 @@ -806,8 +812,8 @@ packages: - supports-color dev: true - /@typescript-eslint/parser@5.59.9(eslint@8.42.0)(typescript@5.1.3): - resolution: {integrity: sha512-FsPkRvBtcLQ/eVK1ivDiNYBjn3TGJdXy2fhXX+rc7czWl4ARwnpArwbihSOHI2Peg9WbtGHrbThfBUkZZGTtvQ==} + /@typescript-eslint/parser@5.59.11(eslint@8.42.0)(typescript@5.1.3): + resolution: {integrity: sha512-s9ZF3M+Nym6CAZEkJJeO2TFHHDsKAM3ecNkLuH4i4s8/RCPnF5JRip2GyviYkeEAcwGMJxkqG9h2dAsnA1nZpA==} engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} peerDependencies: eslint: ^6.0.0 || ^7.0.0 || ^8.0.0 @@ -816,9 +822,9 @@ packages: typescript: optional: true dependencies: - '@typescript-eslint/scope-manager': 5.59.9 - '@typescript-eslint/types': 5.59.9 - '@typescript-eslint/typescript-estree': 5.59.9(typescript@5.1.3) + '@typescript-eslint/scope-manager': 5.59.11 + '@typescript-eslint/types': 5.59.11 + '@typescript-eslint/typescript-estree': 5.59.11(typescript@5.1.3) debug: 4.3.4 eslint: 8.42.0 typescript: 5.1.3 @@ -826,16 +832,16 @@ packages: - supports-color dev: true - /@typescript-eslint/scope-manager@5.59.9: - resolution: {integrity: sha512-8RA+E+w78z1+2dzvK/tGZ2cpGigBZ58VMEHDZtpE1v+LLjzrYGc8mMaTONSxKyEkz3IuXFM0IqYiGHlCsmlZxQ==} + /@typescript-eslint/scope-manager@5.59.11: + resolution: {integrity: sha512-dHFOsxoLFtrIcSj5h0QoBT/89hxQONwmn3FOQ0GOQcLOOXm+MIrS8zEAhs4tWl5MraxCY3ZJpaXQQdFMc2Tu+Q==} engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} dependencies: - '@typescript-eslint/types': 5.59.9 - '@typescript-eslint/visitor-keys': 5.59.9 + '@typescript-eslint/types': 5.59.11 + '@typescript-eslint/visitor-keys': 5.59.11 dev: true - /@typescript-eslint/type-utils@5.59.9(eslint@8.42.0)(typescript@5.1.3): - resolution: {integrity: sha512-ksEsT0/mEHg9e3qZu98AlSrONAQtrSTljL3ow9CGej8eRo7pe+yaC/mvTjptp23Xo/xIf2mLZKC6KPv4Sji26Q==} + /@typescript-eslint/type-utils@5.59.11(eslint@8.42.0)(typescript@5.1.3): + resolution: {integrity: sha512-LZqVY8hMiVRF2a7/swmkStMYSoXMFlzL6sXV6U/2gL5cwnLWQgLEG8tjWPpaE4rMIdZ6VKWwcffPlo1jPfk43g==} engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} peerDependencies: eslint: '*' @@ -844,8 +850,8 @@ packages: typescript: optional: true dependencies: - '@typescript-eslint/typescript-estree': 5.59.9(typescript@5.1.3) - '@typescript-eslint/utils': 5.59.9(eslint@8.42.0)(typescript@5.1.3) + '@typescript-eslint/typescript-estree': 5.59.11(typescript@5.1.3) + '@typescript-eslint/utils': 5.59.11(eslint@8.42.0)(typescript@5.1.3) debug: 4.3.4 eslint: 8.42.0 tsutils: 3.21.0(typescript@5.1.3) @@ -854,13 +860,13 @@ packages: - supports-color dev: true - /@typescript-eslint/types@5.59.9: - resolution: {integrity: sha512-uW8H5NRgTVneSVTfiCVffBb8AbwWSKg7qcA4Ot3JI3MPCJGsB4Db4BhvAODIIYE5mNj7Q+VJkK7JxmRhk2Lyjw==} + /@typescript-eslint/types@5.59.11: + resolution: {integrity: sha512-epoN6R6tkvBYSc+cllrz+c2sOFWkbisJZWkOE+y3xHtvYaOE6Wk6B8e114McRJwFRjGvYdJwLXQH5c9osME/AA==} engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} dev: true - /@typescript-eslint/typescript-estree@5.59.9(typescript@5.1.3): - resolution: {integrity: sha512-pmM0/VQ7kUhd1QyIxgS+aRvMgw+ZljB3eDb+jYyp6d2bC0mQWLzUDF+DLwCTkQ3tlNyVsvZRXjFyV0LkU/aXjA==} + /@typescript-eslint/typescript-estree@5.59.11(typescript@5.1.3): + resolution: {integrity: sha512-YupOpot5hJO0maupJXixi6l5ETdrITxeo5eBOeuV7RSKgYdU3G5cxO49/9WRnJq9EMrB7AuTSLH/bqOsXi7wPA==} engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} peerDependencies: typescript: '*' @@ -868,8 +874,8 @@ packages: typescript: optional: true dependencies: - '@typescript-eslint/types': 5.59.9 - '@typescript-eslint/visitor-keys': 5.59.9 + '@typescript-eslint/types': 5.59.11 + '@typescript-eslint/visitor-keys': 5.59.11 debug: 4.3.4 globby: 11.1.0 is-glob: 4.0.3 @@ -880,8 +886,8 @@ packages: - supports-color dev: true - /@typescript-eslint/utils@5.59.9(eslint@8.42.0)(typescript@5.1.3): - resolution: {integrity: sha512-1PuMYsju/38I5Ggblaeb98TOoUvjhRvLpLa1DoTOFaLWqaXl/1iQ1eGurTXgBY58NUdtfTXKP5xBq7q9NDaLKg==} + /@typescript-eslint/utils@5.59.11(eslint@8.42.0)(typescript@5.1.3): + resolution: {integrity: sha512-didu2rHSOMUdJThLk4aZ1Or8IcO3HzCw/ZvEjTTIfjIrcdd5cvSIwwDy2AOlE7htSNp7QIZ10fLMyRCveesMLg==} engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} peerDependencies: eslint: ^6.0.0 || ^7.0.0 || ^8.0.0 @@ -889,9 +895,9 @@ packages: '@eslint-community/eslint-utils': 4.4.0(eslint@8.42.0) '@types/json-schema': 7.0.12 '@types/semver': 7.5.0 - '@typescript-eslint/scope-manager': 5.59.9 - '@typescript-eslint/types': 5.59.9 - '@typescript-eslint/typescript-estree': 5.59.9(typescript@5.1.3) + '@typescript-eslint/scope-manager': 5.59.11 + '@typescript-eslint/types': 5.59.11 + '@typescript-eslint/typescript-estree': 5.59.11(typescript@5.1.3) eslint: 8.42.0 eslint-scope: 5.1.1 semver: 7.5.1 @@ -900,11 +906,11 @@ packages: - typescript dev: true - /@typescript-eslint/visitor-keys@5.59.9: - resolution: {integrity: sha512-bT7s0td97KMaLwpEBckbzj/YohnvXtqbe2XgqNvTl6RJVakY5mvENOTPvw5u66nljfZxthESpDozs86U+oLY8Q==} + /@typescript-eslint/visitor-keys@5.59.11: + resolution: {integrity: sha512-KGYniTGG3AMTuKF9QBD7EIrvufkB6O6uX3knP73xbKLMpH+QRPcgnCxjWXSHjMRuOxFLovljqQgQpR0c7GvjoA==} engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} dependencies: - '@typescript-eslint/types': 5.59.9 + '@typescript-eslint/types': 5.59.11 eslint-visitor-keys: 3.4.1 dev: true @@ -2243,6 +2249,11 @@ packages: engines: {node: '>=8'} dev: true + /is-absolute-url@4.0.1: + resolution: {integrity: sha512-/51/TKE88Lmm7Gc4/8btclNXWS+g50wXhYJq8HWIBAGUBnoAdRu1aXeh364t/O7wXDAcTJDP8PNuNKWUDWie+A==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + dev: false + /is-array-buffer@3.0.2: resolution: {integrity: sha512-y+FyyR/w8vfIRq4eQcM1EYgSTnmHXPqaF+IgzgraytCFq5Xh8lllDVmAZolPJiZttZLeFSINPYMaEJ7/vWUa1w==} dependencies: @@ -2383,6 +2394,13 @@ packages: has-tostringtag: 1.0.0 dev: true + /is-relative-url@4.0.0: + resolution: {integrity: sha512-PkzoL1qKAYXNFct5IKdKRH/iBQou/oCC85QhXj6WKtUQBliZ4Yfd3Zk27RHu9KQG8r6zgvAA2AQKC9p+rqTszg==} + engines: {node: '>=14.16'} + dependencies: + is-absolute-url: 4.0.1 + dev: false + /is-shared-array-buffer@1.0.2: resolution: {integrity: sha512-sqN2UDu1/0y6uvXyStCOzyhAjCSlHceFoMKJW8W9EU9cvic/QdsZ0kEU93HEy3IUEFZIiH/3w+AH/UQbPHNdhA==} dependencies: @@ -2833,7 +2851,7 @@ packages: resolution: {integrity: sha512-8+Ib8rRJ4L0o3kfmyVCL7gzrohyDe0cMFTBa2d364yIrEGMEoetznKJx899YxjybU6bL9SQkYPSBBs1gyYs8Xg==} dependencies: '@sinonjs/commons': 2.0.0 - '@sinonjs/fake-timers': 10.2.0 + '@sinonjs/fake-timers': 10.1.0 '@sinonjs/text-encoding': 0.7.2 just-extend: 4.2.1 path-to-regexp: 1.8.0 @@ -2880,6 +2898,11 @@ packages: engines: {node: '>=0.10.0'} dev: true + /normalize-url@8.0.0: + resolution: {integrity: sha512-uVFpKhj5MheNBJRTiMZ9pE/7hD1QTeEvugSJW/OmLzAp78PB5O6adfMNTvmfKhXBkvCzC+rqifWcVYpGFwTjnw==} + engines: {node: '>=14.16'} + dev: false + /npm-run-all@4.1.5: resolution: {integrity: sha512-Oo82gJDAVcaMdi3nuoKFavkIHBRVqQ1qvMb+9LHk/cF4P6B2m8aP04hGf7oL6wZ9BuGwX1onlLhpuoofSyoQDQ==} engines: {node: '>= 4'} @@ -3058,8 +3081,8 @@ packages: engines: {node: '>=12'} dev: true - /p-timeout@6.1.1: - resolution: {integrity: sha512-yqz2Wi4fiFRpMmK0L2pGAU49naSUaP23fFIQL2Y6YT+qDGPoFwpvgQM/wzc6F8JoenUkIlAFa4Ql7NguXBxI7w==} + /p-timeout@6.1.2: + resolution: {integrity: sha512-UbD77BuZ9Bc9aABo74gfXhNvzC9Tx7SxtHSh1fxvx3jTLLYvmVhiQZZrJzqqU0jKbN32kb5VOKiLEQI/3bIjgQ==} engines: {node: '>=14.16'} dev: false @@ -3375,8 +3398,8 @@ packages: glob: 7.2.3 dev: true - /rollup@3.24.1: - resolution: {integrity: sha512-REHe5dx30ERBRFS0iENPHy+t6wtSEYkjrhwNsLyh3qpRaZ1+aylvMUdMBUHWUD/RjjLmLzEvY8Z9XRlpcdIkHA==} + /rollup@3.25.1: + resolution: {integrity: sha512-tywOR+rwIt5m2ZAWSe5AIJcTat8vGlnPFAv15ycCrw33t6iFsXZ6mzHVFh2psSjxQPmI+xgzMZZizUAukBI4aQ==} engines: {node: '>=14.18.0', npm: '>=8.0.0'} hasBin: true optionalDependencies: @@ -3477,11 +3500,11 @@ packages: engines: {node: '>=14'} dev: true - /sinon@15.1.0: - resolution: {integrity: sha512-cS5FgpDdE9/zx7no8bxROHymSlPLZzq0ChbbLk1DrxBfc+eTeBK3y8nIL+nu/0QeYydhhbLIr7ecHJpywjQaoQ==} + /sinon@15.1.2: + resolution: {integrity: sha512-uG1pU54Fis4EfYOPoEi13fmRHgZNg/u+3aReSEzHsN52Bpf+bMVfsBQS5MjouI+rTuG6UBIINlpuuO2Epr7SiA==} dependencies: '@sinonjs/commons': 3.0.0 - '@sinonjs/fake-timers': 10.2.0 + '@sinonjs/fake-timers': 10.1.0 '@sinonjs/samsam': 8.0.0 diff: 5.1.0 nise: 5.1.4 @@ -3842,7 +3865,7 @@ packages: joycon: 3.1.1 postcss-load-config: 3.1.4 resolve-from: 5.0.0 - rollup: 3.24.1 + rollup: 3.25.1 source-map: 0.8.0-beta.0 sucrase: 3.32.0 tree-kill: 1.2.2 diff --git a/scratch/apify.ts b/scratch/apify.ts new file mode 100644 index 00000000..aabba731 --- /dev/null +++ b/scratch/apify.ts @@ -0,0 +1,13 @@ +import { ApifyClient } from 'apify-client' +import 'dotenv/config' + +async function main() { + const apify = new ApifyClient({ + token: process.env.APIFY_API_KEY + }) + + const actor = await apify.actor('apify/website-content-crawler').get() + console.log(actor) +} + +main() diff --git a/src/services/bing-web-search.ts b/src/services/bing-web-search.ts index 532d0af4..1b7b5338 100644 --- a/src/services/bing-web-search.ts +++ b/src/services/bing-web-search.ts @@ -1,6 +1,6 @@ -import ky from 'ky' +import defaultKy from 'ky' -export const BING_BASE_URL = 'https://api.bing.microsoft.com' +export const BING_API_BASE_URL = 'https://api.bing.microsoft.com' export interface BingWebSearchQuery { q: string @@ -235,22 +235,30 @@ interface FluffyContractualRule { } export class BingWebSearchClient { + api: typeof defaultKy + apiKey: string - baseUrl: string + apiBaseUrl: string constructor({ apiKey = process.env.BING_API_KEY, - baseUrl = BING_BASE_URL + apiBaseUrl = BING_API_BASE_URL, + ky = defaultKy }: { apiKey?: string - baseUrl?: string + apiBaseUrl?: string + ky?: typeof defaultKy } = {}) { if (!apiKey) { throw new Error(`Error BingWebSearchClient missing required "apiKey"`) } this.apiKey = apiKey - this.baseUrl = baseUrl + this.apiBaseUrl = apiBaseUrl + + this.api = ky.extend({ + prefixUrl: this.apiBaseUrl + }) } async search(queryOrOpts: string | BingWebSearchQuery) { @@ -269,9 +277,9 @@ export class BingWebSearchClient { ...queryOrOpts } - console.log(searchParams) - return ky - .get(`${this.baseUrl}/v7.0/search`, { + // console.log(searchParams) + return this.api + .get('v7.0/search', { headers: { 'Ocp-Apim-Subscription-Key': this.apiKey }, diff --git a/src/services/diffbot.ts b/src/services/diffbot.ts new file mode 100644 index 00000000..bbd8a97c --- /dev/null +++ b/src/services/diffbot.ts @@ -0,0 +1,303 @@ +import defaultKy from 'ky' + +export const DIFFBOT_API_BASE_URL = 'https://api.diffbot.com' +export const DIFFBOT_KNOWLEDGE_GRAPH_API_BASE_URL = 'https://kg.diffbot.com' + +export interface DiffbotExtractOptions { + /** Specify optional fields to be returned from any fully-extracted pages, e.g.: &fields=querystring,links. See available fields within each API's individual documentation pages. + * @see https://docs.diffbot.com/reference/extract-optional-fields + */ + fields?: string[] + + /** (*Undocumented*) Pass paging=false to disable automatic concatenation of multiple-page articles. (By default, Diffbot will concatenate up to 20 pages of a single article.) */ + paging?: boolean + + /** Pass discussion=false to disable automatic extraction of comments or reviews from pages identified as articles or products. This will not affect pages identified as discussions. */ + discussion?: boolean + + /** Sets a value in milliseconds to wait for the retrieval/fetch of content from the requested URL. The default timeout for the third-party response is 30 seconds (30000). */ + timeout?: number + + /** Used to specify the IP address of a custom proxy that will be used to fetch the target page, instead of Diffbot's default IPs/proxies. (Ex: &proxy=168.212.226.204) */ + proxy?: string + + /** Used to specify the authentication parameters that will be used with the proxy specified in the &proxy parameter. (Ex: &proxyAuth=username:password) */ + proxyAuth?: string + + /** `none` will instruct Extract to not use proxies, even if proxies have been enabled for this particular URL globally. */ + useProxy?: string + + /** @see https://docs.diffbot.com/reference/extract-custom-javascript */ + customJs?: string + + /** @see https://docs.diffbot.com/reference/extract-custom-headers */ + customHeaders?: Record +} + +export interface DiffbotExtractAnalyzeOptions extends DiffbotExtractOptions { + /** Web page URL of the analyze to process */ + url: string + + /** By default the Analyze API will fully extract all pages that match an existing Automatic API -- articles, products or image pages. Set mode to a specific page-type (e.g., mode=article) to extract content only from that specific page-type. All other pages will simply return the default Analyze fields. */ + mode?: string + + /** Force any non-extracted pages (those with a type of "other") through a specific API. For example, to route all "other" pages through the Article API, pass &fallback=article. Pages that utilize this functionality will return a fallbackType field at the top-level of the response and a originalType field within each extracted object, both of which will indicate the fallback API used. */ + fallback?: string +} + +export interface DiffbotExtractArticleOptions extends DiffbotExtractOptions { + /** Web page URL of the analyze to process */ + url: string + + /** Set the maximum number of automatically-generated tags to return. By default a maximum of ten tags will be returned. */ + maxTags?: number + + /** Set the minimum relevance score of tags to return, between 0.0 and 1.0. By default only tags with a score equal to or above 0.5 will be returned. */ + tagConfidence?: number + + /** Used to request the output of the Diffbot Natural Language API in the field naturalLanguage. Example: &naturalLanguage=entities,facts,categories,sentiment. */ + naturalLanguage?: string[] +} + +export interface DiffbotExtractResponse { + request: DiffbotRequest + objects: DiffbotObject[] +} + +export type DiffbotExtractArticleResponse = DiffbotExtractResponse + +export interface DiffbotExtractAnalyzeResponse extends DiffbotExtractResponse { + type: string + title: string + humanLanguage: string +} + +export interface DiffbotObject { + date: string + sentiment: number + images: DiffbotImage[] + author: string + estimatedDate: string + publisherRegion: string + icon: string + diffbotUri: string + siteName: string + type: string + title: string + tags: DiffbotTag[] + publisherCountry: string + humanLanguage: string + authorUrl: string + pageUrl: string + html: string + text: string + categories?: DiffbotCategory[] + authors: DiffbotAuthor[] + breadcrumb?: DiffbotBreadcrumb[] + meta?: any +} + +interface DiffbotAuthor { + name: string + link: string +} + +interface DiffbotCategory { + score: number + name: string + id: string +} + +export interface DiffbotBreadcrumb { + link: string + name: string +} + +interface DiffbotImage { + url: string + diffbotUri: string + + naturalWidth: number + naturalHeight: number + width: number + height: number + + isCached?: boolean + primary?: boolean +} + +interface DiffbotTag { + score: number + sentiment: number + count: number + label: string + uri: string + rdfTypes: string[] +} + +interface DiffbotRequest { + pageUrl: string + api: string + version: number +} + +export interface Image { + naturalHeight: number + diffbotUri: string + url: string + naturalWidth: number + primary: boolean +} + +export interface Tag { + score: number + sentiment: number + count: number + label: string + uri: string + rdfTypes: string[] +} + +export interface Request { + pageUrl: string + api: string + version: number +} + +export interface DiffbotSearchKnowledgeGraphOptions { + type?: 'query' | 'text' | 'queryTextFallback' | 'crawl' + query: string + col?: string + from?: number + size?: number + + // NOTE: we only support `json`, so these options are not needed + // We can always convert from json to another format if needed. + // format?: 'json' | 'jsonl' | 'csv' | 'xls' | 'xlsx' + // exportspec?: string + // exportseparator?: string + // exportfile?: string + + filter?: string + jsonmode?: 'extended' | 'id' + nonCanonicalFacts?: boolean + noDedupArticles?: boolean + cluster?: 'all' | 'best' | 'dedupe' + report?: boolean +} + +export interface DiffbotSearchKnowledgeGraphResponse { + version: number + hits: number + results: number + kgversion: string + diffbot_type: string + facet: boolean + data: DiffbotKnowledgeGraphNode[] +} + +export interface DiffbotKnowledgeGraphNode { + score: number + entity: DiffbotKnowledgeGraphEntity + entity_ctx: any + errors: string[] + callbackQuery: string + upperBound: number + lowerBound: number + count: number + value: string + uri: string +} + +export interface DiffbotKnowledgeGraphEntity { + id: string + images: DiffbotImage[] + diffbotUri: string + name: string + origins: string[] +} + +export class DiffbotClient { + api: typeof defaultKy + apiKnowledgeGraph: typeof defaultKy + + apiKey: string + apiBaseUrl: string + apiKnowledgeGraphBaseUrl: string + + constructor({ + apiKey = process.env.DIFFBOT_API_KEY, + apiBaseUrl = DIFFBOT_API_BASE_URL, + apiKnowledgeGraphBaseUrl = DIFFBOT_KNOWLEDGE_GRAPH_API_BASE_URL, + timeoutMs = 60_000, + ky = defaultKy + }: { + apiKey?: string + apiBaseUrl?: string + apiKnowledgeGraphBaseUrl?: string + timeoutMs?: number + ky?: typeof defaultKy + } = {}) { + if (!apiKey) { + throw new Error(`Error DiffbotClient missing required "apiKey"`) + } + + this.apiKey = apiKey + this.apiBaseUrl = apiBaseUrl + this.apiKnowledgeGraphBaseUrl = apiKnowledgeGraphBaseUrl + + this.api = ky.extend({ prefixUrl: apiBaseUrl, timeout: timeoutMs }) + this.apiKnowledgeGraph = ky.extend({ + prefixUrl: apiKnowledgeGraphBaseUrl, + timeout: timeoutMs + }) + } + + protected async _extract< + T extends DiffbotExtractResponse = DiffbotExtractResponse + >(endpoint: string, options: DiffbotExtractOptions): Promise { + const { customJs, customHeaders, ...rest } = options + const searchParams: Record = { + ...rest, + token: this.apiKey + } + const headers = { + ...Object.fromEntries( + [['X-Forward-X-Evaluate', customJs]].filter(([, value]) => value) + ), + ...customHeaders + } + + for (const [key, value] of Object.entries(rest)) { + if (Array.isArray(value)) { + searchParams[key] = value.join(',') + } + } + + return this.api + .get(endpoint, { + searchParams, + headers + }) + .json() + } + + async extractAnalyze(options: DiffbotExtractAnalyzeOptions) { + return this._extract('v3/analyze', options) + } + + async extractArticle(options: DiffbotExtractArticleOptions) { + return this._extract('v3/article', options) + } + + async searchKnowledgeGraph(options: DiffbotSearchKnowledgeGraphOptions) { + return this.apiKnowledgeGraph + .get('kg/v3/dql', { + searchParams: { + ...options, + token: this.apiKey + } + }) + .json() + } +} diff --git a/src/services/index.ts b/src/services/index.ts index 1620288e..201ca095 100644 --- a/src/services/index.ts +++ b/src/services/index.ts @@ -1,4 +1,5 @@ export * from './bing-web-search' +export * from './diffbot' export * from './metaphor' export * from './novu' export * from './serpapi' diff --git a/src/services/metaphor.ts b/src/services/metaphor.ts index c30641fc..59aaebe3 100644 --- a/src/services/metaphor.ts +++ b/src/services/metaphor.ts @@ -1,7 +1,7 @@ -import ky from 'ky' +import defaultKy from 'ky' import { z } from 'zod' -export const METAPHOR_BASE_URL = 'https://api.metaphor.systems' +export const METAPHOR_API_BASE_URL = 'https://api.metaphor.systems' // https://metaphorapi.readme.io/reference/search export const MetaphorSearchInputSchema = z.object({ @@ -33,27 +33,35 @@ export const MetaphorSearchOutputSchema = z.object({ export type MetaphorSearchOutput = z.infer export class MetaphorClient { + api: typeof defaultKy + apiKey: string - baseUrl: string + apiBaseUrl: string constructor({ apiKey = process.env.METAPHOR_API_KEY, - baseUrl = METAPHOR_BASE_URL + apiBaseUrl = METAPHOR_API_BASE_URL, + ky = defaultKy }: { apiKey?: string - baseUrl?: string + apiBaseUrl?: string + ky?: typeof defaultKy } = {}) { if (!apiKey) { throw new Error(`Error MetaphorClient missing required "apiKey"`) } this.apiKey = apiKey - this.baseUrl = baseUrl + this.apiBaseUrl = apiBaseUrl + + this.api = ky.extend({ + prefixUrl: this.apiBaseUrl + }) } async search(params: MetaphorSearchInput) { - return ky - .post(`${this.baseUrl}/search`, { + return this.api + .post('search', { headers: { 'x-api-key': this.apiKey }, diff --git a/src/services/novu.ts b/src/services/novu.ts index 6e3f4fc1..047b58a9 100644 --- a/src/services/novu.ts +++ b/src/services/novu.ts @@ -1,6 +1,6 @@ -import ky from 'ky' +import defaultKy from 'ky' -export const NOVU_BASE_URL = 'https://api.novu.co/v1' +export const NOVU_API_BASE_URL = 'https://api.novu.co/v1' export type NovuSubscriber = { subscriberId: string @@ -19,22 +19,30 @@ export type NovuTriggerEventResponse = { } export class NovuClient { + api: typeof defaultKy + apiKey: string - baseUrl: string + apiBaseUrl: string constructor({ apiKey = process.env.NOVU_API_KEY, - baseUrl = NOVU_BASE_URL + apiBaseUrl = NOVU_API_BASE_URL, + ky = defaultKy }: { apiKey?: string - baseUrl?: string + apiBaseUrl?: string + ky?: typeof defaultKy } = {}) { if (!apiKey) { throw new Error(`Error NovuClient missing required "apiKey"`) } this.apiKey = apiKey - this.baseUrl = baseUrl + this.apiBaseUrl = apiBaseUrl + + this.api = ky.extend({ + prefixUrl: this.apiBaseUrl + }) } async triggerEvent({ @@ -46,19 +54,18 @@ export class NovuClient { payload: Record to: NovuSubscriber[] }) { - const url = `${this.baseUrl}/events/trigger` - const headers = { - Authorization: `ApiKey ${this.apiKey}`, - 'Content-Type': 'application/json' - } - const response = await ky.post(url, { - headers, - json: { - name, - payload, - to - } - }) - return response.json() + return this.api + .post('events/trigger', { + headers: { + Authorization: `ApiKey ${this.apiKey}`, + 'Content-Type': 'application/json' + }, + json: { + name, + payload, + to + } + }) + .json() } } diff --git a/src/services/serpapi.ts b/src/services/serpapi.ts index 29a14143..36a0c804 100644 --- a/src/services/serpapi.ts +++ b/src/services/serpapi.ts @@ -1,4 +1,4 @@ -import ky from 'ky' +import defaultKy from 'ky' /** * All types have been exported from the `serpapi` package, which we're @@ -350,7 +350,8 @@ export type SerpAPISearchResponse = BaseResponse export interface SerpAPIClientOptions extends Partial { apiKey?: string - baseUrl?: string + apiBaseUrl?: string + ky?: typeof defaultKy } export const SERPAPI_BASE_URL = 'https://serpapi.com' @@ -361,13 +362,16 @@ export const SERPAPI_BASE_URL = 'https://serpapi.com' * @see https://serpapi.com/search-api */ export class SerpAPIClient { + api: typeof defaultKy + apiKey: string - baseUrl: string + apiBaseUrl: string params: Partial constructor({ apiKey = process.env.SERPAPI_API_KEY ?? process.env.SERP_API_KEY, - baseUrl = SERPAPI_BASE_URL, + apiBaseUrl = SERPAPI_BASE_URL, + ky = defaultKy, ...params }: SerpAPIClientOptions = {}) { if (!apiKey) { @@ -375,8 +379,12 @@ export class SerpAPIClient { } this.apiKey = apiKey - this.baseUrl = baseUrl + this.apiBaseUrl = apiBaseUrl this.params = params + + this.api = ky.extend({ + prefixUrl: this.apiBaseUrl + }) } async search(queryOrOpts: string | { query: string }) { @@ -384,8 +392,8 @@ export class SerpAPIClient { typeof queryOrOpts === 'string' ? queryOrOpts : queryOrOpts.query const { timeout, ...rest } = this.params - return ky - .get(`${this.baseUrl}/search`, { + return this.api + .get('search', { searchParams: { ...rest, engine: 'google', diff --git a/src/services/slack.ts b/src/services/slack.ts index 9f0c22de..62e8aee6 100644 --- a/src/services/slack.ts +++ b/src/services/slack.ts @@ -1,4 +1,4 @@ -import ky from 'ky' +import defaultKy from 'ky' import { sleep } from '@/utils' @@ -256,30 +256,33 @@ export type SlackSendAndWaitOptions = { } export class SlackClient { - private api: typeof ky + protected api: typeof defaultKy protected defaultChannel?: string constructor({ apiKey = process.env.SLACK_API_KEY, baseUrl = SLACK_API_BASE_URL, - defaultChannel = process.env.SLACK_DEFAULT_CHANNEL + defaultChannel = process.env.SLACK_DEFAULT_CHANNEL, + ky = defaultKy }: { apiKey?: string baseUrl?: string defaultChannel?: string + ky?: typeof defaultKy } = {}) { if (!apiKey) { throw new Error(`Error SlackClient missing required "apiKey"`) } - this.api = ky.create({ + this.defaultChannel = defaultChannel + + this.api = ky.extend({ prefixUrl: baseUrl, headers: { Authorization: `Bearer ${apiKey}` } }) - this.defaultChannel = defaultChannel } /** @@ -328,10 +331,11 @@ export class SlackClient { * Returns a list of messages that were sent in a channel after a given timestamp both directly and in threads. */ private async fetchCandidates(channel: string, ts: string) { - let candidates: SlackMessage[] = [] const history = await this.fetchConversationHistory({ channel }) const directReplies = await this.fetchReplies({ channel, ts }) + let candidates: SlackMessage[] = [] + if (directReplies.ok) { candidates = candidates.concat(directReplies.messages) } @@ -349,6 +353,7 @@ export class SlackClient { candidates.sort((a, b) => { return parseFloat(b.ts) - parseFloat(a.ts) }) + return candidates } @@ -357,7 +362,7 @@ export class SlackClient { * * ### Notes * - * - The implementation will poll for replies to the message until the timeout is reached. This is not ideal, but it is the only way to retrieve replies to a message in Slack without spinning up a local server to receive webhook events. + * - The implementation will poll for replies to the message until the timeout is reached. This is not ideal, but it is the only way to retrieve replies to a message in Slack without spinning up a local server to receive webhook events. */ public async sendAndWaitForReply({ text, diff --git a/src/services/twilio-conversation.ts b/src/services/twilio-conversation.ts index 82240d97..d3c691ef 100644 --- a/src/services/twilio-conversation.ts +++ b/src/services/twilio-conversation.ts @@ -1,9 +1,9 @@ -import ky, { type KyResponse } from 'ky' +import defaultKy from 'ky' import { DEFAULT_BOT_NAME } from '@/constants' import { sleep } from '@/utils' -export const TWILIO_CONVERSATION_BASE_URL = +export const TWILIO_CONVERSATION_API_BASE_URL = 'https://conversations.twilio.com/v1' export const DEFAULT_TWILIO_TIMEOUT_MS = 120_000 @@ -129,7 +129,8 @@ export type TwilioSendAndWaitOptions = { * @see {@link https://www.twilio.com/docs/conversations/api} */ export class TwilioConversationClient { - api: typeof ky + api: typeof defaultKy + phoneNumber: string botName: string @@ -137,18 +138,26 @@ export class TwilioConversationClient { accountSid = process.env.TWILIO_ACCOUNT_SID, authToken = process.env.TWILIO_AUTH_TOKEN, phoneNumber = process.env.TWILIO_PHONE_NUMBER, - baseUrl = TWILIO_CONVERSATION_BASE_URL, - botName = DEFAULT_BOT_NAME + apiBaseUrl = TWILIO_CONVERSATION_API_BASE_URL, + botName = DEFAULT_BOT_NAME, + ky = defaultKy }: { accountSid?: string authToken?: string phoneNumber?: string - baseUrl?: string + apiBaseUrl?: string botName?: string + ky?: typeof defaultKy } = {}) { - if (!accountSid || !authToken) { + if (!accountSid) { throw new Error( - `Error TwilioConversationClient missing required "accountSid" and/or "authToken"` + `Error TwilioConversationClient missing required "accountSid"` + ) + } + + if (!authToken) { + throw new Error( + `Error TwilioConversationClient missing required "authToken"` ) } @@ -161,8 +170,8 @@ export class TwilioConversationClient { this.botName = botName this.phoneNumber = phoneNumber - this.api = ky.create({ - prefixUrl: baseUrl, + this.api = ky.extend({ + prefixUrl: apiBaseUrl, headers: { Authorization: 'Basic ' + @@ -175,7 +184,7 @@ export class TwilioConversationClient { /** * Deletes a conversation and all its messages. */ - async deleteConversation(conversationSid: string): Promise { + async deleteConversation(conversationSid: string) { return this.api.delete(`Conversations/${conversationSid}`) } diff --git a/src/url-utils.ts b/src/url-utils.ts new file mode 100644 index 00000000..ee545860 --- /dev/null +++ b/src/url-utils.ts @@ -0,0 +1,44 @@ +import isRelativeUrl from 'is-relative-url' +import normalizeUrlImpl from 'normalize-url' +import QuickLRU from 'quick-lru' + +// const protocolAllowList = new Set(['https:', 'http:']) +const normalizedUrlCache = new QuickLRU({ + maxSize: 4000 +}) + +export function normalizeUrl(url: string): string | null { + let normalizedUrl: string | null | undefined + + try { + if (!url || (isRelativeUrl(url) && !url.startsWith('//'))) { + return null + } + + normalizedUrl = normalizedUrlCache.get(url) + + if (normalizedUrl !== undefined) { + return normalizedUrl + } + + normalizedUrl = normalizeUrlImpl(url, { + stripWWW: false, + defaultProtocol: 'https', + normalizeProtocol: true, + forceHttps: false, + stripHash: false, + stripTextFragment: true, + removeQueryParameters: [/^utm_\w+/i, 'ref'], + removeTrailingSlash: true, + removeSingleSlash: true, + removeExplicitPort: true, + sortQueryParameters: true + }) + } catch (err) { + // ignore invalid urls + normalizedUrl = null + } + + normalizedUrlCache.set(url, normalizedUrl!) + return normalizedUrl +} diff --git a/test/_utils.ts b/test/_utils.ts index f8589980..3383f8a4 100644 --- a/test/_utils.ts +++ b/test/_utils.ts @@ -4,10 +4,12 @@ import 'dotenv/config' import hashObject from 'hash-obj' import Redis from 'ioredis' import Keyv from 'keyv' +import defaultKy from 'ky' import { OpenAIClient } from 'openai-fetch' import pMemoize from 'p-memoize' import { Agentic } from '@/agentic' +import { normalizeUrl } from '@/url-utils' export const fakeOpenAIAPIKey = 'fake-openai-api-key' export const fakeAnthropicAPIKey = 'fake-anthropic-api-key' @@ -39,6 +41,97 @@ keyv.has = async (key, ...rest) => { return res } +function getCacheKeyForRequest(request: Request): string | null { + const method = request.method.toLowerCase() + + if (method === 'get') { + const url = normalizeUrl(request.url) + + if (url) { + const cacheParams = { + // TODO: request.headers isn't a normal JS object... + headers: { ...request.headers } + } + + // console.log('getCacheKeyForRequest', { url, cacheParams }) + const cacheKey = getCacheKey(`http:${method} ${url}`, cacheParams) + return cacheKey + } + } + + return null +} + +/** + * Custom `ky` instance that caches GET JSON requests. + * + * TODO: + * - support non-GET requests + * - support non-JSON responses + */ +export const ky = defaultKy.extend({ + hooks: { + beforeRequest: [ + async (request) => { + try { + // console.log(`beforeRequest ${request.method} ${request.url}`) + + const cacheKey = getCacheKeyForRequest(request) + // console.log({ cacheKey }) + if (!cacheKey) { + return + } + + if (!(await keyv.has(cacheKey))) { + return + } + + const cachedResponse = await keyv.get(cacheKey) + // console.log({ cachedResponse }) + + if (!cachedResponse) { + return + } + + return new Response(JSON.stringify(cachedResponse), { + status: 200, + headers: { 'Content-Type': 'application/json' } + }) + } catch (err) { + console.error('ky beforeResponse cache error', err) + } + } + ], + + afterResponse: [ + async (request, _options, response) => { + try { + // console.log( + // `afterRequest ${request.method} ${request.url} ⇒ ${response.status}` + // ) + + if (response.status < 200 || response.status >= 300) { + return + } + + const cacheKey = getCacheKeyForRequest(request) + // console.log({ cacheKey }) + if (!cacheKey) { + return + } + + const responseBody = await response.json() + // console.log({ responseBody }) + + await keyv.set(cacheKey, responseBody) + } catch (err) { + console.error('ky afterResponse cache error', err) + } + } + ] + } +}) + export class OpenAITestClient extends OpenAIClient { createChatCompletion = pMemoize(super.createChatCompletion, { cacheKey: (params) => getCacheKey('openai:chat', params), diff --git a/test/bing.test.ts b/test/services/bing.test.ts similarity index 69% rename from test/bing.test.ts rename to test/services/bing.test.ts index 2569c804..04479a62 100644 --- a/test/bing.test.ts +++ b/test/services/bing.test.ts @@ -2,7 +2,7 @@ import test from 'ava' import { BingWebSearchClient } from '@/services' -import './_utils' +import { ky } from '../_utils' test('BingWebSearchClient.search', async (t) => { if (!process.env.BING_API_KEY) { @@ -10,9 +10,9 @@ test('BingWebSearchClient.search', async (t) => { } t.timeout(2 * 60 * 1000) - const client = new BingWebSearchClient() + const client = new BingWebSearchClient({ ky }) const result = await client.search('coffee') // console.log(result) - t.truthy(result?.webPages) + t.true(result?.webPages?.value?.length > 0) }) diff --git a/test/services/diffbot.test.ts b/test/services/diffbot.test.ts new file mode 100644 index 00000000..6547bcb0 --- /dev/null +++ b/test/services/diffbot.test.ts @@ -0,0 +1,37 @@ +import test from 'ava' + +import { DiffbotClient } from '@/services' + +import { isCI, ky } from '../_utils' + +test('Diffbot.analyze', async (t) => { + if (!process.env.DIFFBOT_API_KEY || isCI) { + return t.pass() + } + + t.timeout(2 * 60 * 1000) + const client = new DiffbotClient({ ky }) + + const result = await client.extractAnalyze({ + url: 'https://transitivebullsh.it' + }) + // console.log(result) + t.is(result.type, 'list') + t.is(result.objects?.length, 1) +}) + +test('Diffbot.article', async (t) => { + if (!process.env.DIFFBOT_API_KEY || isCI) { + return t.pass() + } + + t.timeout(2 * 60 * 1000) + const client = new DiffbotClient({ ky }) + + const result = await client.extractArticle({ + url: 'https://www.nytimes.com/2023/05/31/magazine/ai-start-up-accelerator-san-francisco.html' + // fields: ['meta'] + }) + // console.log(JSON.stringify(result, null, 2)) + t.is(result.objects[0].type, 'article') +}) diff --git a/test/novu.test.ts b/test/services/novu.test.ts similarity index 86% rename from test/novu.test.ts rename to test/services/novu.test.ts index 125046f0..76b166cd 100644 --- a/test/novu.test.ts +++ b/test/services/novu.test.ts @@ -2,7 +2,7 @@ import test from 'ava' import { NovuClient } from '@/services/novu' -import './_utils' +import { ky } from '../_utils' test('NovuClient.triggerEvent', async (t) => { if (!process.env.NOVU_API_KEY) { @@ -10,7 +10,7 @@ test('NovuClient.triggerEvent', async (t) => { } t.timeout(2 * 60 * 1000) - const client = new NovuClient() + const client = new NovuClient({ ky }) const result = await client.triggerEvent({ name: 'send-email', diff --git a/test/serpapi.test.ts b/test/services/serpapi.test.ts similarity index 80% rename from test/serpapi.test.ts rename to test/services/serpapi.test.ts index 5adf4856..36054a69 100644 --- a/test/serpapi.test.ts +++ b/test/services/serpapi.test.ts @@ -2,7 +2,7 @@ import test from 'ava' import { SerpAPIClient } from '@/services/serpapi' -import './_utils' +import { ky } from '../_utils' test('SerpAPIClient.search', async (t) => { if (!process.env.SERPAPI_API_KEY) { @@ -10,7 +10,7 @@ test('SerpAPIClient.search', async (t) => { } t.timeout(2 * 60 * 1000) - const client = new SerpAPIClient() + const client = new SerpAPIClient({ ky }) const result = await client.search('coffee') // console.log(result) diff --git a/test/slack.test.ts b/test/services/slack.test.ts similarity index 98% rename from test/slack.test.ts rename to test/services/slack.test.ts index 1ea71768..11f4f2ac 100644 --- a/test/slack.test.ts +++ b/test/services/slack.test.ts @@ -2,7 +2,7 @@ import test from 'ava' import { SlackClient } from '@/services/slack' -import './_utils' +import '../_utils' test('SlackClient.sendMessage', async (t) => { if (!process.env.SLACK_API_KEY || !process.env.SLACK_DEFAULT_CHANNEL) { diff --git a/test/twilio-conversation.test.ts b/test/services/twilio-conversation.test.ts similarity index 99% rename from test/twilio-conversation.test.ts rename to test/services/twilio-conversation.test.ts index 341b4fa9..d7633357 100644 --- a/test/twilio-conversation.test.ts +++ b/test/services/twilio-conversation.test.ts @@ -2,7 +2,7 @@ import test from 'ava' import { TwilioConversationClient } from '@/services/twilio-conversation' -import './_utils' +import '../_utils' test.serial('TwilioConversationClient.createConversation', async (t) => { if (!process.env.TWILIO_ACCOUNT_SID || !process.env.TWILIO_AUTH_TOKEN) { diff --git a/test/url-utils.test.ts b/test/url-utils.test.ts new file mode 100644 index 00000000..fb901e44 --- /dev/null +++ b/test/url-utils.test.ts @@ -0,0 +1,27 @@ +import test from 'ava' + +import { normalizeUrl } from '@/url-utils' + +test('normalizeUrl', async (t) => { + t.is(normalizeUrl('https://www.google.com'), 'https://www.google.com') + t.is(normalizeUrl('//www.google.com'), 'https://www.google.com') + t.is( + normalizeUrl('https://www.google.com/foo?'), + 'https://www.google.com/foo' + ) + t.is( + normalizeUrl('https://www.google.com/?foo=bar&dog=cat'), + 'https://www.google.com/?dog=cat&foo=bar' + ) + t.is( + normalizeUrl('https://google.com/abc/123//'), + 'https://google.com/abc/123' + ) +}) + +test('normalizeUrl - invalid urls', async (t) => { + t.is(normalizeUrl('/foo'), null) + t.is(normalizeUrl('/foo/bar/baz'), null) + t.is(normalizeUrl('://foo.com'), null) + t.is(normalizeUrl('foo'), null) +}) diff --git a/tsconfig.json b/tsconfig.json index d3b520bb..34b845fa 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,7 +1,7 @@ { "compilerOptions": { "target": "es2020", - "lib": ["esnext", "es2022.error"], + "lib": ["esnext", "es2022.error", "DOM"], "allowJs": true, "skipLibCheck": true, "strict": true,