From 74c3cea5d384e501631c68d255085551cdb36d64 Mon Sep 17 00:00:00 2001 From: Travis Fischer Date: Fri, 2 Jun 2023 00:23:13 -0700 Subject: [PATCH] feat: add SerpAPIClient --- legacy/package.json | 2 ++ legacy/pnpm-lock.yaml | 48 ++++++++++++++++++++++++++++++++ legacy/readme.md | 2 +- legacy/src/services/serpapi.ts | 51 ++++++++++++++++++++++++++++++++++ legacy/test/serpapi.test.ts | 17 ++++++++++++ 5 files changed, 119 insertions(+), 1 deletion(-) create mode 100644 legacy/src/services/serpapi.ts create mode 100644 legacy/test/serpapi.test.ts diff --git a/legacy/package.json b/legacy/package.json index 00d7a5aa..07940d3d 100644 --- a/legacy/package.json +++ b/legacy/package.json @@ -45,6 +45,7 @@ "ky": "^0.33.3", "openai-fetch": "^1.5.0", "p-map": "^6.0.0", + "p-retry": "^5.1.2", "p-timeout": "^6.1.1", "quick-lru": "^6.1.1", "ts-dedent": "^2.2.0", @@ -72,6 +73,7 @@ "npm-run-all": "^4.1.5", "p-memoize": "^7.1.1", "prettier": "^2.8.8", + "serpapi": "^1.1.1", "sinon": "^15.1.0", "tsup": "^6.7.0", "tsx": "^3.12.7", diff --git a/legacy/pnpm-lock.yaml b/legacy/pnpm-lock.yaml index 17f96be1..55c569fd 100644 --- a/legacy/pnpm-lock.yaml +++ b/legacy/pnpm-lock.yaml @@ -26,6 +26,9 @@ dependencies: p-map: specifier: ^6.0.0 version: 6.0.0 + p-retry: + specifier: ^5.1.2 + version: 5.1.2 p-timeout: specifier: ^6.1.1 version: 6.1.1 @@ -103,6 +106,9 @@ devDependencies: prettier: specifier: ^2.8.8 version: 2.8.8 + serpapi: + specifier: ^1.1.1 + version: 1.1.1 sinon: specifier: ^15.1.0 version: 15.1.0 @@ -659,6 +665,10 @@ packages: resolution: {integrity: sha512-Gj7cI7z+98M282Tqmp2K5EIsoouUEzbBJhQQzDE3jSIRk6r9gsz0oUokqIUR4u1R3dMHo0pDHM7sNOHyhulypw==} dev: true + /@types/retry@0.12.1: + resolution: {integrity: sha512-xoDlM2S4ortawSWORYqsdU+2rxdh4LRW9ytc3zmT37RIKQh6IHyKwwtKhKis9ah8ol07DCkZxPt8BBvPjC6v4g==} + dev: false + /@types/semver@7.5.0: resolution: {integrity: sha512-G8hZ6XJiHnuhQKR7ZmysCeJWE08o8T0AXtk5darsCaTVsYZhhgUrq53jizaR2FvsoeCwJhlmwTjkXBY5Pn/ZHw==} dev: true @@ -1049,6 +1059,13 @@ packages: load-tsconfig: 0.2.5 dev: true + /busboy@1.6.0: + resolution: {integrity: sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA==} + engines: {node: '>=10.16.0'} + dependencies: + streamsearch: 1.1.0 + dev: true + /cac@6.7.14: resolution: {integrity: sha512-b6Ilus+c3RrdDk+JhLKUAQfzzgLEPy6wcXqS7f/xe1EETvsDP6GORG7SFuOs6cID5YkqchW/LXZbX5bc8j7ZcQ==} engines: {node: '>=8'} @@ -2886,6 +2903,14 @@ packages: type-fest: 3.11.1 dev: true + /p-retry@5.1.2: + resolution: {integrity: sha512-couX95waDu98NfNZV+i/iLt+fdVxmI7CbrrdC2uDWfPdUAApyxT4wmDlyOtR5KtTDmkDO0zDScDjDou9YHhd9g==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + dependencies: + '@types/retry': 0.12.1 + retry: 0.13.1 + dev: false + /p-timeout@5.1.0: resolution: {integrity: sha512-auFDyzzzGZZZdHz3BtET9VEz0SE/uMEAx7uWfGPucfzEwwe/xH0iVeZibQmANYE/hp9T2+UUZT5m+BKyrDp3Ew==} engines: {node: '>=12'} @@ -3176,6 +3201,11 @@ packages: signal-exit: 3.0.7 dev: true + /retry@0.13.1: + resolution: {integrity: sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==} + engines: {node: '>= 4'} + dev: false + /reusify@1.0.4: resolution: {integrity: sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==} engines: {iojs: '>=1.0.0', node: '>=0.10.0'} @@ -3240,6 +3270,12 @@ packages: type-fest: 0.13.1 dev: true + /serpapi@1.1.1: + resolution: {integrity: sha512-t5Bqu/6VMJ9naX8K+qCgUStpZOaNQFvIM4AudhMJLS6sqQT/EHaYrhGidDZHVx8QvcEdY6y1wNlxizOCtvJtUQ==} + dependencies: + undici: 5.22.1 + dev: true + /shebang-command@1.2.0: resolution: {integrity: sha512-EV3L1+UQWGor21OmnvojK36mhg+TyIKDh3iFBKBohr5xeXIhNBcx8oWdgkTEEQ+BEFFYdLRuqMfd5L84N1V5Vg==} engines: {node: '>=0.10.0'} @@ -3399,6 +3435,11 @@ packages: resolution: {integrity: sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==} dev: true + /streamsearch@1.1.0: + resolution: {integrity: sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==} + engines: {node: '>=10.0.0'} + dev: true + /string-argv@0.3.2: resolution: {integrity: sha512-aqD2Q0144Z+/RqG52NeHEkZauTAUWJO8c6yTftGJKO3Tja5tUgIfmIl6kExvhtxSDP7fXB6DvzkfMpCd/F3G+Q==} engines: {node: '>=0.6.19'} @@ -3744,6 +3785,13 @@ packages: which-boxed-primitive: 1.0.2 dev: true + /undici@5.22.1: + resolution: {integrity: sha512-Ji2IJhFXZY0x/0tVBXeQwgPlLWw13GVzpsWPQ3rV50IFMMof2I55PZZxtm4P6iNq+L5znYN9nSTAq0ZyE6lSJw==} + engines: {node: '>=14.0'} + dependencies: + busboy: 1.6.0 + dev: true + /uri-js@4.4.1: resolution: {integrity: sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==} dependencies: diff --git a/legacy/readme.md b/legacy/readme.md index cd40499a..99406efa 100644 --- a/legacy/readme.md +++ b/legacy/readme.md @@ -44,7 +44,7 @@ cp .env.example .env **Optional** - `ANTHROPIC_API_KEY` - [Anthropic](https://www.anthropic.com) API key ([docs](https://console.anthropic.com/docs)) -- `SERP_API_KEY` - [SerpApi](https://serpapi.com) API key ([docs](https://serpapi.com/search-api)) +- `SERPAPI_API_KEY` - [SerpApi](https://serpapi.com) API key ([docs](https://serpapi.com/search-api)) - `METAPHOR_API_KEY` - [Metaphor](https://metaphor.systems) API key ([docs](https://metaphorapi.readme.io/)) ### Local Testing diff --git a/legacy/src/services/serpapi.ts b/legacy/src/services/serpapi.ts new file mode 100644 index 00000000..1a2b5593 --- /dev/null +++ b/legacy/src/services/serpapi.ts @@ -0,0 +1,51 @@ +import ky from 'ky' +import type { BaseResponse, GoogleParameters } from 'serpapi' + +export type SerpAPIParams = Omit +export type SerpAPISearchResponse = BaseResponse + +export interface SerpAPIClientOptions extends Partial { + apiKey?: string + baseUrl?: string +} + +/** + * Lightweight wrapper around SerpAPI that only supports Google search. + */ +export class SerpAPIClient { + apiKey: string + baseUrl: string + params: Partial + + constructor({ + apiKey = process.env.SERPAPI_API_KEY ?? process.env.SERP_API_KEY, + baseUrl = 'https://serpapi.com', + ...params + }: SerpAPIClientOptions = {}) { + if (!apiKey) { + throw new Error(`Error SerpAPIClient missing required "apiKey"`) + } + + this.apiKey = apiKey + this.baseUrl = baseUrl + this.params = params + } + + async search(queryOrOpts: string | { query: string }) { + const query = + typeof queryOrOpts === 'string' ? queryOrOpts : queryOrOpts.query + const { timeout, ...rest } = this.params + + return ky + .get(`${this.baseUrl}/search`, { + searchParams: { + ...rest, + engine: 'google', + api_key: this.apiKey, + q: query + }, + timeout + }) + .json() + } +} diff --git a/legacy/test/serpapi.test.ts b/legacy/test/serpapi.test.ts new file mode 100644 index 00000000..cec7ceca --- /dev/null +++ b/legacy/test/serpapi.test.ts @@ -0,0 +1,17 @@ +import test from 'ava' + +import { SerpAPIClient } from '../src/services/serpapi' +import './_utils' + +test('SerpAPIClient.search', async (t) => { + if (!process.env.SERPAPI_API_KEY) { + return t.pass() + } + + t.timeout(2 * 60 * 1000) + const client = new SerpAPIClient() + + const result = await client.search('coffee') + console.log(result) + t.truthy(result) +})