From fa0628f8ca70ab9b692fbc83d4cc46edb5af47f2 Mon Sep 17 00:00:00 2001 From: Travis Fischer Date: Mon, 10 Jun 2024 01:30:56 -0500 Subject: [PATCH] feat: minor improvements to clearbit, diffbot, perigon, and proxycurl --- src/services/clearbit-client.ts | 57 +++++++++++++++++--------------- src/services/diffbot-client.ts | 2 +- src/services/perigon-client.ts | 48 ++++++++++++++------------- src/services/proxycurl-client.ts | 12 +++++-- 4 files changed, 66 insertions(+), 53 deletions(-) diff --git a/src/services/clearbit-client.ts b/src/services/clearbit-client.ts index a04b5761..bb6ac476 100644 --- a/src/services/clearbit-client.ts +++ b/src/services/clearbit-client.ts @@ -2,7 +2,13 @@ import defaultKy from 'ky' import pThrottle from 'p-throttle' import type { DeepNullable, KyInstance } from '../types.js' -import { assert, delay, getEnv, throttleKy } from '../utils.js' +import { + assert, + delay, + getEnv, + sanitizeSearchParams, + throttleKy +} from '../utils.js' export namespace clearbit { // Allow up to 600 requests per minute by default. @@ -11,6 +17,8 @@ export namespace clearbit { interval: 60 * 1000 }) + export const MAX_PAGE_SIZE = 100 + export interface CompanyEnrichmentOptions { domain: string webhook_url?: string @@ -358,19 +366,8 @@ export namespace clearbit { role: string seniority: string } -} -/** - * The Clearbit API helps with resolving and enriching people and company data. - * - * @see https://dashboard.clearbit.com/docs - */ -export class ClearbitClient { - protected readonly ky: KyInstance - protected readonly apiKey: string - protected readonly _maxPageSize = 100 - - static readonly PersonRoles = [ + export const PersonRoles = [ 'communications', 'customer_service', 'education', @@ -391,7 +388,7 @@ export class ClearbitClient { 'sales' ] - static readonly SenioritiesV2 = [ + export const SenioritiesV2 = [ 'Executive', 'VP', 'Owner', @@ -402,9 +399,9 @@ export class ClearbitClient { 'Entry' ] - static readonly Seniorities = ['executive', 'director', 'manager'] + export const Seniorities = ['executive', 'director', 'manager'] - static readonly SubIndustries = [ + export const SubIndustries = [ 'Automotive', 'Consumer Discretionary', 'Consumer Goods', @@ -516,6 +513,16 @@ export class ClearbitClient { 'Energy', 'Utilities' ] +} + +/** + * The Clearbit API helps with resolving and enriching people and company data. + * + * @see https://dashboard.clearbit.com/docs + */ +export class ClearbitClient { + protected readonly ky: KyInstance + protected readonly apiKey: string constructor({ apiKey = getEnv('CLEARBIT_API_KEY'), @@ -573,14 +580,13 @@ export class ClearbitClient { async prospectorPeopleV2(options: clearbit.PeopleSearchOptionsV2) { return this.ky .get('https://prospector.clearbit.com/v2/people/search', { - // @ts-expect-error location is a string[] and searchparams shows a TS error heres - searchParams: { + searchParams: sanitizeSearchParams({ ...options, page_size: Math.min( - this._maxPageSize, - options.page_size || this._maxPageSize + clearbit.MAX_PAGE_SIZE, + options.page_size || clearbit.MAX_PAGE_SIZE ) - } + }) }) .json() } @@ -588,15 +594,14 @@ export class ClearbitClient { async prospectorPeopleV1(options: clearbit.PeopleSearchOptionsV1) { return this.ky .get('https://prospector.clearbit.com/v1/people/search', { - // @ts-expect-error location is a string[] and searchparams shows a TS error heres - searchParams: { + searchParams: sanitizeSearchParams({ email: false, ...options, page_size: Math.min( - this._maxPageSize, - options.page_size || this._maxPageSize + clearbit.MAX_PAGE_SIZE, + options.page_size || clearbit.MAX_PAGE_SIZE ) - } + }) }) .json() } diff --git a/src/services/diffbot-client.ts b/src/services/diffbot-client.ts index 2bbb39b5..de329041 100644 --- a/src/services/diffbot-client.ts +++ b/src/services/diffbot-client.ts @@ -286,7 +286,7 @@ export namespace diffbot { .optional() .describe('Name of the entity'), url: z - .array(z.string()) + .union([z.string(), z.array(z.string())]) .optional() .describe('Origin or homepage URL of the entity'), phone: z.string().optional().describe('Phone number of the entity'), diff --git a/src/services/perigon-client.ts b/src/services/perigon-client.ts index 729104a1..e77c28d7 100644 --- a/src/services/perigon-client.ts +++ b/src/services/perigon-client.ts @@ -168,7 +168,7 @@ export namespace perigon { 'Labels to filter by, could be "Opinion", "Paid-news", "Non-news", etc. If multiple parameters are passed, they will be applied as OR operations.' ), excludeLabel: z - .union([ArticleLabelSchema, z.literal('Low Content')]) + .array(z.union([ArticleLabelSchema, z.literal('Low Content')])) .optional() .describe( 'Exclude results that include specific labels ("Opinion", "Non-news", "Paid News", etc.). You can filter multiple by repeating the parameter.' @@ -681,19 +681,20 @@ export class PerigonClient extends AIFunctionsProvider { }) }) async searchArticles(opts: perigon.ArticlesSearchOptions) { + const searchParams = sanitizeSearchParams({ + sortBy: 'relevance', + ...opts, + apiKey: this.apiKey, + size: Math.max( + 1, + Math.min(perigon.MAX_PAGE_SIZE, opts.size || perigon.DEFAULT_PAGE_SIZE) + ) + }) + console.log('perigon.searchArticles', searchParams) + return this.ky .get('all', { - searchParams: sanitizeSearchParams({ - ...opts, - apiKey: this.apiKey, - size: Math.max( - 1, - Math.min( - perigon.MAX_PAGE_SIZE, - opts.size || perigon.DEFAULT_PAGE_SIZE - ) - ) - }) + searchParams }) .json() } @@ -720,19 +721,20 @@ export class PerigonClient extends AIFunctionsProvider { }) }) async searchStories(opts: perigon.StoriesSearchOptions) { + const searchParams = sanitizeSearchParams({ + sortBy: 'relevance', + ...opts, + apiKey: this.apiKey, + size: Math.max( + 1, + Math.min(perigon.MAX_PAGE_SIZE, opts.size || perigon.DEFAULT_PAGE_SIZE) + ) + }) + console.log('perigon.searchStories', searchParams) + return this.ky .get('stories/all', { - searchParams: sanitizeSearchParams({ - ...opts, - apiKey: this.apiKey, - size: Math.max( - 1, - Math.min( - perigon.MAX_PAGE_SIZE, - opts.size || perigon.DEFAULT_PAGE_SIZE - ) - ) - }) + searchParams }) .json() } diff --git a/src/services/proxycurl-client.ts b/src/services/proxycurl-client.ts index 0d184391..b0f44079 100644 --- a/src/services/proxycurl-client.ts +++ b/src/services/proxycurl-client.ts @@ -1,3 +1,4 @@ +import type { Simplify } from 'type-fest' import defaultKy, { type KyInstance } from 'ky' import pThrottle from 'p-throttle' import { z } from 'zod' @@ -52,8 +53,13 @@ export namespace proxycurl { typeof CompanyProfileEndpointParamsQueryClassSchema > + /** + * Requires one of: + * - `facebook_profile_url` + * - `linkedin_profile_url` + * - `twitter_profile_url` + */ export const PersonProfileEndpointParamsQueryClassSchema = z.object({ - // requires one of `facebook_profile_url`, `linkedin_profile_url`, or `twitter_profile_url` facebook_profile_url: z.string().optional(), linkedin_profile_url: z.string().optional(), twitter_profile_url: z.string().optional(), @@ -68,8 +74,8 @@ export namespace proxycurl { fallback_to_cache: FallbackToCacheSchema, use_cache: UseCacheSchema }) - export type PersonProfileEndpointParamsQueryClass = z.infer< - typeof PersonProfileEndpointParamsQueryClassSchema + export type PersonProfileEndpointParamsQueryClass = Simplify< + z.infer > export const PersonLookupEndpointParamsQueryClassSchema = z.object({