⬅

src/services/clearbit-client.ts

@@ -5,11 +5,10 @@ import type { DeepNullable, KyInstance } from '../types.js'
 import { assert, delay, getEnv, throttleKy } from '../utils.js'
 
 export namespace clearbit {
-  // Only allow 20 clearbit API requests per 60s
+  // Allow up to 20 requests per minute by default.
   export const throttle = pThrottle({
     limit: 20,
-    interval: 60 * 1000,
-    strict: true
+    interval: 60 * 1000
   })
 
   export interface CompanyEnrichmentOptions {
@@ -524,7 +523,10 @@ export class ClearbitClient {
     throttle?: boolean
     ky?: KyInstance
   } = {}) {
-    assert(apiKey, 'Error clearbit client missing required "apiKey"')
+    assert(
+      apiKey,
+      'ClearbitClient missing required "apiKey" (defaults to "CLEARBIT_API_KEY")'
+    )
 
     this.apiKey = apiKey
 

src/services/diffbot-client.ts

@@ -7,6 +7,7 @@ export namespace diffbot {
   export const API_BASE_URL = 'https://api.diffbot.com'
   export const KNOWLEDGE_GRAPH_API_BASE_URL = 'https://kg.diffbot.com'
 
+  // Allow up to 5 requests per second by default.
   export const throttle = pThrottle({
     limit: 5,
     interval: 1000,

src/services/people-data-labs-client.ts

@@ -6,6 +6,7 @@ import { assert, getEnv, throttleKy } from '../utils.js'
 export namespace peopledatalabs {
   export const BASE_URL = 'https://api.peopledatalabs.com/v5/'
 
+  // Allow up to 20 requests per minute by default.
   export const throttle = pThrottle({
     limit: 20,
     interval: 60 * 1000,

src/services/perigon-client.ts

@@ -4,6 +4,7 @@ import pThrottle from 'p-throttle'
 import { assert, getEnv, throttleKy } from '../utils.js'
 
 export namespace perigon {
+  // Allow up to 20 requests per minute by default.
   export const throttle = pThrottle({
     limit: 20,
     interval: 60 * 1000,

src/services/predict-leads-client.ts

@@ -7,10 +7,10 @@ import { aiFunction, AIFunctionsProvider } from '../fns.js'
 import { assert, getEnv, pruneUndefined, throttleKy } from '../utils.js'
 
 export namespace predictleads {
+  // Allow up to 20 requests per minute by default.
   export const throttle = pThrottle({
     limit: 20,
-    interval: 60 * 1000,
-    strict: true
+    interval: 60 * 1000
   })
 
   export const DEFAULT_PAGE_SIZE = 100

src/services/proxycurl-client.ts

@@ -1,11 +1,18 @@
 import defaultKy, { type KyInstance } from 'ky'
+import pThrottle from 'p-throttle'
 import { z } from 'zod'
 
 import { aiFunction, AIFunctionsProvider } from '../fns.js'
-import { assert, getEnv } from '../utils.js'
+import { assert, getEnv, throttleKy } from '../utils.js'
 
 // All proxycurl types are auto-generated from their openapi spec
 export namespace proxycurl {
+  // Allow up to 1500 requests per minute by default.
+  export const throttle = pThrottle({
+    limit: 1500,
+    interval: 5 * 60 * 1000
+  })
+
   export const CompanyTypeSchema = z.enum([
     'EDUCATIONAL',
     'GOVERNMENT_AGENCY',
@@ -2012,10 +2019,12 @@ export class ProxycurlClient extends AIFunctionsProvider {
     apiKey = getEnv('PROXYCURL_API_KEY'),
     apiBaseUrl = getEnv('PROXYCURL_API_BASE_URL') ??
       'https://nubela.co/proxycurl',
+    throttle = true,
     ky = defaultKy
   }: {
     apiKey?: string
     apiBaseUrl?: string
+    throttle?: boolean
     ky?: KyInstance
   } = {}) {
     assert(
@@ -2031,7 +2040,9 @@ export class ProxycurlClient extends AIFunctionsProvider {
     this.apiKey = apiKey
     this.apiBaseUrl = apiBaseUrl
 
-    this.ky = ky.extend({
+    const throttledKy = throttle ? throttleKy(ky, proxycurl.throttle) : ky
+
+    this.ky = throttledKy.extend({
       prefixUrl: apiBaseUrl,
       headers: {
         Authorization: `Bearer ${apiKey}`

src/services/wikipedia-client.ts

@@ -6,7 +6,7 @@ import { aiFunction, AIFunctionsProvider } from '../fns.js'
 import { assert, getEnv, throttleKy } from '../utils.js'
 
 export namespace wikipedia {
-  // Only allow 200 requests per second by default.
+  // Allow up to 200 requests per second by default.
   export const throttle = pThrottle({
     limit: 200,
     interval: 1000