From ac76b5098bf8a9fe00ce94b3f4b4229b745954e3 Mon Sep 17 00:00:00 2001
From: Travis Fischer <fisch0920@gmail.com>
Date: Sun, 2 Jun 2024 21:49:15 -0500
Subject: [PATCH] =?UTF-8?q?=E2=AC=85?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 src/services/clearbit-client.ts         | 10 ++++++----
 src/services/diffbot-client.ts          |  1 +
 src/services/people-data-labs-client.ts |  1 +
 src/services/perigon-client.ts          |  1 +
 src/services/predict-leads-client.ts    |  4 ++--
 src/services/proxycurl-client.ts        | 15 +++++++++++++--
 src/services/wikipedia-client.ts        |  2 +-
 7 files changed, 25 insertions(+), 9 deletions(-)

diff --git a/src/services/clearbit-client.ts b/src/services/clearbit-client.ts
index b213559..c945df1 100644
--- a/src/services/clearbit-client.ts
+++ b/src/services/clearbit-client.ts
@@ -5,11 +5,10 @@ import type { DeepNullable, KyInstance } from '../types.js'
 import { assert, delay, getEnv, throttleKy } from '../utils.js'
 
 export namespace clearbit {
-  // Only allow 20 clearbit API requests per 60s
+  // Allow up to 20 requests per minute by default.
   export const throttle = pThrottle({
     limit: 20,
-    interval: 60 * 1000,
-    strict: true
+    interval: 60 * 1000
   })
 
   export interface CompanyEnrichmentOptions {
@@ -524,7 +523,10 @@ export class ClearbitClient {
     throttle?: boolean
     ky?: KyInstance
   } = {}) {
-    assert(apiKey, 'Error clearbit client missing required "apiKey"')
+    assert(
+      apiKey,
+      'ClearbitClient missing required "apiKey" (defaults to "CLEARBIT_API_KEY")'
+    )
 
     this.apiKey = apiKey
 
diff --git a/src/services/diffbot-client.ts b/src/services/diffbot-client.ts
index 9f6780b..da90cba 100644
--- a/src/services/diffbot-client.ts
+++ b/src/services/diffbot-client.ts
@@ -7,6 +7,7 @@ export namespace diffbot {
   export const API_BASE_URL = 'https://api.diffbot.com'
   export const KNOWLEDGE_GRAPH_API_BASE_URL = 'https://kg.diffbot.com'
 
+  // Allow up to 5 requests per second by default.
   export const throttle = pThrottle({
     limit: 5,
     interval: 1000,
diff --git a/src/services/people-data-labs-client.ts b/src/services/people-data-labs-client.ts
index e0846ea..195f53b 100644
--- a/src/services/people-data-labs-client.ts
+++ b/src/services/people-data-labs-client.ts
@@ -6,6 +6,7 @@ import { assert, getEnv, throttleKy } from '../utils.js'
 export namespace peopledatalabs {
   export const BASE_URL = 'https://api.peopledatalabs.com/v5/'
 
+  // Allow up to 20 requests per minute by default.
   export const throttle = pThrottle({
     limit: 20,
     interval: 60 * 1000,
diff --git a/src/services/perigon-client.ts b/src/services/perigon-client.ts
index 60d19fa..54a863f 100644
--- a/src/services/perigon-client.ts
+++ b/src/services/perigon-client.ts
@@ -4,6 +4,7 @@ import pThrottle from 'p-throttle'
 import { assert, getEnv, throttleKy } from '../utils.js'
 
 export namespace perigon {
+  // Allow up to 20 requests per minute by default.
   export const throttle = pThrottle({
     limit: 20,
     interval: 60 * 1000,
diff --git a/src/services/predict-leads-client.ts b/src/services/predict-leads-client.ts
index 33b52b0..947a1cb 100644
--- a/src/services/predict-leads-client.ts
+++ b/src/services/predict-leads-client.ts
@@ -7,10 +7,10 @@ import { aiFunction, AIFunctionsProvider } from '../fns.js'
 import { assert, getEnv, pruneUndefined, throttleKy } from '../utils.js'
 
 export namespace predictleads {
+  // Allow up to 20 requests per minute by default.
   export const throttle = pThrottle({
     limit: 20,
-    interval: 60 * 1000,
-    strict: true
+    interval: 60 * 1000
   })
 
   export const DEFAULT_PAGE_SIZE = 100
diff --git a/src/services/proxycurl-client.ts b/src/services/proxycurl-client.ts
index e922e59..2de52ee 100644
--- a/src/services/proxycurl-client.ts
+++ b/src/services/proxycurl-client.ts
@@ -1,11 +1,18 @@
 import defaultKy, { type KyInstance } from 'ky'
+import pThrottle from 'p-throttle'
 import { z } from 'zod'
 
 import { aiFunction, AIFunctionsProvider } from '../fns.js'
-import { assert, getEnv } from '../utils.js'
+import { assert, getEnv, throttleKy } from '../utils.js'
 
 // All proxycurl types are auto-generated from their openapi spec
 export namespace proxycurl {
+  // Allow up to 1500 requests per minute by default.
+  export const throttle = pThrottle({
+    limit: 1500,
+    interval: 5 * 60 * 1000
+  })
+
   export const CompanyTypeSchema = z.enum([
     'EDUCATIONAL',
     'GOVERNMENT_AGENCY',
@@ -2012,10 +2019,12 @@ export class ProxycurlClient extends AIFunctionsProvider {
     apiKey = getEnv('PROXYCURL_API_KEY'),
     apiBaseUrl = getEnv('PROXYCURL_API_BASE_URL') ??
       'https://nubela.co/proxycurl',
+    throttle = true,
     ky = defaultKy
   }: {
     apiKey?: string
     apiBaseUrl?: string
+    throttle?: boolean
     ky?: KyInstance
   } = {}) {
     assert(
@@ -2031,7 +2040,9 @@ export class ProxycurlClient extends AIFunctionsProvider {
     this.apiKey = apiKey
     this.apiBaseUrl = apiBaseUrl
 
-    this.ky = ky.extend({
+    const throttledKy = throttle ? throttleKy(ky, proxycurl.throttle) : ky
+
+    this.ky = throttledKy.extend({
       prefixUrl: apiBaseUrl,
       headers: {
         Authorization: `Bearer ${apiKey}`
diff --git a/src/services/wikipedia-client.ts b/src/services/wikipedia-client.ts
index e8f0ebe..ed044b6 100644
--- a/src/services/wikipedia-client.ts
+++ b/src/services/wikipedia-client.ts
@@ -6,7 +6,7 @@ import { aiFunction, AIFunctionsProvider } from '../fns.js'
 import { assert, getEnv, throttleKy } from '../utils.js'
 
 export namespace wikipedia {
-  // Only allow 200 requests per second by default.
+  // Allow up to 200 requests per second by default.
   export const throttle = pThrottle({
     limit: 200,
     interval: 1000