🔢

bin/scratch.ts

@@ -0,0 +1,23 @@
+#!/usr/bin/env node
+import 'dotenv/config'
+
+import { gracefulExit } from 'exit-hook'
+import restoreCursor from 'restore-cursor'
+
+import type * as types from '@/types.js'
+
+/**
+ * Scratch for quick testing.
+ */
+async function main() {
+  restoreCursor()
+
+  return gracefulExit(0)
+}
+
+try {
+  await main()
+} catch (err) {
+  console.error('unexpected error', err)
+  gracefulExit(1)
+}

package.json

@@ -45,6 +45,7 @@
   },
   "dependencies": {
     "@dexaai/dexter": "^2.0.0",
+    "chalk": "^5.3.0",
     "delay": "^6.0.0",
     "dotenv": "^16.4.5",
     "execa": "^8.0.1",
@@ -55,6 +56,7 @@
     "p-map": "^7.0.2",
     "p-retry": "^6.2.0",
     "p-throttle": "^6.1.0",
+    "restore-cursor": "^5.0.0",
     "tiny-invariant": "^1.3.3",
     "type-fest": "^4.16.0",
     "zod": "^3.23.3"

pnpm-lock.yaml

@@ -8,6 +8,9 @@ dependencies:
   '@dexaai/dexter':
     specifier: ^2.0.0
     version: 2.0.2
+  chalk:
+    specifier: ^5.3.0
+    version: 5.3.0
   delay:
     specifier: ^6.0.0
     version: 6.0.0
@@ -38,6 +41,9 @@ dependencies:
   p-throttle:
     specifier: ^6.1.0
     version: 6.1.0
+  restore-cursor:
+    specifier: ^5.0.0
+    version: 5.0.0
   tiny-invariant:
     specifier: ^1.3.3
     version: 1.3.3
@@ -1659,7 +1665,6 @@ packages:
   /chalk@5.3.0:
     resolution: {integrity: sha512-dLitG79d+GV1Nb/VYcCDFivJeK1hiukt9QjRNVOsUtTy1rR1YJsmpGGTZ3qJos+uw7WmWF4wUwBd9jxjocFC2w==}
     engines: {node: ^12.17.0 || ^14.13 || >=16.0.0}
-    dev: true
 
   /chardet@0.7.0:
     resolution: {integrity: sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA==}
@@ -5170,6 +5175,14 @@ packages:
       signal-exit: 3.0.7
     dev: true
 
+  /restore-cursor@5.0.0:
+    resolution: {integrity: sha512-Hp93f349DvdEqJFHiPyzNzVjT7lDDFtQJWRotQVQNl3CHr4j7oMHStQB9UH/CJSHTrevAZXFvomgzy8lXjrK0w==}
+    engines: {node: '>=18'}
+    dependencies:
+      onetime: 6.0.0
+      signal-exit: 4.1.0
+    dev: false
+
   /retry@0.13.1:
     resolution: {integrity: sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==}
     engines: {node: '>= 4'}

src/_utils.ts

@@ -0,0 +1,44 @@
+import 'dotenv/config'
+
+import defaultKy, {
+  type AfterResponseHook,
+  type BeforeRequestHook,
+  type KyInstance
+} from 'ky'
+
+const AGENTIC_TEST_MOCK_HEADER = 'x-agentic-test-mock'
+
+function defaultBeforeRequest(request: Request): Response {
+  return new Response(
+    JSON.stringify({
+      url: request.url,
+      method: request.method,
+      headers: request.headers
+    }),
+    {
+      status: 200,
+      headers: {
+        'Content-Type': 'application/json',
+        [AGENTIC_TEST_MOCK_HEADER]: '1'
+      }
+    }
+  )
+}
+
+export function mockKyInstance(
+  ky: KyInstance = defaultKy,
+  {
+    beforeRequest = defaultBeforeRequest,
+    afterResponse
+  }: {
+    beforeRequest?: BeforeRequestHook
+    afterResponse?: AfterResponseHook
+  } = {}
+): KyInstance {
+  return ky.extend({
+    hooks: {
+      beforeRequest: beforeRequest ? [beforeRequest] : [],
+      afterResponse: afterResponse ? [afterResponse] : []
+    }
+  })
+}

src/config.ts

@@ -1,5 +0,0 @@
-import dotenv from 'dotenv'
-
-import type * as types from './types.js'
-
-dotenv.config()

src/fns.ts

@@ -0,0 +1,124 @@
+import 'reflect-metadata'
+
+import type { z } from 'zod'
+
+import type * as types from './types.js'
+import { FunctionSet } from './function-set.js'
+import { ToolSet } from './tool-set.js'
+import { zodToJsonSchema } from './zod-to-json-schema.js'
+
+export const invocableMetadataKey = Symbol('invocable')
+
+export interface Invocable {
+  name: string
+  description?: string
+  inputSchema?: z.AnyZodObject
+  callback: (args: Record<string, any>) => Promise<any>
+}
+
+export abstract class AIToolsProvider {
+  private _tools?: ToolSet
+  private _functions?: FunctionSet
+
+  get namespace() {
+    return this.constructor.name
+  }
+
+  get tools(): ToolSet {
+    if (!this._tools) {
+      this._tools = ToolSet.fromFunctionSet(this.functions)
+    }
+
+    return this._tools
+  }
+
+  get functions(): FunctionSet {
+    if (!this._functions) {
+      const invocables = getInvocables(this)
+      const functions = invocables.map(getFunctionSpec)
+      this._functions = new FunctionSet(functions)
+    }
+
+    return this._functions
+  }
+}
+
+export function getFunctionSpec(invocable: Invocable): types.AIFunctionSpec {
+  const { name, description, inputSchema } = invocable
+
+  return {
+    name,
+    description,
+    parameters: inputSchema
+      ? zodToJsonSchema(inputSchema)
+      : {
+          type: 'object',
+          properties: {}
+        }
+  }
+}
+
+/**
+ * Constraints:
+ *   - params must be an object, so the underlying function should only expect a
+ *     single parameter
+ *   - for the return value type `T | MaybePromise<T>`, `T` must be serializable
+ *     to JSON
+ */
+export function aiFunction({
+  name,
+  description,
+  inputSchema
+}: {
+  name?: string
+  description?: string
+
+  // params must be an object, so the underlying function should only expect a
+  // single parameter
+  inputSchema?: z.AnyZodObject
+}) {
+  return function (
+    target: object,
+    propertyKey: string,
+    descriptor: PropertyDescriptor
+  ) {
+    const existingInvocables = getPrivateInvocables(target)
+
+    existingInvocables.push({
+      propertyKey,
+      description,
+      name,
+      inputSchema
+    })
+
+    setPrivateInvocables(target, existingInvocables)
+
+    return descriptor.get ?? descriptor.value
+  }
+}
+
+export function getInvocables(target: object): Invocable[] {
+  const invocables = getPrivateInvocables(target)
+  const namespace = target.constructor.name
+
+  return invocables.map((invocable) => ({
+    ...invocable,
+    name: invocable.name ?? `${namespace}_${invocable.propertyKey}`,
+    callback: (target as any)[invocable.propertyKey].bind(target)
+  }))
+}
+
+interface PrivateInvocable {
+  propertyKey: string
+  name?: string
+  description?: string
+  inputSchema?: z.AnyZodObject
+}
+
+function getPrivateInvocables(target: object): PrivateInvocable[] {
+  return Reflect.getMetadata(invocableMetadataKey, target) ?? []
+}
+
+function setPrivateInvocables(target: object, invocables: PrivateInvocable[]) {
+  Reflect.defineMetadata(invocableMetadataKey, invocables, target)
+}

src/function-set.ts

@@ -0,0 +1,70 @@
+import type { ToolSet } from './tool-set.js'
+import type * as types from './types.ts'
+
+export class FunctionSet implements Iterable<types.AIFunctionSpec> {
+  protected _map: Map<string, types.AIFunctionSpec>
+
+  constructor(functions?: readonly types.AIFunctionSpec[] | null) {
+    this._map = new Map(functions ? functions.map((fn) => [fn.name, fn]) : null)
+  }
+
+  get size(): number {
+    return this._map.size
+  }
+
+  add(fn: types.AIFunctionSpec): this {
+    this._map.set(fn.name, fn)
+    return this
+  }
+
+  get(name: string): types.AIFunctionSpec | undefined {
+    return this._map.get(name)
+  }
+
+  set(name: string, fn: types.AIFunctionSpec): this {
+    this._map.set(name, fn)
+    return this
+  }
+
+  has(name: string): boolean {
+    return this._map.has(name)
+  }
+
+  clear(): void {
+    this._map.clear()
+  }
+
+  delete(name: string): boolean {
+    return this._map.delete(name)
+  }
+
+  pick(...keys: string[]): FunctionSet {
+    const keysToIncludeSet = new Set(keys)
+    return new FunctionSet(
+      Array.from(this).filter((fn) => keysToIncludeSet.has(fn.name))
+    )
+  }
+
+  omit(...keys: string[]): FunctionSet {
+    const keysToExcludeSet = new Set(keys)
+    return new FunctionSet(
+      Array.from(this).filter((fn) => !keysToExcludeSet.has(fn.name))
+    )
+  }
+
+  get entries(): IterableIterator<types.AIFunctionSpec> {
+    return this._map.values()
+  }
+
+  [Symbol.iterator](): Iterator<types.AIFunctionSpec> {
+    return this.entries
+  }
+
+  static fromToolSet(toolSet: ToolSet): FunctionSet {
+    return new FunctionSet(
+      Array.from(toolSet)
+        .filter((tool) => tool.type === 'function')
+        .map((tool) => tool.function)
+    )
+  }
+}

src/index.ts

@@ -1,2 +1,7 @@
+export * from './fns.js'
+export * from './function-set.js'
+export * from './parse-structured-output.js'
+export * from './services/index.js'
+export * from './tool-set.js'
 export type * from './types.js'
 export * from './utils.js'

src/services/clearbit.ts

@@ -1,368 +1,372 @@
 import defaultKy from 'ky'
 import pThrottle from 'p-throttle'
 
-import type * as types from '../types.js'
+import type { DeepNullable } from '../types.js'
 import { assert, delay, getEnv, throttleKy } from '../utils.js'
 
+// Only allow 20 clearbit API requests per 60s
 const clearbitAPIThrottle = pThrottle({
   limit: 20,
   interval: 60 * 1000,
   strict: true
 })
 
-export interface CompanyEnrichmentOptions {
-  domain: string
-  webhook_url?: string
-  company_name?: string
-  linkedin?: string
-  twitter?: string
-  facebook?: string
-}
+export namespace clearbit {
+  export interface CompanyEnrichmentOptions {
+    domain: string
+    webhook_url?: string
+    company_name?: string
+    linkedin?: string
+    twitter?: string
+    facebook?: string
+  }
 
-type CompanyNullableProps = {
-  name: string
-  legalName: string
-  domain: string
-  domainAliases: string[]
-  site: {
-    phoneNumbers: string[]
-    emailAddresses: string[]
-  }
-  category: {
-    sector: string
-    industryGroup: string
-    industry: string
-    subIndustry: string
-    gicsCode: string
-    sicCode: string
-    sic4Codes: string[]
-    naicsCode: string
-    naics6Codes: string[]
-    naics6Codes2022: string[]
-  }
-  tags: string[]
-  description: string
-  foundedYear: number
-  location: string
-  timeZone: string
-  utcOffset: number
-  geo: {
-    streetNumber: string
-    streetName: string
-    subPremise: string
-    streetAddress: string
-    city: string
-    postalCode: string
-    state: string
-    stateCode: string
-    country: string
-    countryCode: string
-    lat: number
-    lng: number
-  }
-  logo: string
-  facebook: {
-    handle: string
-    likes: number
-  }
-  linkedin: {
-    handle: string
-  }
-  twitter: {
-    handle: string
-    id: string
-    bio: string
-    followers: number
-    following: number
+  export type CompanyNullableProps = {
+    name: string
+    legalName: string
+    domain: string
+    domainAliases: string[]
+    site: {
+      phoneNumbers: string[]
+      emailAddresses: string[]
+    }
+    category: {
+      sector: string
+      industryGroup: string
+      industry: string
+      subIndustry: string
+      gicsCode: string
+      sicCode: string
+      sic4Codes: string[]
+      naicsCode: string
+      naics6Codes: string[]
+      naics6Codes2022: string[]
+    }
+    tags: string[]
+    description: string
+    foundedYear: number
     location: string
+    timeZone: string
+    utcOffset: number
+    geo: {
+      streetNumber: string
+      streetName: string
+      subPremise: string
+      streetAddress: string
+      city: string
+      postalCode: string
+      state: string
+      stateCode: string
+      country: string
+      countryCode: string
+      lat: number
+      lng: number
+    }
+    logo: string
+    facebook: {
+      handle: string
+      likes: number
+    }
+    linkedin: {
+      handle: string
+    }
+    twitter: {
+      handle: string
+      id: string
+      bio: string
+      followers: number
+      following: number
+      location: string
+      site: string
+      avatar: string
+    }
+    crunchbase: {
+      handle: string
+    }
+    emailProvider: boolean
+    type: string
+    ticker: string
+    identifiers: {
+      usEIN: string
+      usCIK: string
+    }
+    phone: string
+    metrics: {
+      alexaUsRank: number
+      alexaGlobalRank: number
+      trafficRank: string
+      employees: number
+      employeesRange: string
+      marketCap: string
+      raised: number
+      annualRevenue: string
+      estimatedAnnualRevenue: string
+      fiscalYearEnd: string
+    }
+    indexedAt: string
+    tech: string[]
+    techCategories: string[]
+    parent: {
+      domain: string
+    }
+    ultimateParent: {
+      domain: string
+    }
+  }
+
+  export type EmailLookupResponse = DeepNullable<{
+    id: string
+    name: {
+      fullName: string
+      givenName: string
+      familyName: string
+    }
+    email: string
+    location: string
+    timeZone: string
+    utcOffset: number
+    geo: {
+      city: string
+      state: string
+      stateCode: string
+      country: string
+      countryCode: string
+      lat: number
+      lng: number
+    }
+    bio: string
     site: string
     avatar: string
-  }
-  crunchbase: {
-    handle: string
-  }
-  emailProvider: boolean
-  type: string
-  ticker: string
-  identifiers: {
-    usEIN: string
-    usCIK: string
-  }
-  phone: string
-  metrics: {
-    alexaUsRank: number
-    alexaGlobalRank: number
-    trafficRank: string
-    employees: number
-    employeesRange: string
-    marketCap: string
-    raised: number
-    annualRevenue: string
-    estimatedAnnualRevenue: string
-    fiscalYearEnd: string
-  }
-  indexedAt: string
-  tech: string[]
-  techCategories: string[]
-  parent: {
-    domain: string
-  }
-  ultimateParent: {
-    domain: string
-  }
-}
+    employment: {
+      domain: string
+      name: string
+      title: string
+      role: string
+      subRole: string
+      seniority: string
+    }
+    facebook: {
+      handle: string
+    }
+    github: {
+      handle: string
+      id: string
+      avatar: string
+      company: string
+      blog: string
+      followers: number
+      following: number
+    }
+    twitter: {
+      handle: string
+      id: string
+      bio: string
+      followers: number
+      following: number
+      statuses: number
+      favorites: number
+      location: string
+      site: string
+      avatar: string
+    }
+    linkedin: {
+      handle: string
+    }
+    googleplus: {
+      handle: null
+    }
+    gravatar: {
+      handle: string
+      urls: {
+        value: string
+        title: string
+      }[]
+      avatar: string
+      avatars: {
+        url: string
+        type: string
+      }[]
+    }
+    fuzzy: boolean
+    emailProvider: boolean
+    indexedAt: string
+    phone: string
+    activeAt: string
+    inactiveAt: string
+  }>
 
-type EmailLookupResponse = DeepNullable<{
-  id: string
-  name: {
-    fullName: string
-    givenName: string
-    familyName: string
+  export type CompanyResponse = {
+    id: string
+  } & DeepNullable<CompanyNullableProps>
+
+  export interface CompanySearchOptions {
+    /**
+     * See clearbit docs: https://dashboard.clearbit.com/docs?shell#discovery-api-tech-queries
+     * Examples:
+     * tech:google_apps
+     * or:(twitter_followers:10000~ type:nonprofit)
+     */
+    query: string
+    page?: number
+    page_size?: number
+    limit?: number
+    sort?: string
   }
-  email: string
-  location: string
-  timeZone: string
-  utcOffset: number
-  geo: {
-    city: string
-    state: string
-    stateCode: string
-    country: string
-    countryCode: string
-    lat: number
-    lng: number
+
+  export interface CompanySearchResponse {
+    total: number
+    page: number
+    results: CompanyResponse[]
   }
-  bio: string
-  site: string
-  avatar: string
-  employment: {
+
+  export interface BasicCompanyResponse {
     domain: string
+    logo: string
     name: string
+  }
+
+  export interface PeopleSearchOptionsV2 {
+    domains?: string[]
+    names?: string[]
+    roles?: string[]
+    seniorities?: string[]
+    titles?: string[]
+    locations?: string[]
+    employees_ranges?: string[]
+    company_tags?: string[]
+    company_tech?: string[]
+    company_types?: string[]
+    industries?: string[]
+    revenue_ranges?: string[]
+    linkedin_profile_handles?: string[]
+    page?: number
+    page_size?: number
+    suppression?: string
+  }
+
+  // Prospector types
+  export interface ProspectorResponseV2 {
+    page: number
+    page_size: number
+    total: number
+    results: PersonAttributesV2[]
+  }
+
+  export interface EmploymentAttributes {
+    company: string
+    domain: string
+    linkedin: string
     title: string
     role: string
     subRole: string
     seniority: string
+    startDate: string
+    endDate: string
+    present: boolean
+    highlight: boolean
   }
-  facebook: {
-    handle: string
+
+  export interface EmailAttributes {
+    address: string
+    type: string
   }
-  github: {
-    handle: string
+
+  export interface PhoneAttributes {
+    number: string
+    type: string
+  }
+
+  interface Name {
+    givenName: string
+    familyName: string
+    fullName: string
+  }
+
+  export type PersonAttributesV2 = {
     id: string
+  } & DeepNullable<{
+    name: Name
     avatar: string
-    company: string
-    blog: string
-    followers: number
-    following: number
-  }
-  twitter: {
-    handle: string
-    id: string
-    bio: string
-    followers: number
-    following: number
-    statuses: number
-    favorites: number
     location: string
-    site: string
-    avatar: string
+    linkedin: string
+    employments: EmploymentAttributes[]
+    emails: EmailAttributes[]
+    phones: PhoneAttributes[]
+  }>
+
+  export type PeopleSearchOptionsV1 = {
+    domain: string
+    role?: string
+    roles?: string[]
+    seniority?: string
+    seniorities?: string[]
+    title?: string
+    titles?: string[]
+    city?: string
+    cities?: string[]
+    state?: string
+    states?: string[]
+    country?: string
+    countries?: string[]
+    name?: string
+    query?: string
+    page?: number
+    page_size?: number
+    suppression?: string
+    email?: boolean
   }
-  linkedin: {
-    handle: string
+
+  export interface Company {
+    name: string
   }
-  googleplus: {
-    handle: null
+
+  export interface PeopleSearchResponseV1 {
+    id: string
+    name: Name
+    title: string
+    role: string
+    subRole: string
+    seniority: string
+    company: Company
+    email: string
+    verified: boolean
+    phone: string
   }
-  gravatar: {
-    handle: string
-    urls: {
-      value: string
-      title: string
-    }[]
-    avatar: string
-    avatars: {
-      url: string
-      type: string
-    }[]
+
+  export interface ProspectorResponseV1 {
+    page: number
+    page_size: number
+    total: number
+    results: PeopleSearchResponseV1[]
   }
-  fuzzy: boolean
-  emailProvider: boolean
-  indexedAt: string
-  phone: string
-  activeAt: string
-  inactiveAt: string
-}>
 
-export type CompanyResponse = {
-  id: string
-} & DeepNullable<CompanyNullableProps>
+  export interface GeoIP {
+    city: string
+    state: string
+    stateCode: string
+    country: string
+    countryCode: string
+  }
 
-export interface CompanySearchOptions {
-  /**
-   * See clearbit docs: https://dashboard.clearbit.com/docs?shell#discovery-api-tech-queries
-   * Examples:
-   * tech:google_apps
-   * or:(twitter_followers:10000~ type:nonprofit)
-   */
-  query: string
-  page?: number
-  page_size?: number
-  limit?: number
-  sort?: string
-}
-
-export interface CompanySearchResponse {
-  total: number
-  page: number
-  results: CompanyResponse[]
-}
-
-export interface BasicCompanyResponse {
-  domain: string
-  logo: string
-  name: string
-}
-
-export interface PeopleSearchOptionsV2 {
-  domains?: string[]
-  names?: string[]
-  roles?: string[]
-  seniorities?: string[]
-  titles?: string[]
-  locations?: string[]
-  employees_ranges?: string[]
-  company_tags?: string[]
-  company_tech?: string[]
-  company_types?: string[]
-  industries?: string[]
-  revenue_ranges?: string[]
-  linkedin_profile_handles?: string[]
-  page?: number
-  page_size?: number
-  suppression?: string
-}
-
-// Prospector types
-export interface ProspectorResponseV2 {
-  page: number
-  page_size: number
-  total: number
-  results: PersonAttributesV2[]
-}
-
-export interface EmploymentAttributes {
-  company: string
-  domain: string
-  linkedin: string
-  title: string
-  role: string
-  subRole: string
-  seniority: string
-  startDate: string
-  endDate: string
-  present: boolean
-  highlight: boolean
-}
-
-export interface EmailAttributes {
-  address: string
-  type: string
-}
-
-export interface PhoneAttributes {
-  number: string
-  type: string
-}
-
-interface Name {
-  givenName: string
-  familyName: string
-  fullName: string
-}
-
-export type PersonAttributesV2 = {
-  id: string
-} & DeepNullable<{
-  name: Name
-  avatar: string
-  location: string
-  linkedin: string
-  employments: EmploymentAttributes[]
-  emails: EmailAttributes[]
-  phones: PhoneAttributes[]
-}>
-
-type PeopleSearchOptionsV1 = {
-  domain: string
-  role?: string
-  roles?: string[]
-  seniority?: string
-  seniorities?: string[]
-  title?: string
-  titles?: string[]
-  city?: string
-  cities?: string[]
-  state?: string
-  states?: string[]
-  country?: string
-  countries?: string[]
-  name?: string
-  query?: string
-  page?: number
-  page_size?: number
-  suppression?: string
-}
-
-interface Company {
-  name: string
-}
-
-interface PeopleSearchResponseV1 {
-  id: string
-  name: Name
-  title: string
-  role: string
-  subRole: string
-  seniority: string
-  company: Company
-  email: string
-  verified: boolean
-  phone: string
-}
-
-export interface ProspectorResponseV1 {
-  page: number
-  page_size: number
-  total: number
-  results: PeopleSearchResponseV1[]
-}
-
-interface GeoIP {
-  city: string
-  state: string
-  stateCode: string
-  country: string
-  countryCode: string
-}
-
-interface CompanyRevealResponse {
-  ip: string
-  fuzzy: boolean
-  domain: string
-  type: string
-  company?: CompanyResponse
-  geoIP: GeoIP
-  confidenceScore: 'very_high' | 'high' | 'medium' | 'low'
-  role: string
-  seniority: string
+  export interface CompanyRevealResponse {
+    ip: string
+    fuzzy: boolean
+    domain: string
+    type: string
+    company?: CompanyResponse
+    geoIP: GeoIP
+    confidenceScore: 'very_high' | 'high' | 'medium' | 'low'
+    role: string
+    seniority: string
+  }
 }
 
 export class ClearbitClient {
-  api: typeof defaultKy
-  apiKey: string
-  _maxPageSize = 100
+  readonly ky: typeof defaultKy
+  readonly apiKey: string
+  readonly _maxPageSize = 100
 
-  static PersonRoles = [
+  static readonly PersonRoles = [
     'communications',
     'customer_service',
     'education',
@@ -383,7 +387,7 @@ export class ClearbitClient {
     'sales'
   ]
 
-  static SenioritiesV2 = [
+  static readonly SenioritiesV2 = [
     'Executive',
     'VP',
     'Owner',
@@ -394,9 +398,9 @@ export class ClearbitClient {
     'Entry'
   ]
 
-  static Seniorities = ['executive', 'director', 'manager']
+  static readonly Seniorities = ['executive', 'director', 'manager']
 
-  static SubIndustries: string[] = [
+  static readonly SubIndustries = [
     'Automotive',
     'Consumer Discretionary',
     'Consumer Goods',
@@ -510,12 +514,11 @@ export class ClearbitClient {
   ]
 
   constructor({
-    apiKey = getEnv('CLEARBIT_KEY'),
+    apiKey = getEnv('CLEARBIT_API_KEY'),
     timeoutMs = 30_000,
     ky = defaultKy
   }: {
     apiKey?: string
-    apiBaseUrl?: string
     timeoutMs?: number
     ky?: typeof defaultKy
   } = {}) {
@@ -525,7 +528,7 @@ export class ClearbitClient {
 
     const throttledKy = throttleKy(ky, clearbitAPIThrottle)
 
-    this.api = throttledKy.extend({
+    this.ky = throttledKy.extend({
       timeout: timeoutMs,
       headers: {
         Authorization: `Basic ${Buffer.from(`${apiKey}:`).toString('base64')}`
@@ -533,33 +536,33 @@ export class ClearbitClient {
     })
   }
 
-  async companyEnrichment(options: CompanyEnrichmentOptions) {
-    return this.api
+  async companyEnrichment(options: clearbit.CompanyEnrichmentOptions) {
+    return this.ky
       .get('https://company-stream.clearbit.com/v2/companies/find', {
         searchParams: { ...options }
       })
-      .json<CompanyResponse>()
+      .json<clearbit.CompanyResponse>()
       .catch((_) => undefined)
   }
 
-  async companySearch(options: CompanySearchOptions) {
-    return this.api
+  async companySearch(options: clearbit.CompanySearchOptions) {
+    return this.ky
       .get('https://discovery.clearbit.com/v1/companies/search', {
         searchParams: { ...options }
       })
-      .json<CompanySearchResponse>()
+      .json<clearbit.CompanySearchResponse>()
   }
 
   async companyAutocomplete(name: string) {
-    return this.api
+    return this.ky
       .get('https://autocomplete.clearbit.com/v1/companies/suggest', {
         searchParams: { query: name }
       })
-      .json<BasicCompanyResponse[]>()
+      .json<clearbit.BasicCompanyResponse[]>()
   }
 
-  async prospectorPeopleV2(options: PeopleSearchOptionsV2) {
-    return this.api
+  async prospectorPeopleV2(options: clearbit.PeopleSearchOptionsV2) {
+    return this.ky
       .get('https://prospector.clearbit.com/v2/people/search', {
         // @ts-expect-error location is a string[] and searchparams shows a TS error heres
         searchParams: {
@@ -570,42 +573,42 @@ export class ClearbitClient {
           )
         }
       })
-      .json<ProspectorResponseV2>()
+      .json<clearbit.ProspectorResponseV2>()
   }
 
-  async prospectorPeopleV1(options: PeopleSearchOptionsV1, loadEmail = false) {
-    return this.api
+  async prospectorPeopleV1(options: clearbit.PeopleSearchOptionsV1) {
+    return this.ky
       .get('https://prospector.clearbit.com/v1/people/search', {
         // @ts-expect-error location is a string[] and searchparams shows a TS error heres
         searchParams: {
+          email: false,
           ...options,
           page_size: Math.min(
             this._maxPageSize,
             options.page_size || this._maxPageSize
-          ),
-          email: loadEmail
+          )
         }
       })
-      .json<ProspectorResponseV1>()
+      .json<clearbit.ProspectorResponseV1>()
   }
 
   // TODO Status code = 202 means the response was queued.
-  // Implement webhook when needed. The polling works well, in most cases we need to try
-  // again once to get a 200 response.
+  // Implement webhook when needed. The polling works well, in most cases we need
+  // to try again once to get a 200 response.
   async emailLookup({
     email,
-    maxRetries
+    maxRetries = 2
   }: {
     email: string
     maxRetries?: number
-  }) {
+  }): Promise<clearbit.EmailLookupResponse> {
     const url = 'https://person.clearbit.com/v2/people/find'
-    let response = await this.api.get(url, {
+    let response = await this.ky.get(url, {
       searchParams: { email }
     })
 
     if (response.status !== 202 || !maxRetries) {
-      return response.json<EmailLookupResponse>()
+      return response.json<clearbit.EmailLookupResponse>()
     }
 
     if (maxRetries && response.status === 202) {
@@ -614,37 +617,39 @@ export class ClearbitClient {
       while (running && count < maxRetries) {
         console.log(`Email Lookup was queued, retry ${count + 1}.`)
         await delay(1000)
-        response = await this.api.get(url, {
+        response = await this.ky.get(url, {
           searchParams: { email }
         })
         count++
         running = response.status === 202
       }
-      return response.json<EmailLookupResponse>()
+      return response.json<clearbit.EmailLookupResponse>()
     }
+
+    throw new Error('clearbit email lookup error 202', { cause: response })
   }
 
   async nameToDomain(name: string) {
-    return this.api
+    return this.ky
       .get('https://company.clearbit.com/v1/domains/find', {
         searchParams: { name }
       })
-      .json<BasicCompanyResponse>()
+      .json<clearbit.BasicCompanyResponse>()
       .catch((_) => undefined)
   }
 
   async revealCompanyFromIp(ip: string) {
-    return this.api
+    return this.ky
       .get('https://reveal.clearbit.com/v1/companies/find', {
         searchParams: { ip }
       })
-      .json<CompanyRevealResponse>()
+      .json<clearbit.CompanyRevealResponse>()
       .catch((_) => undefined)
   }
 
   static filterEmploymentProspectorV2(
     companyName: string,
-    employments: Array<DeepNullable<EmploymentAttributes> | null> | null
+    employments: Array<DeepNullable<clearbit.EmploymentAttributes> | null> | null
   ) {
     if (employments && employments.length > 0) {
       // We filter by employment endDate because some people could have multiple jobs at the same time.

src/services/diffbot.ts

@@ -0,0 +1,453 @@
+import defaultKy, { type KyInstance } from 'ky'
+import { AbortError } from 'p-retry'
+import pThrottle from 'p-throttle'
+
+import { assert, getEnv, throttleKy } from '../utils.js'
+
+const diffbotAPIThrottle = pThrottle({
+  limit: 5,
+  interval: 1000,
+  strict: true
+})
+
+export namespace diffbot {
+  export const API_BASE_URL = 'https://api.diffbot.com'
+  export const KNOWLEDGE_GRAPH_API_BASE_URL = 'https://kg.diffbot.com'
+
+  export interface DiffbotExtractOptions {
+    /** Specify optional fields to be returned from any fully-extracted pages, e.g.: &fields=querystring,links. See available fields within each API's individual documentation pages.
+     * @see https://docs.diffbot.com/reference/extract-optional-fields
+     */
+    fields?: string[]
+
+    /** (*Undocumented*) Pass paging=false to disable automatic concatenation of multiple-page articles. (By default, Diffbot will concatenate up to 20 pages of a single article.) */
+    paging?: boolean
+
+    /** Pass discussion=false to disable automatic extraction of comments or reviews from pages identified as articles or products. This will not affect pages identified as discussions. */
+    discussion?: boolean
+
+    /** Sets a value in milliseconds to wait for the retrieval/fetch of content from the requested URL. The default timeout for the third-party response is 30 seconds (30000). */
+    timeout?: number
+
+    /** Used to specify the IP address of a custom proxy that will be used to fetch the target page, instead of Diffbot's default IPs/proxies. (Ex: &proxy=168.212.226.204) */
+    proxy?: string
+
+    /** Used to specify the authentication parameters that will be used with the proxy specified in the &proxy parameter. (Ex: &proxyAuth=username:password) */
+    proxyAuth?: string
+
+    /** `none` will instruct Extract to not use proxies, even if proxies have been enabled for this particular URL globally. */
+    useProxy?: string
+
+    /** @see https://docs.diffbot.com/reference/extract-custom-javascript */
+    customJs?: string
+
+    /** @see https://docs.diffbot.com/reference/extract-custom-headers */
+    customHeaders?: Record<string, string>
+  }
+
+  export interface DiffbotExtractAnalyzeOptions extends DiffbotExtractOptions {
+    /** Web page URL of the analyze to process */
+    url: string
+
+    /** By default the Analyze API will fully extract all pages that match an existing Automatic API -- articles, products or image pages. Set mode to a specific page-type (e.g., mode=article) to extract content only from that specific page-type. All other pages will simply return the default Analyze fields. */
+    mode?: string
+
+    /** Force any non-extracted pages (those with a type of "other") through a specific API. For example, to route all "other" pages through the Article API, pass &fallback=article. Pages that utilize this functionality will return a fallbackType field at the top-level of the response and a originalType field within each extracted object, both of which will indicate the fallback API used. */
+    fallback?: string
+  }
+
+  export interface DiffbotExtractArticleOptions extends DiffbotExtractOptions {
+    /** Web page URL of the analyze to process */
+    url: string
+
+    /** Set the maximum number of automatically-generated tags to return. By default a maximum of ten tags will be returned. */
+    maxTags?: number
+
+    /** Set the minimum relevance score of tags to return, between 0.0 and 1.0. By default only tags with a score equal to or above 0.5 will be returned. */
+    tagConfidence?: number
+
+    /** Used to request the output of the Diffbot Natural Language API in the field naturalLanguage. Example: &naturalLanguage=entities,facts,categories,sentiment. */
+    naturalLanguage?: string[]
+  }
+
+  export interface DiffbotExtractResponse {
+    request: DiffbotRequest
+    objects: DiffbotObject[]
+  }
+
+  export type DiffbotExtractArticleResponse = DiffbotExtractResponse
+
+  export interface DiffbotExtractAnalyzeResponse
+    extends DiffbotExtractResponse {
+    type: string
+    title: string
+    humanLanguage: string
+  }
+
+  export interface DiffbotObject {
+    date: string
+    sentiment: number
+    images: DiffbotImage[]
+    author: string
+    estimatedDate: string
+    publisherRegion: string
+    icon: string
+    diffbotUri: string
+    siteName: string
+    type: string
+    title: string
+    tags: DiffbotTag[]
+    publisherCountry: string
+    humanLanguage: string
+    authorUrl: string
+    pageUrl: string
+    html: string
+    text: string
+    categories?: DiffbotCategory[]
+    authors: DiffbotAuthor[]
+    breadcrumb?: DiffbotBreadcrumb[]
+    items?: DiffbotListItem[]
+    meta?: any
+  }
+
+  interface DiffbotListItem {
+    title: string
+    link: string
+    summary: string
+    image?: string
+  }
+
+  interface DiffbotAuthor {
+    name: string
+    link: string
+  }
+
+  interface DiffbotCategory {
+    score: number
+    name: string
+    id: string
+  }
+
+  export interface DiffbotBreadcrumb {
+    link: string
+    name: string
+  }
+
+  interface DiffbotImage {
+    url: string
+    diffbotUri: string
+
+    naturalWidth: number
+    naturalHeight: number
+    width: number
+    height: number
+
+    isCached?: boolean
+    primary?: boolean
+  }
+
+  interface DiffbotTag {
+    score: number
+    sentiment: number
+    count: number
+    label: string
+    uri: string
+    rdfTypes: string[]
+  }
+
+  interface DiffbotRequest {
+    pageUrl: string
+    api: string
+    version: number
+  }
+
+  export interface Image {
+    naturalHeight: number
+    diffbotUri: string
+    url: string
+    naturalWidth: number
+    primary: boolean
+  }
+
+  export interface Tag {
+    score: number
+    sentiment: number
+    count: number
+    label: string
+    uri: string
+    rdfTypes: string[]
+  }
+
+  export interface Request {
+    pageUrl: string
+    api: string
+    version: number
+  }
+
+  export interface DiffbotKnowledgeGraphSearchOptions {
+    type?: 'query' | 'text' | 'queryTextFallback' | 'crawl'
+    query: string
+    col?: string
+    from?: number
+    size?: number
+
+    // NOTE: we only support `json`, so these options are not needed
+    // We can always convert from json to another format if needed.
+    // format?: 'json' | 'jsonl' | 'csv' | 'xls' | 'xlsx'
+    // exportspec?: string
+    // exportseparator?: string
+    // exportfile?: string
+
+    filter?: string
+    jsonmode?: 'extended' | 'id'
+    nonCanonicalFacts?: boolean
+    noDedupArticles?: boolean
+    cluster?: 'all' | 'best' | 'dedupe'
+    report?: boolean
+  }
+
+  export interface DiffbotKnowledgeGraphEnhanceOptions {
+    type: 'Person' | 'Organization'
+
+    id?: string
+    name?: string
+    url?: string
+    phone?: string
+    email?: string
+    employer?: string
+    title?: string
+    school?: string
+    location?: string
+    ip?: string
+    customId?: string
+
+    size?: number
+    threshold?: number
+
+    refresh?: boolean
+    search?: boolean
+    useCache?: boolean
+
+    filter?: string
+    jsonmode?: 'extended' | 'id'
+    nonCanonicalFacts?: boolean
+  }
+
+  export interface DiffbotKnowledgeGraphResponse {
+    data: DiffbotKnowledgeGraphNode[]
+    version: number
+    hits: number
+    results: number
+    kgversion: string
+    diffbot_type: string
+    facet?: boolean
+    errors?: any[]
+  }
+
+  export interface DiffbotKnowledgeGraphNode {
+    score: number
+    esscore?: number
+    entity: DiffbotKnowledgeGraphEntity
+    entity_ctx: any
+    errors: string[]
+    callbackQuery: string
+    upperBound: number
+    lowerBound: number
+    count: number
+    value: string
+    uri: string
+  }
+
+  export interface DiffbotKnowledgeGraphEntity {
+    id: string
+    diffbotUri: string
+    type?: string
+    name: string
+    images: DiffbotImage[]
+    origins: string[]
+    nbOrigins?: number
+
+    gender?: DiffbotGender
+    githubUri?: string
+    importance?: number
+    description?: string
+    homepageUri?: string
+    allNames?: string[]
+    skills?: DiffbotSkill[]
+    crawlTimestamp?: number
+    summary?: string
+    image?: string
+    types?: string[]
+    nbIncomingEdges?: number
+    allUris?: string[]
+    employments?: DiffbotEmployment[]
+    locations?: DiffbotLocation[]
+    location?: DiffbotLocation
+    allOriginHashes?: string[]
+    nameDetail?: DiffbotNameDetail
+  }
+
+  interface DiffbotEmployment {
+    employer: Entity
+  }
+
+  interface Entity {
+    image?: string
+    types?: string[]
+    name: string
+    diffbotUri?: string
+    type: EntityType
+    summary?: string
+  }
+
+  type EntityType = 'Organization' | 'Place'
+
+  interface DiffbotGender {
+    normalizedValue: string
+  }
+
+  interface DiffbotLocation {
+    country: Entity
+    isCurrent: boolean
+    address: string
+    latitude: number
+    precision: number
+    surfaceForm: string
+    region: Entity
+    longitude: number
+  }
+
+  interface DiffbotNameDetail {
+    firstName: string
+    lastName: string
+  }
+
+  interface DiffbotSkill {
+    name: string
+    diffbotUri: string
+  }
+}
+
+export class DiffbotClient {
+  readonly ky: KyInstance
+  readonly kyKnowledgeGraph: typeof defaultKy
+
+  readonly apiKey: string
+  readonly apiBaseUrl: string
+  readonly apiKnowledgeGraphBaseUrl: string
+
+  constructor({
+    apiKey = getEnv('DIFFBOT_API_KEY'),
+    apiBaseUrl = diffbot.API_BASE_URL,
+    apiKnowledgeGraphBaseUrl = diffbot.KNOWLEDGE_GRAPH_API_BASE_URL,
+    timeoutMs = 30_000,
+    ky = defaultKy
+  }: {
+    apiKey?: string
+    apiBaseUrl?: string
+    apiKnowledgeGraphBaseUrl?: string
+    timeoutMs?: number
+    ky?: KyInstance
+  } = {}) {
+    assert(apiKey, `Error DiffbotClient missing required "apiKey"`)
+
+    this.apiKey = apiKey
+    this.apiBaseUrl = apiBaseUrl
+    this.apiKnowledgeGraphBaseUrl = apiKnowledgeGraphBaseUrl
+
+    const throttledKy = throttleKy(ky, diffbotAPIThrottle)
+
+    this.ky = throttledKy.extend({
+      prefixUrl: apiBaseUrl,
+      timeout: timeoutMs
+    })
+
+    this.kyKnowledgeGraph = throttledKy.extend({
+      prefixUrl: apiKnowledgeGraphBaseUrl,
+      timeout: timeoutMs
+    })
+  }
+
+  protected async _extract<
+    T extends diffbot.DiffbotExtractResponse = diffbot.DiffbotExtractResponse
+  >(endpoint: string, options: diffbot.DiffbotExtractOptions): Promise<T> {
+    const { customJs, customHeaders, ...rest } = options
+    const searchParams: Record<string, any> = {
+      ...rest,
+      token: this.apiKey
+    }
+    const headers = {
+      ...Object.fromEntries(
+        [['X-Forward-X-Evaluate', customJs]].filter(([, value]) => value)
+      ),
+      ...customHeaders
+    }
+
+    for (const [key, value] of Object.entries(rest)) {
+      if (Array.isArray(value)) {
+        searchParams[key] = value.join(',')
+      }
+    }
+
+    // TODO
+    const { url } = searchParams
+    if (url) {
+      const parsedUrl = new URL(url)
+      if (parsedUrl.hostname.includes('theguardian.com')) {
+        throw new AbortError(
+          `Diffbot does not support URLs from domain "${parsedUrl.hostname}"`
+        )
+      }
+    }
+
+    // console.log(`DiffbotClient._extract: ${endpoint}`, searchParams)
+
+    return this.ky
+      .get(endpoint, {
+        searchParams,
+        headers,
+        retry: 1
+      })
+      .json<T>()
+  }
+
+  async extractAnalyze(options: diffbot.DiffbotExtractAnalyzeOptions) {
+    return this._extract<diffbot.DiffbotExtractAnalyzeResponse>(
+      'v3/analyze',
+      options
+    )
+  }
+
+  async extractArticle(options: diffbot.DiffbotExtractArticleOptions) {
+    return this._extract<diffbot.DiffbotExtractArticleResponse>(
+      'v3/article',
+      options
+    )
+  }
+
+  async knowledgeGraphSearch(
+    options: diffbot.DiffbotKnowledgeGraphSearchOptions
+  ) {
+    return this.kyKnowledgeGraph
+      .get('kg/v3/dql', {
+        searchParams: {
+          ...options,
+          token: this.apiKey
+        }
+      })
+      .json<diffbot.DiffbotKnowledgeGraphResponse>()
+  }
+
+  async knowledgeGraphEnhance(
+    options: diffbot.DiffbotKnowledgeGraphEnhanceOptions
+  ) {
+    return this.kyKnowledgeGraph
+      .get('kg/v3/enhance', {
+        searchParams: {
+          ...options,
+          token: this.apiKey
+        }
+      })
+      .json<diffbot.DiffbotKnowledgeGraphResponse>()
+  }
+}

src/services/index.ts

@@ -0,0 +1,9 @@
+export * from './clearbit.js'
+export * from './dexa-client.js'
+export * from './diffbot.js'
+export * from './openai-client.js'
+export * from './scraper-client.js'
+export * from './serpapi.js'
+export * from './serper.js'
+export * from './twitter-client.js'
+export * from './weather.js'

src/services/openai-client.ts

@@ -1,5 +1 @@
-import '../config.js'
-
-import { OpenAI } from 'openai'
-
-export const openaiClient = new OpenAI()
+export * from 'openai'

src/services/scraper-client.ts

@@ -1,5 +1,7 @@
 import defaultKy, { type KyInstance } from 'ky'
 
+import { assert, getEnv } from '../utils.js'
+
 export type ScrapeResult = {
   author: string
   byline: string
@@ -34,16 +36,14 @@ export class ScraperClient {
   readonly ky: KyInstance
 
   constructor({
-    apiBaseUrl = process.env.SCRAPER_API_BASE_URL,
+    apiBaseUrl = getEnv('SCRAPER_API_BASE_URL'),
     ky = defaultKy
   }: {
     apiKey?: string
     apiBaseUrl?: string
     ky?: KyInstance
   } = {}) {
-    if (!apiBaseUrl) {
-      throw new Error('SCRAPER_API_BASE_URL is required')
-    }
+    assert(apiBaseUrl, 'SCRAPER_API_BASE_URL is required')
 
     this.apiBaseUrl = apiBaseUrl
     this.ky = ky.extend({ prefixUrl: this.apiBaseUrl })

src/services/serpapi.ts

@@ -10,6 +10,8 @@ import { getEnv } from '../utils.js'
  */
 
 export namespace serpapi {
+  export const BASE_URL = 'https://serpapi.com'
+
   export type BaseResponse<P = Record<string | number | symbol, never>> = {
     search_metadata: {
       id: string
@@ -365,20 +367,20 @@ export namespace serpapi {
     twitter_results?: TwitterResults
   }
 
-  interface TwitterResults {
+  export interface TwitterResults {
     title: string
     link: string
     displayed_link: string
     tweets: Tweet[]
   }
 
-  interface Tweet {
+  export interface Tweet {
     link: string
     snippet: string
     published_date: string
   }
 
-  interface AnswerBox {
+  export interface AnswerBox {
     type: string
     title: string
     link: string
@@ -391,7 +393,7 @@ export namespace serpapi {
     cached_page_link: string
   }
 
-  interface InlineImage {
+  export interface InlineImage {
     link: string
     source: string
     thumbnail: string
@@ -400,21 +402,21 @@ export namespace serpapi {
     title?: string
   }
 
-  interface InlinePeopleAlsoSearchFor {
+  export interface InlinePeopleAlsoSearchFor {
     title: string
     items: SearchItem[]
     see_more_link: string
     see_more_serpapi_link: string
   }
 
-  interface SearchItem {
+  export interface SearchItem {
     name: string
     image: string
     link: string
     serpapi_link: string
   }
 
-  interface KnowledgeGraph {
+  export interface KnowledgeGraph {
     type: string
     kgmid: string
     knowledge_graph_search_link: string
@@ -429,7 +431,7 @@ export namespace serpapi {
     list: { [key: string]: string[] }
   }
 
-  interface Button {
+  export interface Button {
     text: string
     subtitle: string
     title: string
@@ -445,34 +447,34 @@ export namespace serpapi {
     list?: string[]
   }
 
-  interface HeaderImage {
+  export interface HeaderImage {
     image: string
     source: string
   }
 
-  interface Source {
+  export interface Source {
     name: string
     link: string
   }
 
-  interface LocalMap {
+  export interface LocalMap {
     link: string
     image: string
     gps_coordinates: LocalMapGpsCoordinates
   }
 
-  interface LocalMapGpsCoordinates {
+  export interface LocalMapGpsCoordinates {
     latitude: number
     longitude: number
     altitude: number
   }
 
-  interface LocalResults {
+  export interface LocalResults {
     places: Place[]
     more_locations_link: string
   }
 
-  interface Place {
+  export interface Place {
     position: number
     title: string
     rating?: number
@@ -489,18 +491,18 @@ export namespace serpapi {
     hours?: string
   }
 
-  interface PlaceGpsCoordinates {
+  export interface PlaceGpsCoordinates {
     latitude: number
     longitude: number
   }
 
-  interface ServiceOptions {
+  export interface ServiceOptions {
     dine_in?: boolean
     takeout: boolean
     no_delivery?: boolean
   }
 
-  interface OrganicResult {
+  export interface OrganicResult {
     position: number
     title: string
     link: string
@@ -520,24 +522,24 @@ export namespace serpapi {
     related_questions?: OrganicResultRelatedQuestion[]
   }
 
-  interface AboutThisResult {
+  export interface AboutThisResult {
     keywords: string[]
     languages: string[]
     regions: string[]
   }
 
-  interface OrganicResultRelatedQuestion {
+  export interface OrganicResultRelatedQuestion {
     question: string
     snippet: string
     snippet_links: SnippetLink[]
   }
 
-  interface SnippetLink {
+  export interface SnippetLink {
     text: string
     link: string
   }
 
-  interface RelatedResult {
+  export interface RelatedResult {
     position: number
     title: string
     link: string
@@ -548,32 +550,32 @@ export namespace serpapi {
     cached_page_link: string
   }
 
-  interface RichSnippet {
+  export interface RichSnippet {
     bottom: Bottom
   }
 
-  interface Bottom {
+  export interface Bottom {
     extensions?: string[]
     questions?: string[]
   }
 
-  interface Sitelinks {
+  export interface Sitelinks {
     inline: Inline[]
   }
 
-  interface Inline {
+  export interface Inline {
     title: string
     link: string
   }
 
-  interface Pagination {
+  export interface Pagination {
     current: number
     next: string
     other_pages: { [key: string]: string }
     next_link?: string
   }
 
-  interface SearchResultRelatedQuestion {
+  export interface SearchResultRelatedQuestion {
     question: string
     snippet: string
     title: string
@@ -585,12 +587,12 @@ export namespace serpapi {
     date?: string
   }
 
-  interface RelatedSearch {
+  export interface RelatedSearch {
     query: string
     link: string
   }
 
-  interface SearchInformation {
+  export interface SearchInformation {
     organic_results_state: string
     query_displayed: string
     total_results: number
@@ -598,14 +600,14 @@ export namespace serpapi {
     menu_items: MenuItem[]
   }
 
-  interface MenuItem {
+  export interface MenuItem {
     position: number
     title: string
     link: string
     serpapi_link?: string
   }
 
-  interface SearchMetadata {
+  export interface SearchMetadata {
     id: string
     status: string
     json_endpoint: string
@@ -616,7 +618,7 @@ export namespace serpapi {
     total_time_taken: number
   }
 
-  interface SearchParameters {
+  export interface SearchParameters {
     engine: string
     q: string
     google_domain: string
@@ -630,8 +632,6 @@ export namespace serpapi {
     apiBaseUrl?: string
     ky?: KyInstance
   }
-
-  export const BASE_URL = 'https://serpapi.com'
 }
 
 /**
@@ -670,7 +670,7 @@ export class SerpAPIClient extends AIToolsProvider {
     name: 'serpapiGoogleSearch',
     description:
       'Uses Google Search to return the most relevant web pages for a given query. Can also be used to find up-to-date news and information about many topics.',
-    schema: z.object({
+    inputSchema: z.object({
       q: z.string().describe('search query'),
       num: z.number().int().positive().default(5).optional()
     })

src/services/serper.ts

@@ -2,7 +2,7 @@ import defaultKy, { type KyInstance } from 'ky'
 import { z } from 'zod'
 
 import { aiFunction, AIToolsProvider } from '../fns.js'
-import { getEnv } from '../utils.js'
+import { assert, getEnv } from '../utils.js'
 
 export namespace serper {
   export const BASE_URL = 'https://google.serper.dev'
@@ -199,10 +199,10 @@ export namespace serper {
  * @see https://serper.dev
  */
 export class SerperClient extends AIToolsProvider {
-  protected api: KyInstance
-  protected apiKey: string
-  protected apiBaseUrl: string
-  protected params: Omit<Partial<serper.SearchParams>, 'q'>
+  readonly ky: KyInstance
+  readonly apiKey: string
+  readonly apiBaseUrl: string
+  readonly params: Omit<Partial<serper.SearchParams>, 'q'>
 
   constructor({
     apiKey = getEnv('SERPER_API_KEY'),
@@ -210,11 +210,10 @@ export class SerperClient extends AIToolsProvider {
     ky = defaultKy,
     ...params
   }: serper.ClientOptions = {}) {
-    if (!apiKey) {
-      throw new Error(
-        `SerperClient missing required "apiKey" (defaults to "SERPER_API_KEY" env var)`
-      )
-    }
+    assert(
+      apiKey,
+      `SerperClient missing required "apiKey" (defaults to "SERPER_API_KEY" env var)`
+    )
 
     super()
 
@@ -222,7 +221,7 @@ export class SerperClient extends AIToolsProvider {
     this.apiBaseUrl = apiBaseUrl
     this.params = params
 
-    this.api = ky.extend({
+    this.ky = ky.extend({
       prefixUrl: this.apiBaseUrl,
       headers: {
         'X-API-KEY': this.apiKey
@@ -234,7 +233,7 @@ export class SerperClient extends AIToolsProvider {
     name: 'serperGoogleSearch',
     description:
       'Uses Google Search to return the most relevant web pages for a given query. Can also be used to find up-to-date news and information about many topics.',
-    schema: serper.SearchParamsSchema
+    inputSchema: serper.SearchParamsSchema
   })
   async search(queryOrOpts: string | serper.SearchParams) {
     return this._fetch<serper.SearchResponse>('search', queryOrOpts)
@@ -263,12 +262,12 @@ export class SerperClient extends AIToolsProvider {
   protected async _fetch<T extends serper.Response>(
     endpoint: string,
     queryOrOpts: string | serper.SearchParams
-  ) {
+  ): Promise<T> {
     const params = {
       ...this.params,
       ...(typeof queryOrOpts === 'string' ? { q: queryOrOpts } : queryOrOpts)
     }
 
-    return this.api.post(endpoint, { json: params }).json<T>()
+    return this.ky.post(endpoint, { json: params }).json<T>()
   }
 }

src/services/twitter-client.ts

@@ -1,8 +1,7 @@
 import { Nango } from '@nangohq/node'
 import { auth, Client as TwitterClient } from 'twitter-api-sdk'
 
-import * as config from '../config.js'
-import { assert } from '../utils.js'
+import { assert, getEnv } from '../utils.js'
 
 // The Twitter+Nango client auth connection key
 const nangoTwitterProviderConfigKey = 'twitter-v2'
@@ -23,7 +22,7 @@ let _nango: Nango | null = null
 
 function getNango(): Nango {
   if (!_nango) {
-    const secretKey = process.env.NANGO_SECRET_KEY?.trim()
+    const secretKey = getEnv('NANGO_SECRET_KEY')?.trim()
     if (!secretKey) {
       throw new Error(`Missing required "NANGO_SECRET_KEY"`)
     }
@@ -35,12 +34,18 @@ function getNango(): Nango {
 }
 
 async function getTwitterAuth({
-  scopes = defaultRequiredTwitterOAuthScopes
-}: { scopes?: Set<string> } = {}): Promise<auth.OAuth2User> {
+  scopes,
+  nangoConnectionId,
+  nangoCallbackUrl
+}: {
+  scopes: Set<string>
+  nangoConnectionId: string
+  nangoCallbackUrl: string
+}): Promise<auth.OAuth2User> {
   const nango = getNango()
   const connection = await nango.getConnection(
     nangoTwitterProviderConfigKey,
-    config.nangoConnectionId
+    nangoConnectionId
   )
 
   // console.debug('nango twitter connection', connection)
@@ -65,11 +70,9 @@ async function getTwitterAuth({
 
   if (missingScopes.size > 0) {
     throw new Error(
-      `Nango connection ${
-        config.nangoConnectionId
-      } is missing required OAuth scopes: ${[...missingScopes.values()].join(
-        ', '
-      )}`
+      `Nango connection ${nangoConnectionId} is missing required OAuth scopes: ${[
+        ...missingScopes.values()
+      ].join(', ')}`
     )
   }
 
@@ -78,17 +81,30 @@ async function getTwitterAuth({
 
   return new auth.OAuth2User({
     client_id: twitterClientId,
-    callback: config.nangoCallbackUrl,
+    callback: nangoCallbackUrl,
     scopes: [...scopes.values()] as any,
     token
   })
 }
 
 export async function getTwitterClient({
-  scopes = defaultRequiredTwitterOAuthScopes
-}: { scopes?: Set<string> } = {}): Promise<TwitterClient> {
+  scopes = defaultRequiredTwitterOAuthScopes,
+  nangoConnectionId = getEnv('NANGO_CONNECTION_ID'),
+  nangoCallbackUrl = getEnv('NANGO_CALLBACK_URL')
+}: {
+  scopes?: Set<string>
+  nangoConnectionId?: string
+  nangoCallbackUrl?: string
+} = {}): Promise<TwitterClient> {
+  assert(nangoConnectionId, 'twitter client missing nangoConnectionId')
+  assert(nangoCallbackUrl, 'twitter client missing nangoCallbackUrl')
+
   // NOTE: Nango handles refreshing the oauth access token for us
-  const twitterAuth = await getTwitterAuth({ scopes })
+  const twitterAuth = await getTwitterAuth({
+    scopes,
+    nangoConnectionId,
+    nangoCallbackUrl
+  })
 
   // Twitter API v2 using OAuth 2.0
   return new TwitterClient(twitterAuth)

src/services/weather.ts

@@ -75,9 +75,9 @@ export namespace weatherapi {
 }
 
 export class WeatherClient extends AIToolsProvider {
-  protected api: KyInstance
-  protected apiKey: string
-  protected apiBaseUrl: string
+  readonly ky: KyInstance
+  readonly apiKey: string
+  readonly apiBaseUrl: string
 
   constructor({
     apiKey = getEnv('WEATHER_API_KEY'),
@@ -94,13 +94,13 @@ export class WeatherClient extends AIToolsProvider {
     this.apiKey = apiKey
     this.apiBaseUrl = apiBaseUrl
 
-    this.api = ky.extend({ prefixUrl: apiBaseUrl })
+    this.ky = ky.extend({ prefixUrl: apiBaseUrl })
   }
 
   @aiFunction({
     name: 'getCurrentWeather',
     description: 'Gets info about the current weather at a given location.',
-    schema: z.object({
+    inputSchema: z.object({
       q: z
         .string()
         .describe(
@@ -114,7 +114,7 @@ export class WeatherClient extends AIToolsProvider {
         ? { q: queryOrOptions }
         : queryOrOptions
 
-    return this.api
+    return this.ky
       .get('current.json', {
         searchParams: {
           key: this.apiKey,
@@ -128,7 +128,7 @@ export class WeatherClient extends AIToolsProvider {
     const options =
       typeof ipOrOptions === 'string' ? { q: ipOrOptions } : ipOrOptions
 
-    return this.api
+    return this.ky
       .get('ip.json', {
         searchParams: {
           key: this.apiKey,

src/stringify-for-model.test.ts

@@ -0,0 +1,22 @@
+import { describe, expect, it } from 'vitest'
+
+import { stringifyForModel } from './stringify-for-model.js'
+
+describe('stringifyForModel', () => {
+  it('handles basic objects', () => {
+    const input = {
+      foo: 'bar',
+      nala: ['is', 'cute'],
+      kittens: null,
+      cats: undefined,
+      paws: 4.3
+    }
+    const result = stringifyForModel(input)
+    expect(result).toEqual(JSON.stringify(input, null))
+  })
+
+  it('handles empty input', () => {
+    const result = stringifyForModel()
+    expect(result).toEqual('')
+  })
+})

src/stringify-for-model.ts

@@ -0,0 +1,18 @@
+import type { Jsonifiable } from 'type-fest'
+
+/**
+ * Stringifies a JSON value in a way that's optimized for use with LLM prompts.
+ *
+ * This is intended to be used with `function` and `tool` arguments and responses.
+ */
+export function stringifyForModel(jsonObject?: Jsonifiable): string {
+  if (jsonObject === undefined) {
+    return ''
+  }
+
+  if (typeof jsonObject === 'string') {
+    return jsonObject
+  }
+
+  return JSON.stringify(jsonObject, null, 0)
+}

src/tool-set.ts

@@ -0,0 +1,85 @@
+import type * as types from './types.ts'
+import { FunctionSet } from './function-set.js'
+
+export class ToolSet implements Iterable<types.AIToolSpec> {
+  protected _map: Map<string, types.AIToolSpec>
+
+  constructor(tools?: readonly types.AIToolSpec[] | null) {
+    this._map = new Map(
+      tools ? tools.map((tool) => [tool.function.name, tool]) : null
+    )
+  }
+
+  get size(): number {
+    return this._map.size
+  }
+
+  add(tool: types.AIToolSpec): this {
+    this._map.set(tool.function.name, tool)
+    return this
+  }
+
+  get(name: string): types.AIToolSpec | undefined {
+    return this._map.get(name)
+  }
+
+  set(name: string, tool: types.AIToolSpec): this {
+    this._map.set(name, tool)
+    return this
+  }
+
+  has(name: string): boolean {
+    return this._map.has(name)
+  }
+
+  clear(): void {
+    this._map.clear()
+  }
+
+  delete(name: string): boolean {
+    return this._map.delete(name)
+  }
+
+  pick(...keys: string[]): ToolSet {
+    const keysToIncludeSet = new Set(keys)
+    return new ToolSet(
+      Array.from(this).filter((tool) =>
+        keysToIncludeSet.has(tool.function.name)
+      )
+    )
+  }
+
+  omit(...keys: string[]): ToolSet {
+    const keysToExcludeSet = new Set(keys)
+    return new ToolSet(
+      Array.from(this).filter(
+        (tool) => !keysToExcludeSet.has(tool.function.name)
+      )
+    )
+  }
+
+  get entries(): IterableIterator<types.AIToolSpec> {
+    return this._map.values()
+  }
+
+  [Symbol.iterator](): Iterator<types.AIToolSpec> {
+    return this.entries
+  }
+
+  static fromFunctionSet(functionSet: FunctionSet): ToolSet {
+    return new ToolSet(
+      Array.from(functionSet).map((fn) => ({
+        type: 'function' as const,
+        function: fn
+      }))
+    )
+  }
+
+  static fromFunctionSpecs(functionSpecs: types.AIFunctionSpec[]): ToolSet {
+    return ToolSet.fromFunctionSet(new FunctionSet(functionSpecs))
+  }
+
+  static fromToolSpecs(toolSpecs: types.AIToolSpec[]): ToolSet {
+    return new ToolSet(toolSpecs)
+  }
+}

src/types.ts

@@ -1,2 +1,16 @@
 export type { KyInstance } from 'ky'
 export type { ThrottledFunction } from 'p-throttle'
+
+// TODO
+export type DeepNullable<T> = T | null
+
+export interface AIFunctionSpec {
+  name: string
+  description?: string
+  parameters: Record<string, unknown>
+}
+
+export interface AIToolSpec {
+  type: 'function'
+  function: AIFunctionSpec
+}

src/utils.test.ts

@@ -1,6 +1,9 @@
+import ky from 'ky'
+import pThrottle from 'p-throttle'
 import { expect, test } from 'vitest'
 
-import { omit, pick } from './utils.js'
+import { mockKyInstance } from './_utils.js'
+import { omit, pick, throttleKy } from './utils.js'
 
 test('pick', () => {
   expect(pick({ a: 1, b: 2, c: 3 }, 'a', 'c')).toEqual({ a: 1, c: 3 })
@@ -15,3 +18,33 @@ test('omit', () => {
     omit({ a: { b: 'foo' }, d: -1, foo: null } as any, 'b', 'foo')
   ).toEqual({ a: { b: 'foo' }, d: -1 })
 })
+
+test('throttleKy should rate-limit requests to ky properly', async () => {
+  // TODO: set timeout
+
+  const interval = 1000
+  const throttle = pThrottle({
+    limit: 1,
+    interval,
+    strict: true
+  })
+
+  const ky2 = mockKyInstance(throttleKy(ky, throttle))
+
+  const url = 'https://httpbin.org/get'
+
+  for (let i = 0; i < 10; i++) {
+    const before = Date.now()
+    const res = await ky2.get(url)
+    const after = Date.now()
+
+    const duration = after - before
+    // console.log(duration, res.status)
+    expect(res.status).toBe(200)
+
+    // leave a bit of wiggle room for the interval
+    if (i > 0) {
+      expect(duration >= interval - interval / 5).toBeTruthy()
+    }
+  }
+})

src/zod-to-json-schema.test.ts

@@ -0,0 +1,63 @@
+import { describe, expect, it } from 'vitest'
+import { z } from 'zod'
+
+import { zodToJsonSchema } from './zod-to-json-schema.js'
+
+describe('zodToJsonSchema', () => {
+  it('handles basic objects', () => {
+    const params = zodToJsonSchema(
+      z.object({
+        name: z.string().min(1).describe('Name of the person'),
+        age: z.number().int().optional().describe('Age in years')
+      })
+    )
+
+    expect(params).toEqual({
+      additionalProperties: false,
+      type: 'object',
+      required: ['name'],
+      properties: {
+        name: {
+          type: 'string',
+          description: 'Name of the person',
+          minLength: 1
+        },
+        age: {
+          type: 'integer',
+          description: 'Age in years'
+        }
+      }
+    })
+  })
+
+  it('handles enums and unions', () => {
+    const params = zodToJsonSchema(
+      z.object({
+        name: z.string().min(1).describe('Name of the person'),
+        sexEnum: z.enum(['male', 'female']),
+        sexUnion: z.union([z.literal('male'), z.literal('female')])
+      })
+    )
+
+    expect(params).toEqual({
+      additionalProperties: false,
+      type: 'object',
+      required: ['name', 'sexEnum', 'sexUnion'],
+      properties: {
+        name: {
+          type: 'string',
+          description: 'Name of the person',
+          minLength: 1
+        },
+        sexEnum: {
+          type: 'string',
+          enum: ['male', 'female']
+        },
+        sexUnion: {
+          type: 'string',
+          enum: ['male', 'female']
+        }
+      }
+    })
+  })
+})

src/zod-to-json-schema.ts

@@ -0,0 +1,17 @@
+import type { z } from 'zod'
+import { zodToJsonSchema as zodToJsonSchemaImpl } from 'zod-to-json-schema'
+
+import { omit } from './utils.js'
+
+/** Generate a JSON Schema from a Zod schema. */
+export function zodToJsonSchema(schema: z.ZodType): Record<string, unknown> {
+  return omit(
+    zodToJsonSchemaImpl(schema, { $refStrategy: 'none' }),
+    '$schema',
+    'default',
+    'definitions',
+    'description',
+    'markdownDescription',
+    'additionalProperties'
+  )
+}

tsconfig.json

@@ -12,6 +12,9 @@
     "useDefineForClassFields": true,
     "jsx": "preserve",
 
+    "experimentalDecorators": true,
+    "emitDecoratorMetadata": true,
+
     "strict": true,
     "noUncheckedIndexedAccess": true,
     "forceConsistentCasingInFileNames": true,