🐺

package.json

@@ -44,7 +44,6 @@
     "test:unit": "vitest run"
   },
   "dependencies": {
-    "@dexaai/dexter": "^2.0.0",
     "@nangohq/node": "^0.39.30",
     "chalk": "^5.3.0",
     "delay": "^6.0.0",
@@ -52,12 +51,9 @@
     "exit-hook": "^4.0.0",
     "jsonrepair": "^3.6.1",
     "ky": "^1.2.4",
-    "openai": "^4.47.1",
     "p-map": "^7.0.2",
     "p-retry": "^6.2.0",
     "p-throttle": "^6.1.0",
-    "proxycurl-js-linkedin-profile-scraper": "^1.0.2",
-    "reflect-metadata": "^0.2.2",
     "restore-cursor": "^5.0.0",
     "tiny-invariant": "^1.3.3",
     "twitter-api-sdk": "^1.2.1",

readme.md

@@ -17,6 +17,34 @@
 
 **Coming soon**
 
+## Services
+
+- Clearbit
+- Dexa
+- Diffbot
+- Proxycurl
+- SerpAPI
+- Serper
+- Twitter
+- WeatherAPI
+
+## TODO
+
+- core
+  - company schema
+  - person schema
+  - database
+  - move out to a separate project
+    - agentic
+    - walter
+- services
+  - Exa
+  - Firecrawl
+  - Unstructured
+  - pull from [langchain](https://github.com/langchain-ai/langchainjs/tree/main/langchain)
+  - pull from other libs
+  - pull from [nango](https://docs.nango.dev/integrations/overview)
+
 ## License
 
 PROPRIETARY © [Travis Fischer](https://twitter.com/transitive_bs)

src/fns.ts

@@ -1,4 +1,4 @@
-import 'reflect-metadata'
+import './symbol-polyfill.js'
 
 import type { z } from 'zod'
 
@@ -34,7 +34,7 @@ export abstract class AIToolsProvider {
       const functions = invocables.map((invocable) => ({
         ...invocable,
         name: invocable.name ?? `${namespace}_${invocable.propertyKey}`,
-        callback: (target as any)[invocable.propertyKey].bind(target)
+        callback: (this as any)[invocable.propertyKey].bind(target)
       }))
 
       const functions = invocables.map(getFunctionSpec)
@@ -88,18 +88,14 @@ export function aiFunction<
     context: ClassMethodDecoratorContext<
       This,
       (this: This, ...args: Args) => Return
-    > & {
+    >
-      readonly metadata: {
-        invocables: Invocable[]
-      }
-    }
   ) => {
     const methodName = String(context.name)
     if (!context.metadata.invocables) {
       context.metadata.invocables = []
     }
 
-    context.metadata.invocables.push({
+    ;(context.metadata.invocables as Invocable[]).push({
       name: name ?? methodName,
       description,
       inputSchema,

src/services/clearbit-client.ts

@@ -1,7 +1,7 @@
 import defaultKy from 'ky'
 import pThrottle from 'p-throttle'
 
-import type { DeepNullable } from '../types.js'
+import type { DeepNullable, KyInstance } from '../types.js'
 import { assert, delay, getEnv, throttleKy } from '../utils.js'
 
 // Only allow 20 clearbit API requests per 60s
@@ -362,7 +362,7 @@ export namespace clearbit {
 }
 
 export class ClearbitClient {
-  readonly ky: typeof defaultKy
+  readonly ky: KyInstance
   readonly apiKey: string
   readonly _maxPageSize = 100
 
@@ -516,17 +516,19 @@ export class ClearbitClient {
   constructor({
     apiKey = getEnv('CLEARBIT_API_KEY'),
     timeoutMs = 30_000,
+    throttle = true,
     ky = defaultKy
   }: {
     apiKey?: string
     timeoutMs?: number
-    ky?: typeof defaultKy
+    throttle?: boolean
+    ky?: KyInstance
   } = {}) {
     assert(apiKey, 'Error clearbit client missing required "apiKey"')
 
     this.apiKey = apiKey
 
-    const throttledKy = throttleKy(ky, clearbitAPIThrottle)
+    const throttledKy = throttle ? throttleKy(ky, clearbitAPIThrottle) : ky
 
     this.ky = throttledKy.extend({
       timeout: timeoutMs,
@@ -652,7 +654,8 @@ export class ClearbitClient {
     employments: Array<DeepNullable<clearbit.EmploymentAttributes> | null> | null
   ) {
     if (employments && employments.length > 0) {
-      // We filter by employment endDate because some people could have multiple jobs at the same time.
+      // We filter by employment endDate because some people could have multiple
+      // jobs at the same time.
       // Here we want to filter by people that actively works at a specific company.
       return employments
         .filter((item) => !item?.endDate)
@@ -660,6 +663,7 @@ export class ClearbitClient {
           item?.company?.toLowerCase().includes(companyName.toLowerCase())
         )
     }
+
     return false
   }
 }

src/services/dexa-client.ts

@@ -1,6 +1,6 @@
-import { type Prompt } from '@dexaai/dexter'
 import defaultKy, { type KyInstance } from 'ky'
 
+import type * as types from '../types.js'
 import { assert, getEnv } from '../utils.js'
 
 export class DexaClient {
@@ -11,20 +11,22 @@ export class DexaClient {
   constructor({
     apiKey = getEnv('DEXA_API_KEY'),
     apiBaseUrl = getEnv('DEXA_API_BASE_URL') ?? 'https://dexa.ai',
+    timeoutMs = 60_000,
     ky = defaultKy
   }: {
     apiKey?: string
     apiBaseUrl?: string
+    timeoutMs?: number
     ky?: KyInstance
   } = {}) {
-    assert(apiKey, 'DEXA_API_KEY is required')
+    assert(apiKey, 'DexaClient missing required "apiKey"')
 
     this.apiKey = apiKey
     this.apiBaseUrl = apiBaseUrl
-    this.ky = ky.extend({ prefixUrl: this.apiBaseUrl, timeout: 60_000 })
+    this.ky = ky.extend({ prefixUrl: this.apiBaseUrl, timeout: timeoutMs })
   }
 
-  async askDexa({ messages }: { messages: Prompt.Msg[] }) {
+  async askDexa({ messages }: { messages: types.Msg[] }) {
     return this.ky
       .post('api/ask-dexa', {
         json: {

src/services/diffbot-client.ts

@@ -330,7 +330,7 @@ export namespace diffbot {
 
 export class DiffbotClient {
   readonly ky: KyInstance
-  readonly kyKnowledgeGraph: typeof defaultKy
+  readonly kyKnowledgeGraph: KyInstance
 
   readonly apiKey: string
   readonly apiBaseUrl: string
@@ -341,21 +341,23 @@ export class DiffbotClient {
     apiBaseUrl = diffbot.API_BASE_URL,
     apiKnowledgeGraphBaseUrl = diffbot.KNOWLEDGE_GRAPH_API_BASE_URL,
     timeoutMs = 30_000,
+    throttle = true,
     ky = defaultKy
   }: {
     apiKey?: string
     apiBaseUrl?: string
     apiKnowledgeGraphBaseUrl?: string
     timeoutMs?: number
+    throttle?: boolean
     ky?: KyInstance
   } = {}) {
-    assert(apiKey, `Error DiffbotClient missing required "apiKey"`)
+    assert(apiKey, `DiffbotClient missing required "apiKey"`)
 
     this.apiKey = apiKey
     this.apiBaseUrl = apiBaseUrl
     this.apiKnowledgeGraphBaseUrl = apiKnowledgeGraphBaseUrl
 
-    const throttledKy = throttleKy(ky, diffbotAPIThrottle)
+    const throttledKy = throttle ? throttleKy(ky, diffbotAPIThrottle) : ky
 
     this.ky = throttledKy.extend({
       prefixUrl: apiBaseUrl,

src/services/exa-client.ts

@@ -0,0 +1,264 @@
+import defaultKy, { type KyInstance } from 'ky'
+
+import { assert, getEnv } from '../utils.js'
+
+export namespace exa {
+  /**
+   * Search options for performing a search query.
+   */
+  export type BaseSearchOptions = {
+    /** Number of search results to return. Default 10. Max 10 for basic plans. */
+    numResults?: number
+    /** List of domains to include in the search. */
+    includeDomains?: string[]
+    /** List of domains to exclude in the search. */
+    excludeDomains?: string[]
+    /** Start date for results based on crawl date. */
+    startCrawlDate?: string
+    /** End date for results based on crawl date. */
+    endCrawlDate?: string
+    /** Start date for results based on published date. */
+    startPublishedDate?: string
+    /** End date for results based on published date. */
+    endPublishedDate?: string
+    /** A data category to focus on, with higher comprehensivity and data cleanliness. Currently, the only category is company. */
+    category?: string
+  }
+
+  /**
+   * Search options for performing a search query.
+   */
+  export type RegularSearchOptions = BaseSearchOptions & {
+    /** If true, converts query to a Metaphor query. */
+    useAutoprompt?: boolean
+    /** Type of search, 'keyword' or 'neural'. */
+    type?: string
+  }
+
+  /**
+   * Options for finding similar links.
+   */
+  export type FindSimilarOptions = BaseSearchOptions & {
+    /** If true, excludes links from the base domain of the input. */
+    excludeSourceDomain?: boolean
+  }
+
+  /**
+   * Search options for performing a search query.
+   */
+  export type ContentsOptions = {
+    /** Options for retrieving text contents. */
+    text?: TextContentsOptions | true
+    /** Options for retrieving highlights. */
+    highlights?: HighlightsContentsOptions | true
+  }
+
+  /**
+   * Options for retrieving text from page.
+   */
+  export type TextContentsOptions = {
+    /** The maximum number of characters to return. */
+    maxCharacters?: number
+    /** If true, includes HTML tags in the returned text. Default: false */
+    includeHtmlTags?: boolean
+  }
+
+  /**
+   * Options for retrieving highlights from page.
+   * @typedef {Object} HighlightsContentsOptions
+   */
+  export type HighlightsContentsOptions = {
+    /** The query string to use for highlights search. */
+    query?: string
+    /** The number of sentences to return for each highlight. */
+    numSentences?: number
+    /** The number of highlights to return for each URL. */
+    highlightsPerUrl?: number
+  }
+
+  export type TextResponse = {
+    /** Text from page */
+    text: string
+  }
+
+  export type HighlightsResponse = {
+    /** The highlights as an array of strings. */
+    highlights: string[]
+    /** The corresponding scores as an array of floats, 0 to 1 */
+    highlightScores: number[]
+  }
+
+  export type Default<T extends {}, U> = [keyof T] extends [never] ? U : T
+
+  /**
+   * Depending on 'ContentsOptions', this yields either a 'TextResponse',
+   * a 'HighlightsResponse', both, or an empty object.
+   */
+  export type ContentsResultComponent<T extends ContentsOptions> = Default<
+    (T['text'] extends object | true ? TextResponse : {}) &
+      (T['highlights'] extends object | true ? HighlightsResponse : {}),
+    TextResponse
+  >
+
+  /**
+   * Represents a search result object.
+   */
+  export type SearchResult<T extends ContentsOptions = ContentsOptions> = {
+    /** The title of the search result. */
+    title: string | null
+    /** The URL of the search result. */
+    url: string
+    /** The estimated creation date of the content. */
+    publishedDate?: string
+    /** The author of the content, if available. */
+    author?: string
+    /** Similarity score between the query/url and the result. */
+    score?: number
+    /** The temporary ID for the document. */
+    id: string
+  } & ContentsResultComponent<T>
+
+  /**
+   * Represents a search response object.
+   */
+  export type SearchResponse<T extends ContentsOptions = ContentsOptions> = {
+    /** The list of search results. */
+    results: SearchResult<T>[]
+    /** The autoprompt string, if applicable. */
+    autopromptString?: string
+  }
+}
+
+export class ExaClient {
+  readonly apiKey: string
+  readonly apiBaseUrl: string
+  readonly ky: KyInstance
+
+  constructor({
+    apiKey = getEnv('EXA_API_KEY'),
+    apiBaseUrl = getEnv('EXA_API_BASE_URL') ?? 'https://api.exa.ai',
+    ky = defaultKy
+  }: {
+    apiKey?: string
+    apiBaseUrl?: string
+    ky?: KyInstance
+  } = {}) {
+    assert(apiKey, 'ExaClient missing required "apiKey"')
+
+    this.apiKey = apiKey
+    this.apiBaseUrl = apiBaseUrl
+    this.ky = ky.extend({
+      prefixUrl: this.apiBaseUrl,
+      headers: {
+        'x-api-key': apiKey
+      }
+    })
+  }
+
+  async search(query: string, options?: exa.RegularSearchOptions) {
+    return this.ky
+      .post('search', { json: { ...options, query } })
+      .json<exa.SearchResponse>()
+  }
+
+  /**
+   * Performs a search with a Exa prompt-engineered query and returns the
+   * contents of the documents.
+   *
+   * @param {string} query - The query string.
+   */
+  async searchAndContents<T extends exa.ContentsOptions = exa.ContentsOptions>(
+    query: string,
+    options?: exa.RegularSearchOptions & T
+  ) {
+    const { text, highlights, ...rest } = options || {}
+
+    return this.ky
+      .post('search', {
+        json: {
+          query,
+          contents:
+            !text && !highlights
+              ? { text: true }
+              : {
+                  ...(text ? { text } : {}),
+                  ...(highlights ? { highlights } : {})
+                },
+          ...rest
+        }
+      })
+      .json<exa.SearchResponse<T>>()
+  }
+
+  /**
+   * Finds similar links to the provided URL.
+   *
+   * @param {string} url - The URL for which to find similar links.
+   */
+  async findSimilar(url: string, options?: exa.FindSimilarOptions) {
+    return this.ky
+      .post('findSimilar', { json: { url, ...options } })
+      .json<exa.SearchResponse>()
+  }
+
+  /**
+   * Finds similar links to the provided URL and returns the contents of the
+   * documents.
+   *
+   * @param {string} url - The URL for which to find similar links.
+   */
+  async findSimilarAndContents<
+    T extends exa.ContentsOptions = exa.ContentsOptions
+  >(url: string, options?: exa.FindSimilarOptions & T) {
+    const { text, highlights, ...rest } = options || {}
+
+    return this.ky
+      .post('findSimilar', {
+        json: {
+          url,
+          contents:
+            !text && !highlights
+              ? { text: true }
+              : {
+                  ...(text ? { text } : {}),
+                  ...(highlights ? { highlights } : {})
+                },
+          ...rest
+        }
+      })
+      .json<exa.SearchResponse<T>>()
+  }
+
+  /**
+   * Retrieves contents of documents based on a list of document IDs.
+   *
+   * @param {string | string[] | SearchResult[]} ids - An array of document IDs.
+   */
+  async getContents<T extends exa.ContentsOptions>(
+    ids: string | string[] | exa.SearchResult[],
+    options?: T
+  ) {
+    let requestIds: string[]
+
+    if (typeof ids === 'string') {
+      requestIds = [ids]
+    } else if (typeof ids[0] === 'string') {
+      requestIds = ids as string[]
+    } else {
+      requestIds = (ids as exa.SearchResult[]).map((result) => result.id)
+    }
+
+    if (ids.length === 0) {
+      throw new Error('Must provide at least one ID')
+    }
+
+    return this.ky
+      .post('contents', {
+        json: {
+          ids: requestIds,
+          ...options
+        }
+      })
+      .json<exa.SearchResponse<T>>()
+  }
+}

src/services/index.ts

@@ -1,7 +1,7 @@
 export * from './clearbit-client.js'
 export * from './dexa-client.js'
 export * from './diffbot-client.js'
-export * from './openai-client.js'
+export * from './exa-client.js'
 export * from './proxycurl-client.js'
 export * from './scraper-client.js'
 export * from './serpapi-client.js'

src/services/openai-client.ts

@@ -1 +0,0 @@
-export * from 'openai'

src/services/proxycurl-client.ts

@@ -2003,6 +2003,7 @@ export namespace proxycurl {
 export class ProxycurlClient {
   readonly ky: KyInstance
   readonly apiKey: string
+  readonly apiBaseUrl: string
 
   constructor({
     apiKey = getEnv('PROXYCURL_API_KEY'),
@@ -2014,9 +2015,11 @@ export class ProxycurlClient {
     apiBaseUrl?: string
     ky?: KyInstance
   } = {}) {
-    assert(apiKey, 'Error ProxycurlClient missing required "apiKey"')
+    assert(apiKey, 'ProxycurlClient missing required "apiKey"')
+    assert(apiBaseUrl, 'ProxycurlClient missing required "apiBaseUrl"')
 
     this.apiKey = apiKey
+    this.apiBaseUrl = apiBaseUrl
 
     this.ky = ky.extend({
       prefixUrl: apiBaseUrl,

src/services/scraper-client.ts

@@ -45,7 +45,7 @@ export class ScraperClient {
     apiBaseUrl?: string
     ky?: KyInstance
   } = {}) {
-    assert(apiBaseUrl, 'SCRAPER_API_BASE_URL is required')
+    assert(apiBaseUrl, 'ScraperClient apiBaseUrl is required')
 
     this.apiBaseUrl = apiBaseUrl
     this.ky = ky.extend({ prefixUrl: this.apiBaseUrl })

src/services/serpapi-client.ts

@@ -667,7 +667,7 @@ export class SerpAPIClient extends AIToolsProvider {
   }
 
   @aiFunction({
-    name: 'serpapiGoogleSearch',
+    name: 'serpapi_google_search',
     description:
       'Uses Google Search to return the most relevant web pages for a given query. Can also be used to find up-to-date news and information about many topics.',
     inputSchema: z.object({

src/services/serper-client.ts

@@ -230,10 +230,12 @@ export class SerperClient extends AIToolsProvider {
   }
 
   @aiFunction({
-    name: 'serperGoogleSearch',
+    name: 'serper_google_search',
     description:
       'Uses Google Search to return the most relevant web pages for a given query. Can also be used to find up-to-date news and information about many topics.',
-    inputSchema: serper.SearchParamsSchema
+    inputSchema: serper.SearchParamsSchema.pick({
+      q: true
+    })
   })
   async search(queryOrOpts: string | serper.SearchParams) {
     return this._fetch<serper.SearchResponse>('search', queryOrOpts)

src/services/weather-client.ts

@@ -98,7 +98,7 @@ export class WeatherClient extends AIToolsProvider {
   }
 
   @aiFunction({
-    name: 'getCurrentWeather',
+    name: 'get_current_weather',
     description: 'Gets info about the current weather at a given location.',
     inputSchema: z.object({
       q: z

src/stringify-for-model.ts

@@ -2,8 +2,6 @@ import type { Jsonifiable } from 'type-fest'
 
 /**
  * Stringifies a JSON value in a way that's optimized for use with LLM prompts.
- *
- * This is intended to be used with `function` and `tool` arguments and responses.
  */
 export function stringifyForModel(jsonObject?: Jsonifiable): string {
   if (jsonObject === undefined) {

src/types.ts

@@ -21,16 +21,21 @@ export interface AIToolSpec {
 export interface Msg {
   /** The contents of the message. `content` is required for all messages, and may be null for assistant messages with function calls. */
   content: string | null
+
   /** The role of the messages author. One of `system`, `user`, `assistant`, 'tool', or `function`. */
   role: Msg.Role
+
   /** The name and arguments of a function that should be called, as generated by the model. */
   function_call?: Msg.Call.Function
+
   /** The tool calls generated by the model, such as function calls. */
   tool_calls?: Msg.Call.Tool[]
+
   /**
    * Tool call that this message is responding to.
    */
   tool_call_id?: string
+
   /**
    * The name of the author of this message. `name` is required if role is
    * `function`, and it should be the name of the function whose response is in the
@@ -50,6 +55,7 @@ export namespace Msg {
     export type Function = {
       /** The arguments to call the function with, as generated by the model in JSON format. */
       arguments: string
+
       /** The name of the function to call. */
       name: string
     }
@@ -58,8 +64,10 @@ export namespace Msg {
     export type Tool = {
       /** The ID of the tool call. */
       id: string
+
       /** The type of the tool. Currently, only `function` is supported. */
       type: 'function'
+
       /** The function that the model called. */
       function: Call.Function
     }

tsconfig.json

@@ -12,8 +12,8 @@
     "useDefineForClassFields": true,
     "jsx": "preserve",
 
-    "experimentalDecorators": true,
+    // "experimentalDecorators": true,
-    "emitDecoratorMetadata": true,
+    // "emitDecoratorMetadata": true,
 
     "strict": true,
     "noUncheckedIndexedAccess": true,