feat: move withHumanFeedback to a Task method

examples/expert-question-answering.ts

@@ -2,12 +2,7 @@ import 'dotenv/config'
 import { OpenAIClient } from 'openai-fetch'
 import { z } from 'zod'
 
-import {
+import { Agentic, NovuNotificationTool, SerpAPITool } from '@/index'
-  Agentic,
-  NovuNotificationTool,
-  SerpAPITool,
-  withHumanFeedback
-} from '@/index'
 
 async function main() {
   const openai = new OpenAIClient({ apiKey: process.env.OPENAI_API_KEY! })
@@ -15,33 +10,31 @@ async function main() {
 
   const question = 'How do I build a product that people will love?'
 
-  const task = withHumanFeedback(
+  const { metadata } = await agentic
-    agentic
+    .gpt4(
-      .gpt4(
+      `Generate a list of {n} prominent experts that can answer the following question: {{question}}.`
-        `Generate a list of {n} prominent experts that can answer the following question: {{question}}.`
+    )
-      )
+    .tools([new SerpAPITool()])
-      .tools([new SerpAPITool()])
+    .output(
-      .output(
+      z.array(
-        z.array(
-          z.object({
-            name: z.string(),
-            bio: z.string()
-          })
-        )
-      )
-      .input(
         z.object({
-          question: z.string(),
+          name: z.string(),
-          n: z.number().int().default(5)
+          bio: z.string()
         })
-      ),
+      )
-    {
+    )
+    .input(
+      z.object({
+        question: z.string(),
+        n: z.number().int().default(5)
+      })
+    )
+    .withHumanFeedback({
       type: 'selectN'
-    }
+    })
-  )
+    .callWithMetadata({
-  const { metadata } = await task.callWithMetadata({
+      question
-    question
+    })
-  })
 
   if (
     metadata.feedback &&
@@ -78,6 +71,7 @@ async function main() {
 
       ${expert}: ${answer}`
     }, '')
+
     const notifier = new NovuNotificationTool()
     await notifier.call({
       name: 'send-email',

examples/hf0-demo.ts

@@ -6,43 +6,43 @@ import {
   Agentic,
   HumanFeedbackMechanismTwilio,
   SearchAndCrawlTool,
-  WeatherTool,
+  WeatherTool
-  withHumanFeedback
 } from '@/index'
 
 async function main() {
   const openai = new OpenAIClient({ apiKey: process.env.OPENAI_API_KEY! })
-  const agentic = new Agentic({ openai })
+  const agentic = new Agentic({
+    openai,
+    humanFeedbackDefaults: {
+      mechanism: HumanFeedbackMechanismTwilio
+    }
+  })
 
   const topic = process.argv[2] || 'OpenAI'
 
-  const res0 = await withHumanFeedback(
+  const res0 = await agentic
-    agentic
+    .gpt4({
-      .gpt4({
+      messages: [
-        messages: [
+        {
-          {
+          role: 'system',
-            role: 'system',
+          content: `You are a McKinsey analyst who is an expert at writing executive summaries.`
-            content: `You are a McKinsey analyst who is an expert at writing executive summaries.`
+        },
-          },
+        {
-          {
+          role: 'user',
-            role: 'user',
+          content: `What are the 3 most important questions we would need to answer in order to have a thorough understanding of this topic: {{topic}}? Be concise but creative in your questions, and make sure to capture the true essence of the topic.`
-            content: `What are the 3 most important questions we would need to answer in order to have a thorough understanding of this topic: {{topic}}? Be concise but creative in your questions, and make sure to capture the true essence of the topic.`
+        }
-          }
+      ],
-        ],
+      model: 'gpt-4',
-        model: 'gpt-4',
+      temperature: 1.0
-        temperature: 1.0
+    })
+    .input(
+      z.object({
+        topic: z.string()
       })
-      .input(
+    )
-        z.object({
+    .output(z.array(z.string()).describe('question'))
-          topic: z.string()
+    .withHumanFeedback({ type: 'selectN' })
-        })
+    .callWithMetadata({ topic })
-      )
-      .output(z.array(z.string()).describe('question')),
-    {
-      type: 'selectN',
-      mechanism: HumanFeedbackMechanismTwilio
-    }
-  ).callWithMetadata({ topic })
 
   console.log()
   console.log()

scratch/examples/human-feedback-cli.ts

@@ -2,14 +2,13 @@ import 'dotenv/config'
 import { OpenAIClient } from 'openai-fetch'
 import { z } from 'zod'
 
-import { HumanFeedbackMechanismCLI } from '@/human-feedback'
+import { Agentic } from '@/index'
-import { Agentic, withHumanFeedback } from '@/index'
 
 async function main() {
   const openai = new OpenAIClient({ apiKey: process.env.OPENAI_API_KEY! })
   const ai = new Agentic({ openai })
 
-  const topicJokes = ai
+  const out = await ai
     .gpt3(`Tell me {{num}} jokes about {{topic}}`)
     .input(
       z.object({
@@ -19,19 +18,17 @@ async function main() {
     )
     .output(z.array(z.string()))
     .modelParams({ temperature: 0.9 })
+    .withHumanFeedback({
+      type: 'selectN',
+      annotations: false,
+      abort: false,
+      editing: true
+    })
+    .callWithMetadata({
+      topic: 'politicians',
+      num: 5
+    })
 
-  const topicJokesFeedback = withHumanFeedback(topicJokes, {
-    type: 'selectN',
-    annotations: false,
-    abort: false,
-    editing: true,
-    mechanism: HumanFeedbackMechanismCLI
-  })
-
-  const out = await topicJokesFeedback.callWithMetadata({
-    topic: 'politicians',
-    num: 5
-  })
   const feedback = out.metadata.feedback
   console.log(JSON.stringify(feedback, null, 2))
 }

scratch/examples/human-feedback-slack.ts

@@ -2,14 +2,13 @@ import 'dotenv/config'
 import { OpenAIClient } from 'openai-fetch'
 import { z } from 'zod'
 
-import { HumanFeedbackMechanismSlack } from '@/human-feedback'
+import { Agentic, HumanFeedbackMechanismSlack } from '@/index'
-import { Agentic, withHumanFeedback } from '@/index'
 
 async function main() {
   const openai = new OpenAIClient({ apiKey: process.env.OPENAI_API_KEY! })
   const ai = new Agentic({ openai })
 
-  const topicJokes = ai
+  const out = await ai
     .gpt3(`Tell me {{num}} jokes about {{topic}}`)
     .input(
       z.object({
@@ -19,19 +18,18 @@ async function main() {
     )
     .output(z.array(z.string()))
     .modelParams({ temperature: 0.9 })
+    .withHumanFeedback({
+      type: 'selectN',
+      annotations: false,
+      abort: false,
+      editing: true,
+      mechanism: HumanFeedbackMechanismSlack
+    })
+    .callWithMetadata({
+      topic: 'politicians',
+      num: 5
+    })
 
-  const topicJokesFeedback = withHumanFeedback(topicJokes, {
-    type: 'selectN',
-    annotations: false,
-    abort: false,
-    editing: true,
-    mechanism: HumanFeedbackMechanismSlack
-  })
-
-  const out = await topicJokesFeedback.callWithMetadata({
-    topic: 'politicians',
-    num: 5
-  })
   const feedback = out.metadata.feedback
   console.log(JSON.stringify(feedback, null, 2))
 }

scratch/examples/human-feedback-twilio.ts

@@ -2,14 +2,13 @@ import 'dotenv/config'
 import { OpenAIClient } from 'openai-fetch'
 import { z } from 'zod'
 
-import { HumanFeedbackMechanismTwilio } from '@/human-feedback'
+import { Agentic, HumanFeedbackMechanismTwilio } from '@/index'
-import { Agentic, withHumanFeedback } from '@/index'
 
 async function main() {
   const openai = new OpenAIClient({ apiKey: process.env.OPENAI_API_KEY! })
   const ai = new Agentic({ openai })
 
-  const topicJokes = ai
+  const out = await ai
     .gpt3(`Tell me {{num}} jokes about {{topic}}`)
     .input(
       z.object({
@@ -19,19 +18,18 @@ async function main() {
     )
     .output(z.array(z.string()))
     .modelParams({ temperature: 0.9 })
+    .withHumanFeedback({
+      type: 'selectN',
+      annotations: false,
+      abort: false,
+      editing: true,
+      mechanism: HumanFeedbackMechanismTwilio
+    })
+    .callWithMetadata({
+      topic: 'politicians',
+      num: 5
+    })
 
-  const topicJokesFeedback = withHumanFeedback(topicJokes, {
-    type: 'selectN',
-    annotations: false,
-    abort: false,
-    editing: true,
-    mechanism: HumanFeedbackMechanismTwilio
-  })
-
-  const out = await topicJokesFeedback.callWithMetadata({
-    topic: 'politicians',
-    num: 5
-  })
   const feedback = out.metadata.feedback
   console.log(JSON.stringify(feedback, null, 2))
 }

src/human-feedback/feedback.ts

@@ -1,10 +1,7 @@
-import * as types from '@/types'
 import { Agentic } from '@/agentic'
 import { HumanFeedbackDeclineError } from '@/errors'
 import { BaseTask } from '@/task'
 
-import { HumanFeedbackMechanismCLI } from './cli'
-
 /**
  * Actions the user can take in the feedback selection prompt.
  */
@@ -303,48 +300,3 @@ export abstract class HumanFeedbackMechanism<
     return feedback as FeedbackTypeToMetadata<T>
   }
 }
-
-export function withHumanFeedback<
-  TInput extends types.TaskInput,
-  TOutput extends types.TaskOutput,
-  V extends HumanFeedbackType
->(
-  task: BaseTask<TInput, TOutput>,
-  options: HumanFeedbackOptions<V, TOutput> = {}
-) {
-  task = task.clone()
-
-  // Use Object.assign to merge the options, instance defaults, and hard-coded defaults:
-  const finalOptions: HumanFeedbackOptions<V, TOutput> = Object.assign(
-    {
-      type: 'confirm',
-      abort: false,
-      editing: false,
-      annotations: false,
-      timeoutMs: Number.POSITIVE_INFINITY,
-      mechanism: HumanFeedbackMechanismCLI
-    },
-    // Default options from the instance:
-    task.agentic.humanFeedbackDefaults,
-    // User-provided options (override instance defaults):
-    options
-  )
-
-  if (!finalOptions.mechanism) {
-    throw new Error(
-      'No feedback mechanism provided. Please provide a feedback mechanism to use.'
-    )
-  }
-
-  const feedbackMechanism = new finalOptions.mechanism({
-    task: task,
-    options: finalOptions
-  })
-
-  task.addAfterCallHook(async function onCall(output, ctx) {
-    const feedback = await feedbackMechanism.interact(output)
-    ctx.metadata = { ...ctx.metadata, feedback }
-  })
-
-  return task
-}

src/task.ts

@@ -4,6 +4,11 @@ import { ZodType } from 'zod'
 import * as errors from './errors'
 import * as types from './types'
 import type { Agentic } from './agentic'
+import {
+  HumanFeedbackMechanismCLI,
+  HumanFeedbackOptions,
+  HumanFeedbackType
+} from './human-feedback'
 import { defaultIDGeneratorFn, isValidTaskIdentifier } from './utils'
 
 /**
@@ -29,8 +34,8 @@ export abstract class BaseTask<
   protected _timeoutMs?: number
   protected _retryConfig: types.RetryConfig
 
-  private _preHooks: Array<types.BeforeCallHook<TInput>> = []
+  private _preHooks: Array<types.TaskBeforeCallHook<TInput>> = []
-  private _postHooks: Array<types.AfterCallHook<TInput, TOutput>> = []
+  private _postHooks: Array<types.TaskAfterCallHook<TInput, TOutput>> = []
 
   constructor(options: types.BaseTaskOptions = {}) {
     this._agentic = options.agentic ?? globalThis.__agentic?.deref()
@@ -77,12 +82,14 @@ export abstract class BaseTask<
     return ''
   }
 
-  public addBeforeCallHook(hook: types.BeforeCallHook<TInput>): this {
+  public addBeforeCallHook(hook: types.TaskBeforeCallHook<TInput>): this {
     this._preHooks.push(hook)
     return this
   }
 
-  public addAfterCallHook(hook: types.AfterCallHook<TInput, TOutput>): this {
+  public addAfterCallHook(
+    hook: types.TaskAfterCallHook<TInput, TOutput>
+  ): this {
     this._postHooks.push(hook)
     return this
   }
@@ -106,6 +113,41 @@ export abstract class BaseTask<
     throw new Error(`clone not implemented for task "${this.nameForModel}"`)
   }
 
+  public withHumanFeedback<V extends HumanFeedbackType>(
+    options: HumanFeedbackOptions<V, TOutput> = {}
+  ): this {
+    options = Object.assign(
+      {
+        type: 'confirm',
+        abort: false,
+        editing: false,
+        annotations: false,
+        timeoutMs: Number.POSITIVE_INFINITY,
+        mechanism: HumanFeedbackMechanismCLI
+      },
+      this.agentic.humanFeedbackDefaults,
+      options
+    )
+
+    if (!options.mechanism) {
+      throw new Error(
+        'No feedback mechanism provided. Please provide a feedback mechanism to use.'
+      )
+    }
+
+    const feedbackMechanism = new options.mechanism({
+      task: this,
+      options
+    })
+
+    this.addAfterCallHook(async (output, ctx) => {
+      const feedback = await feedbackMechanism.interact(output)
+      ctx.metadata = { ...ctx.metadata, feedback }
+    })
+
+    return this
+  }
+
   public retryConfig(retryConfig: types.RetryConfig): this {
     this._retryConfig = retryConfig
     return this
@@ -152,15 +194,16 @@ export abstract class BaseTask<
       }
     }
 
-    for (const hook of this._preHooks) {
+    for (const preHook of this._preHooks) {
-      await hook(ctx)
+      await preHook(ctx)
     }
 
     const result = await pRetry(
       async () => {
         const result = await this._call(ctx)
-        for (const hook of this._postHooks) {
+
-          await hook(result, ctx)
+        for (const postHook of this._postHooks) {
+          await postHook(result, ctx)
         }
 
         return result

src/types.ts

@@ -155,11 +155,11 @@ export declare class CancelablePromise<T> extends Promise<T> {
 
 // export type ProgressFunction = (partialResponse: ChatMessage) => void
 
-export type BeforeCallHook<TInput extends TaskInput = void> = (
+export type TaskBeforeCallHook<TInput extends TaskInput = void> = (
   ctx: TaskCallContext<TInput>
 ) => void | Promise<void>
 
-export type AfterCallHook<
+export type TaskAfterCallHook<
   TInput extends TaskInput = void,
   TOutput extends TaskOutput = string
 > = (output: TOutput, ctx: TaskCallContext<TInput>) => void | Promise<void>