diff --git a/legacy/examples/expert-question-answering.ts b/legacy/examples/expert-question-answering.ts index 56e91c88..0c73fb35 100644 --- a/legacy/examples/expert-question-answering.ts +++ b/legacy/examples/expert-question-answering.ts @@ -2,12 +2,7 @@ import 'dotenv/config' import { OpenAIClient } from 'openai-fetch' import { z } from 'zod' -import { - Agentic, - NovuNotificationTool, - SerpAPITool, - withHumanFeedback -} from '@/index' +import { Agentic, NovuNotificationTool, SerpAPITool } from '@/index' async function main() { const openai = new OpenAIClient({ apiKey: process.env.OPENAI_API_KEY! }) @@ -15,33 +10,31 @@ async function main() { const question = 'How do I build a product that people will love?' - const task = withHumanFeedback( - agentic - .gpt4( - `Generate a list of {n} prominent experts that can answer the following question: {{question}}.` - ) - .tools([new SerpAPITool()]) - .output( - z.array( - z.object({ - name: z.string(), - bio: z.string() - }) - ) - ) - .input( + const { metadata } = await agentic + .gpt4( + `Generate a list of {n} prominent experts that can answer the following question: {{question}}.` + ) + .tools([new SerpAPITool()]) + .output( + z.array( z.object({ - question: z.string(), - n: z.number().int().default(5) + name: z.string(), + bio: z.string() }) - ), - { + ) + ) + .input( + z.object({ + question: z.string(), + n: z.number().int().default(5) + }) + ) + .withHumanFeedback({ type: 'selectN' - } - ) - const { metadata } = await task.callWithMetadata({ - question - }) + }) + .callWithMetadata({ + question + }) if ( metadata.feedback && @@ -78,6 +71,7 @@ async function main() { ${expert}: ${answer}` }, '') + const notifier = new NovuNotificationTool() await notifier.call({ name: 'send-email', diff --git a/legacy/examples/hf0-demo.ts b/legacy/examples/hf0-demo.ts index 1ee1e246..ae9b46dc 100644 --- a/legacy/examples/hf0-demo.ts +++ b/legacy/examples/hf0-demo.ts @@ -6,43 +6,43 @@ import { Agentic, HumanFeedbackMechanismTwilio, SearchAndCrawlTool, - WeatherTool, - withHumanFeedback + WeatherTool } from '@/index' async function main() { const openai = new OpenAIClient({ apiKey: process.env.OPENAI_API_KEY! }) - const agentic = new Agentic({ openai }) + const agentic = new Agentic({ + openai, + humanFeedbackDefaults: { + mechanism: HumanFeedbackMechanismTwilio + } + }) const topic = process.argv[2] || 'OpenAI' - const res0 = await withHumanFeedback( - agentic - .gpt4({ - messages: [ - { - role: 'system', - content: `You are a McKinsey analyst who is an expert at writing executive summaries.` - }, - { - role: 'user', - content: `What are the 3 most important questions we would need to answer in order to have a thorough understanding of this topic: {{topic}}? Be concise but creative in your questions, and make sure to capture the true essence of the topic.` - } - ], - model: 'gpt-4', - temperature: 1.0 + const res0 = await agentic + .gpt4({ + messages: [ + { + role: 'system', + content: `You are a McKinsey analyst who is an expert at writing executive summaries.` + }, + { + role: 'user', + content: `What are the 3 most important questions we would need to answer in order to have a thorough understanding of this topic: {{topic}}? Be concise but creative in your questions, and make sure to capture the true essence of the topic.` + } + ], + model: 'gpt-4', + temperature: 1.0 + }) + .input( + z.object({ + topic: z.string() }) - .input( - z.object({ - topic: z.string() - }) - ) - .output(z.array(z.string()).describe('question')), - { - type: 'selectN', - mechanism: HumanFeedbackMechanismTwilio - } - ).callWithMetadata({ topic }) + ) + .output(z.array(z.string()).describe('question')) + .withHumanFeedback({ type: 'selectN' }) + .callWithMetadata({ topic }) console.log() console.log() diff --git a/legacy/scratch/examples/human-feedback-cli.ts b/legacy/scratch/examples/human-feedback-cli.ts index 56312d74..96bbee3d 100644 --- a/legacy/scratch/examples/human-feedback-cli.ts +++ b/legacy/scratch/examples/human-feedback-cli.ts @@ -2,14 +2,13 @@ import 'dotenv/config' import { OpenAIClient } from 'openai-fetch' import { z } from 'zod' -import { HumanFeedbackMechanismCLI } from '@/human-feedback' -import { Agentic, withHumanFeedback } from '@/index' +import { Agentic } from '@/index' async function main() { const openai = new OpenAIClient({ apiKey: process.env.OPENAI_API_KEY! }) const ai = new Agentic({ openai }) - const topicJokes = ai + const out = await ai .gpt3(`Tell me {{num}} jokes about {{topic}}`) .input( z.object({ @@ -19,19 +18,17 @@ async function main() { ) .output(z.array(z.string())) .modelParams({ temperature: 0.9 }) + .withHumanFeedback({ + type: 'selectN', + annotations: false, + abort: false, + editing: true + }) + .callWithMetadata({ + topic: 'politicians', + num: 5 + }) - const topicJokesFeedback = withHumanFeedback(topicJokes, { - type: 'selectN', - annotations: false, - abort: false, - editing: true, - mechanism: HumanFeedbackMechanismCLI - }) - - const out = await topicJokesFeedback.callWithMetadata({ - topic: 'politicians', - num: 5 - }) const feedback = out.metadata.feedback console.log(JSON.stringify(feedback, null, 2)) } diff --git a/legacy/scratch/examples/human-feedback-slack.ts b/legacy/scratch/examples/human-feedback-slack.ts index 57444b9b..50b8ac7b 100644 --- a/legacy/scratch/examples/human-feedback-slack.ts +++ b/legacy/scratch/examples/human-feedback-slack.ts @@ -2,14 +2,13 @@ import 'dotenv/config' import { OpenAIClient } from 'openai-fetch' import { z } from 'zod' -import { HumanFeedbackMechanismSlack } from '@/human-feedback' -import { Agentic, withHumanFeedback } from '@/index' +import { Agentic, HumanFeedbackMechanismSlack } from '@/index' async function main() { const openai = new OpenAIClient({ apiKey: process.env.OPENAI_API_KEY! }) const ai = new Agentic({ openai }) - const topicJokes = ai + const out = await ai .gpt3(`Tell me {{num}} jokes about {{topic}}`) .input( z.object({ @@ -19,19 +18,18 @@ async function main() { ) .output(z.array(z.string())) .modelParams({ temperature: 0.9 }) + .withHumanFeedback({ + type: 'selectN', + annotations: false, + abort: false, + editing: true, + mechanism: HumanFeedbackMechanismSlack + }) + .callWithMetadata({ + topic: 'politicians', + num: 5 + }) - const topicJokesFeedback = withHumanFeedback(topicJokes, { - type: 'selectN', - annotations: false, - abort: false, - editing: true, - mechanism: HumanFeedbackMechanismSlack - }) - - const out = await topicJokesFeedback.callWithMetadata({ - topic: 'politicians', - num: 5 - }) const feedback = out.metadata.feedback console.log(JSON.stringify(feedback, null, 2)) } diff --git a/legacy/scratch/examples/human-feedback-twilio.ts b/legacy/scratch/examples/human-feedback-twilio.ts index ec80b801..527f4d39 100644 --- a/legacy/scratch/examples/human-feedback-twilio.ts +++ b/legacy/scratch/examples/human-feedback-twilio.ts @@ -2,14 +2,13 @@ import 'dotenv/config' import { OpenAIClient } from 'openai-fetch' import { z } from 'zod' -import { HumanFeedbackMechanismTwilio } from '@/human-feedback' -import { Agentic, withHumanFeedback } from '@/index' +import { Agentic, HumanFeedbackMechanismTwilio } from '@/index' async function main() { const openai = new OpenAIClient({ apiKey: process.env.OPENAI_API_KEY! }) const ai = new Agentic({ openai }) - const topicJokes = ai + const out = await ai .gpt3(`Tell me {{num}} jokes about {{topic}}`) .input( z.object({ @@ -19,19 +18,18 @@ async function main() { ) .output(z.array(z.string())) .modelParams({ temperature: 0.9 }) + .withHumanFeedback({ + type: 'selectN', + annotations: false, + abort: false, + editing: true, + mechanism: HumanFeedbackMechanismTwilio + }) + .callWithMetadata({ + topic: 'politicians', + num: 5 + }) - const topicJokesFeedback = withHumanFeedback(topicJokes, { - type: 'selectN', - annotations: false, - abort: false, - editing: true, - mechanism: HumanFeedbackMechanismTwilio - }) - - const out = await topicJokesFeedback.callWithMetadata({ - topic: 'politicians', - num: 5 - }) const feedback = out.metadata.feedback console.log(JSON.stringify(feedback, null, 2)) } diff --git a/legacy/src/human-feedback/feedback.ts b/legacy/src/human-feedback/feedback.ts index aa161bbb..4e8f5dab 100644 --- a/legacy/src/human-feedback/feedback.ts +++ b/legacy/src/human-feedback/feedback.ts @@ -1,10 +1,7 @@ -import * as types from '@/types' import { Agentic } from '@/agentic' import { HumanFeedbackDeclineError } from '@/errors' import { BaseTask } from '@/task' -import { HumanFeedbackMechanismCLI } from './cli' - /** * Actions the user can take in the feedback selection prompt. */ @@ -303,48 +300,3 @@ export abstract class HumanFeedbackMechanism< return feedback as FeedbackTypeToMetadata } } - -export function withHumanFeedback< - TInput extends types.TaskInput, - TOutput extends types.TaskOutput, - V extends HumanFeedbackType ->( - task: BaseTask, - options: HumanFeedbackOptions = {} -) { - task = task.clone() - - // Use Object.assign to merge the options, instance defaults, and hard-coded defaults: - const finalOptions: HumanFeedbackOptions = Object.assign( - { - type: 'confirm', - abort: false, - editing: false, - annotations: false, - timeoutMs: Number.POSITIVE_INFINITY, - mechanism: HumanFeedbackMechanismCLI - }, - // Default options from the instance: - task.agentic.humanFeedbackDefaults, - // User-provided options (override instance defaults): - options - ) - - if (!finalOptions.mechanism) { - throw new Error( - 'No feedback mechanism provided. Please provide a feedback mechanism to use.' - ) - } - - const feedbackMechanism = new finalOptions.mechanism({ - task: task, - options: finalOptions - }) - - task.addAfterCallHook(async function onCall(output, ctx) { - const feedback = await feedbackMechanism.interact(output) - ctx.metadata = { ...ctx.metadata, feedback } - }) - - return task -} diff --git a/legacy/src/task.ts b/legacy/src/task.ts index 99d01b24..6252af27 100644 --- a/legacy/src/task.ts +++ b/legacy/src/task.ts @@ -4,6 +4,11 @@ import { ZodType } from 'zod' import * as errors from './errors' import * as types from './types' import type { Agentic } from './agentic' +import { + HumanFeedbackMechanismCLI, + HumanFeedbackOptions, + HumanFeedbackType +} from './human-feedback' import { defaultIDGeneratorFn, isValidTaskIdentifier } from './utils' /** @@ -29,8 +34,8 @@ export abstract class BaseTask< protected _timeoutMs?: number protected _retryConfig: types.RetryConfig - private _preHooks: Array> = [] - private _postHooks: Array> = [] + private _preHooks: Array> = [] + private _postHooks: Array> = [] constructor(options: types.BaseTaskOptions = {}) { this._agentic = options.agentic ?? globalThis.__agentic?.deref() @@ -77,12 +82,14 @@ export abstract class BaseTask< return '' } - public addBeforeCallHook(hook: types.BeforeCallHook): this { + public addBeforeCallHook(hook: types.TaskBeforeCallHook): this { this._preHooks.push(hook) return this } - public addAfterCallHook(hook: types.AfterCallHook): this { + public addAfterCallHook( + hook: types.TaskAfterCallHook + ): this { this._postHooks.push(hook) return this } @@ -106,6 +113,41 @@ export abstract class BaseTask< throw new Error(`clone not implemented for task "${this.nameForModel}"`) } + public withHumanFeedback( + options: HumanFeedbackOptions = {} + ): this { + options = Object.assign( + { + type: 'confirm', + abort: false, + editing: false, + annotations: false, + timeoutMs: Number.POSITIVE_INFINITY, + mechanism: HumanFeedbackMechanismCLI + }, + this.agentic.humanFeedbackDefaults, + options + ) + + if (!options.mechanism) { + throw new Error( + 'No feedback mechanism provided. Please provide a feedback mechanism to use.' + ) + } + + const feedbackMechanism = new options.mechanism({ + task: this, + options + }) + + this.addAfterCallHook(async (output, ctx) => { + const feedback = await feedbackMechanism.interact(output) + ctx.metadata = { ...ctx.metadata, feedback } + }) + + return this + } + public retryConfig(retryConfig: types.RetryConfig): this { this._retryConfig = retryConfig return this @@ -152,15 +194,16 @@ export abstract class BaseTask< } } - for (const hook of this._preHooks) { - await hook(ctx) + for (const preHook of this._preHooks) { + await preHook(ctx) } const result = await pRetry( async () => { const result = await this._call(ctx) - for (const hook of this._postHooks) { - await hook(result, ctx) + + for (const postHook of this._postHooks) { + await postHook(result, ctx) } return result diff --git a/legacy/src/types.ts b/legacy/src/types.ts index dfa6d8c2..4b457001 100644 --- a/legacy/src/types.ts +++ b/legacy/src/types.ts @@ -155,11 +155,11 @@ export declare class CancelablePromise extends Promise { // export type ProgressFunction = (partialResponse: ChatMessage) => void -export type BeforeCallHook = ( +export type TaskBeforeCallHook = ( ctx: TaskCallContext ) => void | Promise -export type AfterCallHook< +export type TaskAfterCallHook< TInput extends TaskInput = void, TOutput extends TaskOutput = string > = (output: TOutput, ctx: TaskCallContext) => void | Promise