diff --git a/src/agentic.ts b/src/agentic.ts index e90b1d0..d827e20 100644 --- a/src/agentic.ts +++ b/src/agentic.ts @@ -3,8 +3,8 @@ import defaultKy from 'ky' import * as types from './types' import { DEFAULT_OPENAI_MODEL } from './constants' import { - HumanFeedbackMechanism, - HumanFeedbackMechanismCLI + HumanFeedbackMechanismCLI, + HumanFeedbackOptions } from './human-feedback' import { OpenAIChatCompletion } from './llms/openai' import { defaultLogger } from './logger' @@ -22,8 +22,7 @@ export class Agentic { types.BaseLLMOptions, 'provider' | 'model' | 'modelParams' | 'timeoutMs' | 'retryConfig' > - protected _defaultHumanFeedbackMechamism?: HumanFeedbackMechanism - + protected _humanFeedbackDefaults: HumanFeedbackOptions protected _idGeneratorFn: types.IDGeneratorFunction protected _id: string @@ -34,7 +33,7 @@ export class Agentic { types.BaseLLMOptions, 'provider' | 'model' | 'modelParams' | 'timeoutMs' | 'retryConfig' > - defaultHumanFeedbackMechanism?: HumanFeedbackMechanism + humanFeedbackDefaults?: HumanFeedbackOptions idGeneratorFn?: types.IDGeneratorFunction logger?: types.Logger ky?: types.KyInstance @@ -68,9 +67,14 @@ export class Agentic { // TODO // this._anthropicModelDefaults = {} - this._defaultHumanFeedbackMechamism = - opts.defaultHumanFeedbackMechanism ?? - new HumanFeedbackMechanismCLI({ agentic: this }) + this._humanFeedbackDefaults = { + type: 'confirm', + bail: false, + editing: false, + annotations: false, + mechanism: HumanFeedbackMechanismCLI, + ...opts.humanFeedbackDefaults + } this._idGeneratorFn = opts.idGeneratorFn ?? defaultIDGeneratorFn this._id = this._idGeneratorFn() @@ -92,8 +96,8 @@ export class Agentic { return this._logger } - public get defaultHumanFeedbackMechamism() { - return this._defaultHumanFeedbackMechamism + public get humanFeedbackDefaults() { + return this._humanFeedbackDefaults } public get idGeneratorFn(): types.IDGeneratorFunction { diff --git a/src/human-feedback/index.ts b/src/human-feedback/index.ts new file mode 100644 index 0000000..3fe346f --- /dev/null +++ b/src/human-feedback/index.ts @@ -0,0 +1,354 @@ +import checkbox from '@inquirer/checkbox' +import editor from '@inquirer/editor' +import input from '@inquirer/input' +import select from '@inquirer/select' + +import { Agentic } from '@/agentic' +import { BaseTask } from '@/task' +import { TaskResponseMetadata } from '@/types' + +/** + * Actions the user can take in the feedback selection prompt. + */ +export const UserActions = { + Accept: 'accept', + Edit: 'edit', + Decline: 'decline', + Select: 'select', + Exit: 'exit' +} as const + +export type UserActions = (typeof UserActions)[keyof typeof UserActions] + +const UserActionMessages: Record = { + [UserActions.Accept]: 'Accept the output', + [UserActions.Edit]: 'Edit the output (open in editor)', + [UserActions.Decline]: 'Decline the output', + [UserActions.Select]: 'Select outputs to keep', + [UserActions.Exit]: 'Exit' +} + +/** + * Prompt the user to select one of a list of options. + */ +async function askUser( + message: string, + choices: UserActions[] +): Promise { + return select({ + message, + choices: choices.map((choice) => ({ + name: UserActionMessages[choice], + value: choice + })) + }) +} + +/** + * Available types of human feedback. + */ +export type HumanFeedbackType = 'confirm' | 'selectOne' | 'selectN' + +type HumanFeedbackMechanismConstructor = new ( + ...args: any[] +) => T + +/** + * Options for human feedback. + */ +export type HumanFeedbackOptions< + T extends HumanFeedbackMechanism = HumanFeedbackMechanism +> = { + /** + * What type of feedback to request. + */ + type?: HumanFeedbackType + + /** + * Whether the user can bail out of the feedback loop. + */ + bail?: boolean + + /** + * Whether the user can edit the output. + */ + editing?: boolean + + /** + * Whether the user can add free-form text annotations. + */ + annotations?: boolean + + /** + * The human feedback mechanism to use for this task. + */ + mechanism?: HumanFeedbackMechanismConstructor +} + +export abstract class HumanFeedbackMechanism { + protected _agentic: Agentic + + protected _options: HumanFeedbackOptions + + constructor({ + agentic, + options + }: { + agentic: Agentic + options: HumanFeedbackOptions + }) { + this._agentic = agentic + this._options = options + } + + public abstract confirm( + response: any, + metadata: TaskResponseMetadata + ): Promise + public abstract selectOne( + response: any, + metadata: TaskResponseMetadata + ): Promise + public abstract selectN( + response: any, + metadata: TaskResponseMetadata + ): Promise + + public async interact(response: any, metadata: TaskResponseMetadata) { + if (this._options.type === 'selectN') { + await this.selectN(response, metadata) + } else if (this._options.type === 'confirm') { + await this.confirm(response, metadata) + } else if (this._options.type === 'selectOne') { + await this.selectOne(response, metadata) + } + } +} + +export class HumanFeedbackMechanismCLI extends HumanFeedbackMechanism { + constructor({ + agentic, + options + }: { + agentic: Agentic + options: HumanFeedbackOptions + }) { + super({ agentic, options }) + this._agentic = agentic + this._options = options + } + + public async confirm( + response: any, + metadata: TaskResponseMetadata + ): Promise { + const stringified = JSON.stringify(response, null, 2) + const msg = [ + 'The following output was generated:', + stringified, + 'What would you like to do?' + ].join('\n') + const choices: UserActions[] = [UserActions.Accept, UserActions.Decline] + + if (this._options.editing) { + choices.push(UserActions.Edit) + } + + if (this._options.bail) { + choices.push(UserActions.Exit) + } + + const feedback = await askUser(msg, choices) + + metadata.feedback = {} + + switch (feedback) { + case UserActions.Accept: + metadata.feedback.accepted = true + break + + case UserActions.Edit: { + const editedOutput = await editor({ + message: 'Edit the output:', + default: stringified + }) + metadata.feedback.editedOutput = editedOutput + break + } + + case UserActions.Decline: + metadata.feedback.accepted = false + break + + case UserActions.Exit: + throw new Error('Exiting...') + + default: + throw new Error(`Unexpected feedback: ${feedback}`) + } + + if (this._options.annotations) { + const annotation = await input({ + message: + 'Please leave an annotation (leave blank to skip; press enter to submit):' + }) + if (annotation) { + metadata.feedback.annotation = annotation + } + } + } + + public async selectOne( + response: any[], + metadata: TaskResponseMetadata + ): Promise { + const stringified = JSON.stringify(response, null, 2) + const msg = [ + 'The following output was generated:', + stringified, + 'What would you like to do?' + ].join('\n') + const choices: UserActions[] = [UserActions.Select] + + if (this._options.editing) { + choices.push(UserActions.Edit) + } + + if (this._options.bail) { + choices.push(UserActions.Exit) + } + + const feedback = await askUser(msg, choices) + + metadata.feedback = {} + + switch (feedback) { + case UserActions.Edit: { + const editedOutput = await editor({ + message: 'Edit the output:', + default: stringified + }) + metadata.feedback.editedOutput = editedOutput + break + } + + case UserActions.Select: { + const choices = response.map((option) => ({ + name: option, + value: option + })) + const chosen = await select({ message: 'Pick one output:', choices }) + metadata.feedback.chosen = chosen + break + } + + case UserActions.Exit: + throw new Error('Exiting...') + + default: + throw new Error(`Unexpected feedback: ${feedback}`) + } + } + + public async selectN( + response: any[], + metadata: TaskResponseMetadata + ): Promise { + const stringified = JSON.stringify(response, null, 2) + const msg = [ + 'The following output was generated:', + stringified, + 'What would you like to do?' + ].join('\n') + const choices: UserActions[] = [UserActions.Select] + + if (this._options.editing) { + choices.push(UserActions.Edit) + } + + if (this._options.bail) { + choices.push(UserActions.Exit) + } + + const feedback = + choices.length === 1 ? UserActions.Select : await askUser(msg, choices) + + metadata.feedback = {} + + switch (feedback) { + case UserActions.Edit: { + const editedOutput = await editor({ + message: 'Edit the output:', + default: stringified + }) + metadata.feedback.editedOutput = editedOutput + break + } + + case UserActions.Select: { + const choices = response.map((option) => ({ + name: option, + value: option + })) + const chosen = await checkbox({ message: 'Select outputs:', choices }) + metadata.feedback.selected = chosen + break + } + + case UserActions.Exit: + throw new Error('Exiting...') + + default: + throw new Error(`Unexpected feedback: ${feedback}`) + } + } +} + +export function withHumanFeedback( + task: BaseTask, + options: HumanFeedbackOptions = {} +) { + task = task.clone() + + // Default options defined at the instance level + const instanceDefaults = task.agentic.humanFeedbackDefaults + + // Use Object.assign to merge the options, instance defaults, and hard-coded defaults + const finalOptions: HumanFeedbackOptions = Object.assign( + { + type: 'confirm', + bail: false, + editing: false, + annotations: false, + mechanism: HumanFeedbackMechanismCLI + }, + // Defaults from the instance: + instanceDefaults, + // User-provided options (override instance defaults): + options + ) + + if (!finalOptions.mechanism) { + throw new Error( + 'No feedback mechanism provided. Please provide a feedback mechanism to use.' + ) + } + + const feedbackMechanism = new finalOptions.mechanism({ + agentic: task.agentic, + options: finalOptions + }) + + const originalCall = task.callWithMetadata.bind(task) + + task.callWithMetadata = async function (input?: T) { + const response = await originalCall(input) + + // Process the response and add feedback to metadata + await feedbackMechanism.interact(response.result, response.metadata) + + return response + } + + return task +}