From e4a65cb982e485cdaa2c5f27ba17c724cce3671e Mon Sep 17 00:00:00 2001 From: Philipp Burckhardt Date: Tue, 13 Jun 2023 11:13:26 -0400 Subject: [PATCH 01/22] feat: add initial implementation --- src/agentic.ts | 24 ++- src/human-feedback/index.ts | 354 ++++++++++++++++++++++++++++++++++++ 2 files changed, 368 insertions(+), 10 deletions(-) create mode 100644 src/human-feedback/index.ts diff --git a/src/agentic.ts b/src/agentic.ts index e90b1d0..d827e20 100644 --- a/src/agentic.ts +++ b/src/agentic.ts @@ -3,8 +3,8 @@ import defaultKy from 'ky' import * as types from './types' import { DEFAULT_OPENAI_MODEL } from './constants' import { - HumanFeedbackMechanism, - HumanFeedbackMechanismCLI + HumanFeedbackMechanismCLI, + HumanFeedbackOptions } from './human-feedback' import { OpenAIChatCompletion } from './llms/openai' import { defaultLogger } from './logger' @@ -22,8 +22,7 @@ export class Agentic { types.BaseLLMOptions, 'provider' | 'model' | 'modelParams' | 'timeoutMs' | 'retryConfig' > - protected _defaultHumanFeedbackMechamism?: HumanFeedbackMechanism - + protected _humanFeedbackDefaults: HumanFeedbackOptions protected _idGeneratorFn: types.IDGeneratorFunction protected _id: string @@ -34,7 +33,7 @@ export class Agentic { types.BaseLLMOptions, 'provider' | 'model' | 'modelParams' | 'timeoutMs' | 'retryConfig' > - defaultHumanFeedbackMechanism?: HumanFeedbackMechanism + humanFeedbackDefaults?: HumanFeedbackOptions idGeneratorFn?: types.IDGeneratorFunction logger?: types.Logger ky?: types.KyInstance @@ -68,9 +67,14 @@ export class Agentic { // TODO // this._anthropicModelDefaults = {} - this._defaultHumanFeedbackMechamism = - opts.defaultHumanFeedbackMechanism ?? - new HumanFeedbackMechanismCLI({ agentic: this }) + this._humanFeedbackDefaults = { + type: 'confirm', + bail: false, + editing: false, + annotations: false, + mechanism: HumanFeedbackMechanismCLI, + ...opts.humanFeedbackDefaults + } this._idGeneratorFn = opts.idGeneratorFn ?? defaultIDGeneratorFn this._id = this._idGeneratorFn() @@ -92,8 +96,8 @@ export class Agentic { return this._logger } - public get defaultHumanFeedbackMechamism() { - return this._defaultHumanFeedbackMechamism + public get humanFeedbackDefaults() { + return this._humanFeedbackDefaults } public get idGeneratorFn(): types.IDGeneratorFunction { diff --git a/src/human-feedback/index.ts b/src/human-feedback/index.ts new file mode 100644 index 0000000..3fe346f --- /dev/null +++ b/src/human-feedback/index.ts @@ -0,0 +1,354 @@ +import checkbox from '@inquirer/checkbox' +import editor from '@inquirer/editor' +import input from '@inquirer/input' +import select from '@inquirer/select' + +import { Agentic } from '@/agentic' +import { BaseTask } from '@/task' +import { TaskResponseMetadata } from '@/types' + +/** + * Actions the user can take in the feedback selection prompt. + */ +export const UserActions = { + Accept: 'accept', + Edit: 'edit', + Decline: 'decline', + Select: 'select', + Exit: 'exit' +} as const + +export type UserActions = (typeof UserActions)[keyof typeof UserActions] + +const UserActionMessages: Record = { + [UserActions.Accept]: 'Accept the output', + [UserActions.Edit]: 'Edit the output (open in editor)', + [UserActions.Decline]: 'Decline the output', + [UserActions.Select]: 'Select outputs to keep', + [UserActions.Exit]: 'Exit' +} + +/** + * Prompt the user to select one of a list of options. + */ +async function askUser( + message: string, + choices: UserActions[] +): Promise { + return select({ + message, + choices: choices.map((choice) => ({ + name: UserActionMessages[choice], + value: choice + })) + }) +} + +/** + * Available types of human feedback. + */ +export type HumanFeedbackType = 'confirm' | 'selectOne' | 'selectN' + +type HumanFeedbackMechanismConstructor = new ( + ...args: any[] +) => T + +/** + * Options for human feedback. + */ +export type HumanFeedbackOptions< + T extends HumanFeedbackMechanism = HumanFeedbackMechanism +> = { + /** + * What type of feedback to request. + */ + type?: HumanFeedbackType + + /** + * Whether the user can bail out of the feedback loop. + */ + bail?: boolean + + /** + * Whether the user can edit the output. + */ + editing?: boolean + + /** + * Whether the user can add free-form text annotations. + */ + annotations?: boolean + + /** + * The human feedback mechanism to use for this task. + */ + mechanism?: HumanFeedbackMechanismConstructor +} + +export abstract class HumanFeedbackMechanism { + protected _agentic: Agentic + + protected _options: HumanFeedbackOptions + + constructor({ + agentic, + options + }: { + agentic: Agentic + options: HumanFeedbackOptions + }) { + this._agentic = agentic + this._options = options + } + + public abstract confirm( + response: any, + metadata: TaskResponseMetadata + ): Promise + public abstract selectOne( + response: any, + metadata: TaskResponseMetadata + ): Promise + public abstract selectN( + response: any, + metadata: TaskResponseMetadata + ): Promise + + public async interact(response: any, metadata: TaskResponseMetadata) { + if (this._options.type === 'selectN') { + await this.selectN(response, metadata) + } else if (this._options.type === 'confirm') { + await this.confirm(response, metadata) + } else if (this._options.type === 'selectOne') { + await this.selectOne(response, metadata) + } + } +} + +export class HumanFeedbackMechanismCLI extends HumanFeedbackMechanism { + constructor({ + agentic, + options + }: { + agentic: Agentic + options: HumanFeedbackOptions + }) { + super({ agentic, options }) + this._agentic = agentic + this._options = options + } + + public async confirm( + response: any, + metadata: TaskResponseMetadata + ): Promise { + const stringified = JSON.stringify(response, null, 2) + const msg = [ + 'The following output was generated:', + stringified, + 'What would you like to do?' + ].join('\n') + const choices: UserActions[] = [UserActions.Accept, UserActions.Decline] + + if (this._options.editing) { + choices.push(UserActions.Edit) + } + + if (this._options.bail) { + choices.push(UserActions.Exit) + } + + const feedback = await askUser(msg, choices) + + metadata.feedback = {} + + switch (feedback) { + case UserActions.Accept: + metadata.feedback.accepted = true + break + + case UserActions.Edit: { + const editedOutput = await editor({ + message: 'Edit the output:', + default: stringified + }) + metadata.feedback.editedOutput = editedOutput + break + } + + case UserActions.Decline: + metadata.feedback.accepted = false + break + + case UserActions.Exit: + throw new Error('Exiting...') + + default: + throw new Error(`Unexpected feedback: ${feedback}`) + } + + if (this._options.annotations) { + const annotation = await input({ + message: + 'Please leave an annotation (leave blank to skip; press enter to submit):' + }) + if (annotation) { + metadata.feedback.annotation = annotation + } + } + } + + public async selectOne( + response: any[], + metadata: TaskResponseMetadata + ): Promise { + const stringified = JSON.stringify(response, null, 2) + const msg = [ + 'The following output was generated:', + stringified, + 'What would you like to do?' + ].join('\n') + const choices: UserActions[] = [UserActions.Select] + + if (this._options.editing) { + choices.push(UserActions.Edit) + } + + if (this._options.bail) { + choices.push(UserActions.Exit) + } + + const feedback = await askUser(msg, choices) + + metadata.feedback = {} + + switch (feedback) { + case UserActions.Edit: { + const editedOutput = await editor({ + message: 'Edit the output:', + default: stringified + }) + metadata.feedback.editedOutput = editedOutput + break + } + + case UserActions.Select: { + const choices = response.map((option) => ({ + name: option, + value: option + })) + const chosen = await select({ message: 'Pick one output:', choices }) + metadata.feedback.chosen = chosen + break + } + + case UserActions.Exit: + throw new Error('Exiting...') + + default: + throw new Error(`Unexpected feedback: ${feedback}`) + } + } + + public async selectN( + response: any[], + metadata: TaskResponseMetadata + ): Promise { + const stringified = JSON.stringify(response, null, 2) + const msg = [ + 'The following output was generated:', + stringified, + 'What would you like to do?' + ].join('\n') + const choices: UserActions[] = [UserActions.Select] + + if (this._options.editing) { + choices.push(UserActions.Edit) + } + + if (this._options.bail) { + choices.push(UserActions.Exit) + } + + const feedback = + choices.length === 1 ? UserActions.Select : await askUser(msg, choices) + + metadata.feedback = {} + + switch (feedback) { + case UserActions.Edit: { + const editedOutput = await editor({ + message: 'Edit the output:', + default: stringified + }) + metadata.feedback.editedOutput = editedOutput + break + } + + case UserActions.Select: { + const choices = response.map((option) => ({ + name: option, + value: option + })) + const chosen = await checkbox({ message: 'Select outputs:', choices }) + metadata.feedback.selected = chosen + break + } + + case UserActions.Exit: + throw new Error('Exiting...') + + default: + throw new Error(`Unexpected feedback: ${feedback}`) + } + } +} + +export function withHumanFeedback( + task: BaseTask, + options: HumanFeedbackOptions = {} +) { + task = task.clone() + + // Default options defined at the instance level + const instanceDefaults = task.agentic.humanFeedbackDefaults + + // Use Object.assign to merge the options, instance defaults, and hard-coded defaults + const finalOptions: HumanFeedbackOptions = Object.assign( + { + type: 'confirm', + bail: false, + editing: false, + annotations: false, + mechanism: HumanFeedbackMechanismCLI + }, + // Defaults from the instance: + instanceDefaults, + // User-provided options (override instance defaults): + options + ) + + if (!finalOptions.mechanism) { + throw new Error( + 'No feedback mechanism provided. Please provide a feedback mechanism to use.' + ) + } + + const feedbackMechanism = new finalOptions.mechanism({ + agentic: task.agentic, + options: finalOptions + }) + + const originalCall = task.callWithMetadata.bind(task) + + task.callWithMetadata = async function (input?: T) { + const response = await originalCall(input) + + // Process the response and add feedback to metadata + await feedbackMechanism.interact(response.result, response.metadata) + + return response + } + + return task +} From 25d285b29029e9dff4998e5f953720ef5cba0463 Mon Sep 17 00:00:00 2001 From: Philipp Burckhardt Date: Tue, 13 Jun 2023 11:21:31 -0400 Subject: [PATCH 02/22] refactor: separate out abstract class and sub-classes --- src/agentic.ts | 6 +- src/human-feedback.ts | 55 --------- src/human-feedback/cli.ts | 226 ++++++++++++++++++++++++++++++++++++ src/human-feedback/index.ts | 221 +---------------------------------- 4 files changed, 229 insertions(+), 279 deletions(-) delete mode 100644 src/human-feedback.ts create mode 100644 src/human-feedback/cli.ts diff --git a/src/agentic.ts b/src/agentic.ts index d827e20..512a8dc 100644 --- a/src/agentic.ts +++ b/src/agentic.ts @@ -2,10 +2,8 @@ import defaultKy from 'ky' import * as types from './types' import { DEFAULT_OPENAI_MODEL } from './constants' -import { - HumanFeedbackMechanismCLI, - HumanFeedbackOptions -} from './human-feedback' +import { HumanFeedbackOptions } from './human-feedback' +import { HumanFeedbackMechanismCLI } from './human-feedback/cli' import { OpenAIChatCompletion } from './llms/openai' import { defaultLogger } from './logger' import { defaultIDGeneratorFn } from './utils' diff --git a/src/human-feedback.ts b/src/human-feedback.ts deleted file mode 100644 index 6f9e3d2..0000000 --- a/src/human-feedback.ts +++ /dev/null @@ -1,55 +0,0 @@ -import * as types from './types' -import { Agentic } from './agentic' -import { BaseTask } from './task' - -export type HumanFeedbackType = 'confirm' | 'selectOne' | 'selectN' - -export type HumanFeedbackOptions = { - type: HumanFeedbackType - - /** - * Whether to allow exiting - */ - bail?: boolean - - editing?: boolean - - annotations?: boolean - - feedbackMechanism?: HumanFeedbackMechanism -} - -export abstract class HumanFeedbackMechanism { - protected _agentic: Agentic - - constructor({ agentic }: { agentic: Agentic }) { - this._agentic = agentic - } - // TODO -} - -export class HumanFeedbackMechanismCLI extends HumanFeedbackMechanism { - // TODO - constructor(opts: { agentic: Agentic }) { - super(opts) - } -} - -export function withHumanFeedback< - TInput extends void | types.JsonObject = void, - TOutput extends types.JsonValue = string ->( - task: BaseTask, - options: HumanFeedbackOptions = { - type: 'confirm', - bail: false, - editing: false, - annotations: false - } -) { - const { feedbackMechanism = task.agentic.defaultHumanFeedbackMechamism } = - options - - // TODO - return task -} diff --git a/src/human-feedback/cli.ts b/src/human-feedback/cli.ts new file mode 100644 index 0000000..a503d0d --- /dev/null +++ b/src/human-feedback/cli.ts @@ -0,0 +1,226 @@ +import checkbox from '@inquirer/checkbox' +import editor from '@inquirer/editor' +import input from '@inquirer/input' +import select from '@inquirer/select' + +import { Agentic } from '@/agentic' +import { TaskResponseMetadata } from '@/types' + +import { HumanFeedbackMechanism, HumanFeedbackOptions } from './index' + +/** + * Actions the user can take in the feedback selection prompt. + */ +export const UserActions = { + Accept: 'accept', + Edit: 'edit', + Decline: 'decline', + Select: 'select', + Exit: 'exit' +} as const + +export type UserActions = (typeof UserActions)[keyof typeof UserActions] + +const UserActionMessages: Record = { + [UserActions.Accept]: 'Accept the output', + [UserActions.Edit]: 'Edit the output (open in editor)', + [UserActions.Decline]: 'Decline the output', + [UserActions.Select]: 'Select outputs to keep', + [UserActions.Exit]: 'Exit' +} + +/** + * Prompt the user to select one of a list of options. + */ +async function askUser( + message: string, + choices: UserActions[] +): Promise { + return select({ + message, + choices: choices.map((choice) => ({ + name: UserActionMessages[choice], + value: choice + })) + }) +} + +export class HumanFeedbackMechanismCLI extends HumanFeedbackMechanism { + constructor({ + agentic, + options + }: { + agentic: Agentic + options: HumanFeedbackOptions + }) { + super({ agentic, options }) + this._agentic = agentic + this._options = options + } + + public async confirm( + response: any, + metadata: TaskResponseMetadata + ): Promise { + const stringified = JSON.stringify(response, null, 2) + const msg = [ + 'The following output was generated:', + stringified, + 'What would you like to do?' + ].join('\n') + const choices: UserActions[] = [UserActions.Accept, UserActions.Decline] + + if (this._options.editing) { + choices.push(UserActions.Edit) + } + + if (this._options.bail) { + choices.push(UserActions.Exit) + } + + const feedback = await askUser(msg, choices) + + metadata.feedback = {} + + switch (feedback) { + case UserActions.Accept: + metadata.feedback.accepted = true + break + + case UserActions.Edit: { + const editedOutput = await editor({ + message: 'Edit the output:', + default: stringified + }) + metadata.feedback.editedOutput = editedOutput + break + } + + case UserActions.Decline: + metadata.feedback.accepted = false + break + + case UserActions.Exit: + throw new Error('Exiting...') + + default: + throw new Error(`Unexpected feedback: ${feedback}`) + } + + if (this._options.annotations) { + const annotation = await input({ + message: + 'Please leave an annotation (leave blank to skip; press enter to submit):' + }) + + if (annotation) { + metadata.feedback.annotation = annotation + } + } + } + + public async selectOne( + response: any[], + metadata: TaskResponseMetadata + ): Promise { + const stringified = JSON.stringify(response, null, 2) + const msg = [ + 'The following output was generated:', + stringified, + 'What would you like to do?' + ].join('\n') + const choices: UserActions[] = [UserActions.Select] + + if (this._options.editing) { + choices.push(UserActions.Edit) + } + + if (this._options.bail) { + choices.push(UserActions.Exit) + } + + const feedback = await askUser(msg, choices) + + metadata.feedback = {} + + switch (feedback) { + case UserActions.Edit: { + const editedOutput = await editor({ + message: 'Edit the output:', + default: stringified + }) + metadata.feedback.editedOutput = editedOutput + break + } + + case UserActions.Select: { + const choices = response.map((option) => ({ + name: option, + value: option + })) + const chosen = await select({ message: 'Pick one output:', choices }) + metadata.feedback.chosen = chosen + break + } + + case UserActions.Exit: + throw new Error('Exiting...') + + default: + throw new Error(`Unexpected feedback: ${feedback}`) + } + } + + public async selectN( + response: any[], + metadata: TaskResponseMetadata + ): Promise { + const stringified = JSON.stringify(response, null, 2) + const msg = [ + 'The following output was generated:', + stringified, + 'What would you like to do?' + ].join('\n') + const choices: UserActions[] = [UserActions.Select] + + if (this._options.editing) { + choices.push(UserActions.Edit) + } + + if (this._options.bail) { + choices.push(UserActions.Exit) + } + + const feedback = + choices.length === 1 ? UserActions.Select : await askUser(msg, choices) + + metadata.feedback = {} + + switch (feedback) { + case UserActions.Edit: { + const editedOutput = await editor({ + message: 'Edit the output:', + default: stringified + }) + metadata.feedback.editedOutput = editedOutput + break + } + + case UserActions.Select: { + const choices = response.map((option) => ({ + name: option, + value: option + })) + const chosen = await checkbox({ message: 'Select outputs:', choices }) + metadata.feedback.selected = chosen + break + } + + case UserActions.Exit: + throw new Error('Exiting...') + + default: + throw new Error(`Unexpected feedback: ${feedback}`) + } + } +} diff --git a/src/human-feedback/index.ts b/src/human-feedback/index.ts index 3fe346f..e2af21d 100644 --- a/src/human-feedback/index.ts +++ b/src/human-feedback/index.ts @@ -1,48 +1,8 @@ -import checkbox from '@inquirer/checkbox' -import editor from '@inquirer/editor' -import input from '@inquirer/input' -import select from '@inquirer/select' - import { Agentic } from '@/agentic' import { BaseTask } from '@/task' import { TaskResponseMetadata } from '@/types' -/** - * Actions the user can take in the feedback selection prompt. - */ -export const UserActions = { - Accept: 'accept', - Edit: 'edit', - Decline: 'decline', - Select: 'select', - Exit: 'exit' -} as const - -export type UserActions = (typeof UserActions)[keyof typeof UserActions] - -const UserActionMessages: Record = { - [UserActions.Accept]: 'Accept the output', - [UserActions.Edit]: 'Edit the output (open in editor)', - [UserActions.Decline]: 'Decline the output', - [UserActions.Select]: 'Select outputs to keep', - [UserActions.Exit]: 'Exit' -} - -/** - * Prompt the user to select one of a list of options. - */ -async function askUser( - message: string, - choices: UserActions[] -): Promise { - return select({ - message, - choices: choices.map((choice) => ({ - name: UserActionMessages[choice], - value: choice - })) - }) -} +import { HumanFeedbackMechanismCLI } from './cli' /** * Available types of human feedback. @@ -125,185 +85,6 @@ export abstract class HumanFeedbackMechanism { } } -export class HumanFeedbackMechanismCLI extends HumanFeedbackMechanism { - constructor({ - agentic, - options - }: { - agentic: Agentic - options: HumanFeedbackOptions - }) { - super({ agentic, options }) - this._agentic = agentic - this._options = options - } - - public async confirm( - response: any, - metadata: TaskResponseMetadata - ): Promise { - const stringified = JSON.stringify(response, null, 2) - const msg = [ - 'The following output was generated:', - stringified, - 'What would you like to do?' - ].join('\n') - const choices: UserActions[] = [UserActions.Accept, UserActions.Decline] - - if (this._options.editing) { - choices.push(UserActions.Edit) - } - - if (this._options.bail) { - choices.push(UserActions.Exit) - } - - const feedback = await askUser(msg, choices) - - metadata.feedback = {} - - switch (feedback) { - case UserActions.Accept: - metadata.feedback.accepted = true - break - - case UserActions.Edit: { - const editedOutput = await editor({ - message: 'Edit the output:', - default: stringified - }) - metadata.feedback.editedOutput = editedOutput - break - } - - case UserActions.Decline: - metadata.feedback.accepted = false - break - - case UserActions.Exit: - throw new Error('Exiting...') - - default: - throw new Error(`Unexpected feedback: ${feedback}`) - } - - if (this._options.annotations) { - const annotation = await input({ - message: - 'Please leave an annotation (leave blank to skip; press enter to submit):' - }) - if (annotation) { - metadata.feedback.annotation = annotation - } - } - } - - public async selectOne( - response: any[], - metadata: TaskResponseMetadata - ): Promise { - const stringified = JSON.stringify(response, null, 2) - const msg = [ - 'The following output was generated:', - stringified, - 'What would you like to do?' - ].join('\n') - const choices: UserActions[] = [UserActions.Select] - - if (this._options.editing) { - choices.push(UserActions.Edit) - } - - if (this._options.bail) { - choices.push(UserActions.Exit) - } - - const feedback = await askUser(msg, choices) - - metadata.feedback = {} - - switch (feedback) { - case UserActions.Edit: { - const editedOutput = await editor({ - message: 'Edit the output:', - default: stringified - }) - metadata.feedback.editedOutput = editedOutput - break - } - - case UserActions.Select: { - const choices = response.map((option) => ({ - name: option, - value: option - })) - const chosen = await select({ message: 'Pick one output:', choices }) - metadata.feedback.chosen = chosen - break - } - - case UserActions.Exit: - throw new Error('Exiting...') - - default: - throw new Error(`Unexpected feedback: ${feedback}`) - } - } - - public async selectN( - response: any[], - metadata: TaskResponseMetadata - ): Promise { - const stringified = JSON.stringify(response, null, 2) - const msg = [ - 'The following output was generated:', - stringified, - 'What would you like to do?' - ].join('\n') - const choices: UserActions[] = [UserActions.Select] - - if (this._options.editing) { - choices.push(UserActions.Edit) - } - - if (this._options.bail) { - choices.push(UserActions.Exit) - } - - const feedback = - choices.length === 1 ? UserActions.Select : await askUser(msg, choices) - - metadata.feedback = {} - - switch (feedback) { - case UserActions.Edit: { - const editedOutput = await editor({ - message: 'Edit the output:', - default: stringified - }) - metadata.feedback.editedOutput = editedOutput - break - } - - case UserActions.Select: { - const choices = response.map((option) => ({ - name: option, - value: option - })) - const chosen = await checkbox({ message: 'Select outputs:', choices }) - metadata.feedback.selected = chosen - break - } - - case UserActions.Exit: - throw new Error('Exiting...') - - default: - throw new Error(`Unexpected feedback: ${feedback}`) - } - } -} - export function withHumanFeedback( task: BaseTask, options: HumanFeedbackOptions = {} From 721f34f3f186fbd44dff6dc35b99c37a634fc8cb Mon Sep 17 00:00:00 2001 From: Philipp Burckhardt Date: Tue, 13 Jun 2023 12:36:45 -0400 Subject: [PATCH 03/22] feat: add initial Slack and Twilio human feedback classes --- src/human-feedback/cli.ts | 28 +--- src/human-feedback/feedback.ts | 156 +++++++++++++++++ src/human-feedback/index.ts | 139 +--------------- src/human-feedback/slack.ts | 286 ++++++++++++++++++++++++++++++++ src/human-feedback/twilio.ts | 294 +++++++++++++++++++++++++++++++++ 5 files changed, 746 insertions(+), 157 deletions(-) create mode 100644 src/human-feedback/feedback.ts create mode 100644 src/human-feedback/slack.ts create mode 100644 src/human-feedback/twilio.ts diff --git a/src/human-feedback/cli.ts b/src/human-feedback/cli.ts index a503d0d..f864f52 100644 --- a/src/human-feedback/cli.ts +++ b/src/human-feedback/cli.ts @@ -6,28 +6,12 @@ import select from '@inquirer/select' import { Agentic } from '@/agentic' import { TaskResponseMetadata } from '@/types' -import { HumanFeedbackMechanism, HumanFeedbackOptions } from './index' - -/** - * Actions the user can take in the feedback selection prompt. - */ -export const UserActions = { - Accept: 'accept', - Edit: 'edit', - Decline: 'decline', - Select: 'select', - Exit: 'exit' -} as const - -export type UserActions = (typeof UserActions)[keyof typeof UserActions] - -const UserActionMessages: Record = { - [UserActions.Accept]: 'Accept the output', - [UserActions.Edit]: 'Edit the output (open in editor)', - [UserActions.Decline]: 'Decline the output', - [UserActions.Select]: 'Select outputs to keep', - [UserActions.Exit]: 'Exit' -} +import { + HumanFeedbackMechanism, + HumanFeedbackOptions, + UserActionMessages, + UserActions +} from './feedback' /** * Prompt the user to select one of a list of options. diff --git a/src/human-feedback/feedback.ts b/src/human-feedback/feedback.ts new file mode 100644 index 0000000..41fac1b --- /dev/null +++ b/src/human-feedback/feedback.ts @@ -0,0 +1,156 @@ +import { Agentic } from '@/agentic' +import { BaseTask } from '@/task' +import { TaskResponseMetadata } from '@/types' + +import { HumanFeedbackMechanismCLI } from './cli' + +/** + * Actions the user can take in the feedback selection prompt. + */ +export const UserActions = { + Accept: 'accept', + Edit: 'edit', + Decline: 'decline', + Select: 'select', + Exit: 'exit' +} as const + +export type UserActions = (typeof UserActions)[keyof typeof UserActions] + +export const UserActionMessages: Record = { + [UserActions.Accept]: 'Accept the output', + [UserActions.Edit]: 'Edit the output', + [UserActions.Decline]: 'Decline the output', + [UserActions.Select]: 'Select outputs to keep', + [UserActions.Exit]: 'Exit' +} + +/** + * Available types of human feedback. + */ +export type HumanFeedbackType = 'confirm' | 'selectOne' | 'selectN' + +type HumanFeedbackMechanismConstructor = new ( + ...args: any[] +) => T + +/** + * Options for human feedback. + */ +export type HumanFeedbackOptions< + T extends HumanFeedbackMechanism = HumanFeedbackMechanism +> = { + /** + * What type of feedback to request. + */ + type?: HumanFeedbackType + + /** + * Whether the user can bail out of the feedback loop. + */ + bail?: boolean + + /** + * Whether the user can edit the output. + */ + editing?: boolean + + /** + * Whether the user can add free-form text annotations. + */ + annotations?: boolean + + /** + * The human feedback mechanism to use for this task. + */ + mechanism?: HumanFeedbackMechanismConstructor +} + +export abstract class HumanFeedbackMechanism { + protected _agentic: Agentic + + protected _options: HumanFeedbackOptions + + constructor({ + agentic, + options + }: { + agentic: Agentic + options: HumanFeedbackOptions + }) { + this._agentic = agentic + this._options = options + } + + public abstract confirm( + response: any, + metadata: TaskResponseMetadata + ): Promise + public abstract selectOne( + response: any, + metadata: TaskResponseMetadata + ): Promise + public abstract selectN( + response: any, + metadata: TaskResponseMetadata + ): Promise + + public async interact(response: any, metadata: TaskResponseMetadata) { + if (this._options.type === 'selectN') { + await this.selectN(response, metadata) + } else if (this._options.type === 'confirm') { + await this.confirm(response, metadata) + } else if (this._options.type === 'selectOne') { + await this.selectOne(response, metadata) + } + } +} + +export function withHumanFeedback( + task: BaseTask, + options: HumanFeedbackOptions = {} +) { + task = task.clone() + + // Default options defined at the instance level + const instanceDefaults = task.agentic.humanFeedbackDefaults + + // Use Object.assign to merge the options, instance defaults, and hard-coded defaults + const finalOptions: HumanFeedbackOptions = Object.assign( + { + type: 'confirm', + bail: false, + editing: false, + annotations: false, + mechanism: HumanFeedbackMechanismCLI + }, + // Defaults from the instance: + instanceDefaults, + // User-provided options (override instance defaults): + options + ) + + if (!finalOptions.mechanism) { + throw new Error( + 'No feedback mechanism provided. Please provide a feedback mechanism to use.' + ) + } + + const feedbackMechanism = new finalOptions.mechanism({ + agentic: task.agentic, + options: finalOptions + }) + + const originalCall = task.callWithMetadata.bind(task) + + task.callWithMetadata = async function (input?: T) { + const response = await originalCall(input) + + // Process the response and add feedback to metadata + await feedbackMechanism.interact(response.result, response.metadata) + + return response + } + + return task +} diff --git a/src/human-feedback/index.ts b/src/human-feedback/index.ts index e2af21d..105c876 100644 --- a/src/human-feedback/index.ts +++ b/src/human-feedback/index.ts @@ -1,135 +1,4 @@ -import { Agentic } from '@/agentic' -import { BaseTask } from '@/task' -import { TaskResponseMetadata } from '@/types' - -import { HumanFeedbackMechanismCLI } from './cli' - -/** - * Available types of human feedback. - */ -export type HumanFeedbackType = 'confirm' | 'selectOne' | 'selectN' - -type HumanFeedbackMechanismConstructor = new ( - ...args: any[] -) => T - -/** - * Options for human feedback. - */ -export type HumanFeedbackOptions< - T extends HumanFeedbackMechanism = HumanFeedbackMechanism -> = { - /** - * What type of feedback to request. - */ - type?: HumanFeedbackType - - /** - * Whether the user can bail out of the feedback loop. - */ - bail?: boolean - - /** - * Whether the user can edit the output. - */ - editing?: boolean - - /** - * Whether the user can add free-form text annotations. - */ - annotations?: boolean - - /** - * The human feedback mechanism to use for this task. - */ - mechanism?: HumanFeedbackMechanismConstructor -} - -export abstract class HumanFeedbackMechanism { - protected _agentic: Agentic - - protected _options: HumanFeedbackOptions - - constructor({ - agentic, - options - }: { - agentic: Agentic - options: HumanFeedbackOptions - }) { - this._agentic = agentic - this._options = options - } - - public abstract confirm( - response: any, - metadata: TaskResponseMetadata - ): Promise - public abstract selectOne( - response: any, - metadata: TaskResponseMetadata - ): Promise - public abstract selectN( - response: any, - metadata: TaskResponseMetadata - ): Promise - - public async interact(response: any, metadata: TaskResponseMetadata) { - if (this._options.type === 'selectN') { - await this.selectN(response, metadata) - } else if (this._options.type === 'confirm') { - await this.confirm(response, metadata) - } else if (this._options.type === 'selectOne') { - await this.selectOne(response, metadata) - } - } -} - -export function withHumanFeedback( - task: BaseTask, - options: HumanFeedbackOptions = {} -) { - task = task.clone() - - // Default options defined at the instance level - const instanceDefaults = task.agentic.humanFeedbackDefaults - - // Use Object.assign to merge the options, instance defaults, and hard-coded defaults - const finalOptions: HumanFeedbackOptions = Object.assign( - { - type: 'confirm', - bail: false, - editing: false, - annotations: false, - mechanism: HumanFeedbackMechanismCLI - }, - // Defaults from the instance: - instanceDefaults, - // User-provided options (override instance defaults): - options - ) - - if (!finalOptions.mechanism) { - throw new Error( - 'No feedback mechanism provided. Please provide a feedback mechanism to use.' - ) - } - - const feedbackMechanism = new finalOptions.mechanism({ - agentic: task.agentic, - options: finalOptions - }) - - const originalCall = task.callWithMetadata.bind(task) - - task.callWithMetadata = async function (input?: T) { - const response = await originalCall(input) - - // Process the response and add feedback to metadata - await feedbackMechanism.interact(response.result, response.metadata) - - return response - } - - return task -} +export * from './cli' +export * from './feedback' +export * from './slack' +export * from './twilio' diff --git a/src/human-feedback/slack.ts b/src/human-feedback/slack.ts new file mode 100644 index 0000000..5f3d1ea --- /dev/null +++ b/src/human-feedback/slack.ts @@ -0,0 +1,286 @@ +import { Agentic } from '@/agentic' +import { SlackClient } from '@/services/slack' +import { TaskResponseMetadata } from '@/types' + +import { + HumanFeedbackMechanism, + HumanFeedbackOptions, + UserActionMessages, + UserActions +} from './feedback' + +export class HumanFeedbackMechanismSlack extends HumanFeedbackMechanism { + private slackClient: SlackClient + + constructor({ + agentic, + options + }: { + agentic: Agentic + options: HumanFeedbackOptions + }) { + super({ agentic, options }) + this.slackClient = new SlackClient() + } + + private async askUser( + message: string, + choices: UserActions[] + ): Promise { + message += '\n\n' + message += choices + .map((choice, idx) => `*${idx}* - ${UserActionMessages[choice]}`) + .join('\n') + message += '\n\n' + message += 'Reply with the number of your choice.' + const response = await this.slackClient.sendAndWaitForReply({ + text: message, + validate: (slackMessage) => { + const choice = parseInt(slackMessage.text) + return !isNaN(choice) && choice >= 0 && choice < choices.length + } + }) + return choices[parseInt(response.text)] + } + + public async confirm( + response: any, + metadata: TaskResponseMetadata + ): Promise { + const stringified = JSON.stringify(response, null, 2) + const msg = [ + 'The following output was generated:', + '```', + stringified, + '```', + 'What would you like to do?' + ].join('\n') + + const choices: UserActions[] = [UserActions.Accept, UserActions.Decline] + + if (this._options.editing) { + choices.push(UserActions.Edit) + } + + if (this._options.bail) { + choices.push(UserActions.Exit) + } + + const feedback = await this.askUser(msg, choices) + + metadata.feedback = {} + + switch (feedback) { + case UserActions.Accept: + metadata.feedback.accepted = true + break + + case UserActions.Edit: { + let { text: editedOutput } = await this.slackClient.sendAndWaitForReply( + { + text: 'Copy and edit the output:' + } + ) + editedOutput = editedOutput.replace(/```$/g, '') + editedOutput = editedOutput.replace(/^```/g, '') + metadata.feedback.editedOutput = editedOutput + break + } + + case UserActions.Decline: + metadata.feedback.accepted = false + break + + case UserActions.Exit: + throw new Error('Exiting...') + + default: + throw new Error(`Unexpected feedback: ${feedback}`) + } + + if (this._options.annotations) { + try { + const annotation = await this.slackClient.sendAndWaitForReply({ + text: 'Please leave an annotation (optional):' + }) + + if (annotation) { + metadata.feedback.annotation = annotation.text + } + } catch (e) { + // Deliberately swallow the error here as the user is not required to leave an annotation + } + } + } + + public async selectOne( + response: any[], + metadata: TaskResponseMetadata + ): Promise { + const stringified = JSON.stringify(response, null, 2) + const msg = [ + 'The following output was generated:', + '```', + stringified, + '```', + 'What would you like to do?' + ].join('\n') + + const choices: UserActions[] = [UserActions.Select] + + if (this._options.editing) { + choices.push(UserActions.Edit) + } + + if (this._options.bail) { + choices.push(UserActions.Exit) + } + + const feedback = + choices.length === 1 + ? UserActions.Select + : await this.askUser(msg, choices) + + metadata.feedback = {} + + switch (feedback) { + case UserActions.Edit: { + let { text: editedOutput } = await this.slackClient.sendAndWaitForReply( + { + text: 'Copy and edit the output:' + } + ) + editedOutput = editedOutput.replace(/```$/g, '') + editedOutput = editedOutput.replace(/^```/g, '') + metadata.feedback.editedOutput = editedOutput + break + } + + case UserActions.Select: { + const { text: selectedOutput } = + await this.slackClient.sendAndWaitForReply({ + text: + 'Pick one output:' + + response.map((r, idx) => `\n*${idx}* - ${r}`).join('') + + '\n\nReply with the number of your choice.', + validate: (slackMessage) => { + const choice = parseInt(slackMessage.text) + return !isNaN(choice) && choice >= 0 && choice < response.length + } + }) + metadata.feedback.chosen = response[parseInt(selectedOutput)] + break + } + + case UserActions.Exit: + throw new Error('Exiting...') + + default: + throw new Error(`Unexpected feedback: ${feedback}`) + } + + if (this._options.annotations) { + try { + const annotation = await this.slackClient.sendAndWaitForReply({ + text: 'Please leave an annotation (optional):' + }) + + if (annotation) { + metadata.feedback.annotation = annotation.text + } + } catch (e) { + // Deliberately swallow the error here as the user is not required to leave an annotation + } + } + } + + public async selectN( + response: any[], + metadata: TaskResponseMetadata + ): Promise { + const stringified = JSON.stringify(response, null, 2) + const msg = [ + 'The following output was generated:', + stringified, + 'What would you like to do?' + ].join('\n') + const choices: UserActions[] = [UserActions.Select] + + if (this._options.editing) { + choices.push(UserActions.Edit) + } + + if (this._options.bail) { + choices.push(UserActions.Exit) + } + + const feedback = + choices.length === 1 + ? UserActions.Select + : await this.askUser(msg, choices) + + metadata.feedback = {} + + switch (feedback) { + case UserActions.Edit: { + let { text: editedOutput } = await this.slackClient.sendAndWaitForReply( + { + text: 'Copy and edit the output:' + } + ) + editedOutput = editedOutput.replace(/```$/g, '') + editedOutput = editedOutput.replace(/^```/g, '') + metadata.feedback.editedOutput = editedOutput + break + } + + case UserActions.Select: { + const { text: selectedOutput } = + await this.slackClient.sendAndWaitForReply({ + text: + 'Select outputs:' + + response.map((r, idx) => `\n*${idx}* - ${r}`).join('') + + '\n\nReply with a comma-separated list of the output numbers of your choice.', + validate: (slackMessage) => { + const choices = slackMessage.text.split(',') + return choices.every((choice) => { + const choiceInt = parseInt(choice) + return ( + !isNaN(choiceInt) && + choiceInt >= 0 && + choiceInt < response.length + ) + }) + } + }) + const chosenOutputs = selectedOutput + .split(',') + .map((choice) => parseInt(choice)) + metadata.feedback.selected = response.filter((_, idx) => { + return chosenOutputs.includes(idx) + }) + break + } + + case UserActions.Exit: + throw new Error('Exiting...') + + default: + throw new Error(`Unexpected feedback: ${feedback}`) + } + + if (this._options.annotations) { + try { + const annotation = await this.slackClient.sendAndWaitForReply({ + text: 'Please leave an annotation (optional):' + }) + + if (annotation) { + metadata.feedback.annotation = annotation.text + } + } catch (e) { + // Deliberately swallow the error here as the user is not required to leave an annotation + } + } + } +} diff --git a/src/human-feedback/twilio.ts b/src/human-feedback/twilio.ts new file mode 100644 index 0000000..383f35b --- /dev/null +++ b/src/human-feedback/twilio.ts @@ -0,0 +1,294 @@ +import { Agentic } from '@/agentic' +import { TwilioConversationClient } from '@/services/twilio-conversation' +import { TaskResponseMetadata } from '@/types' + +import { + HumanFeedbackMechanism, + HumanFeedbackOptions, + UserActionMessages, + UserActions +} from './feedback' + +export class HumanFeedbackMechanismTwilio extends HumanFeedbackMechanism { + private twilioClient: TwilioConversationClient + + constructor({ + agentic, + options + }: { + agentic: Agentic + options: HumanFeedbackOptions + }) { + super({ agentic, options }) + this.twilioClient = new TwilioConversationClient() + } + + private async askUser( + message: string, + choices: UserActions[] + ): Promise { + message += '\n\n' + message += choices + .map((choice, idx) => `*${idx}* - ${UserActionMessages[choice]}`) + .join('\n') + message += '\n\n' + message += 'Reply with the number of your choice.' + const response = await this.twilioClient.sendAndWaitForReply({ + name: 'human-feedback-ask', + text: message, + validate: (message) => { + const choice = parseInt(message.body) + return !isNaN(choice) && choice >= 0 && choice < choices.length + } + }) + return choices[parseInt(response.body)] + } + + public async confirm( + response: any, + metadata: TaskResponseMetadata + ): Promise { + const stringified = JSON.stringify(response, null, 2) + const msg = [ + 'The following output was generated:', + '', + stringified, + '', + 'What would you like to do?' + ].join('\n') + + const choices: UserActions[] = [UserActions.Accept, UserActions.Decline] + + if (this._options.editing) { + choices.push(UserActions.Edit) + } + + if (this._options.bail) { + choices.push(UserActions.Exit) + } + + const feedback = await this.askUser(msg, choices) + + metadata.feedback = {} + + switch (feedback) { + case UserActions.Accept: + metadata.feedback.accepted = true + break + + case UserActions.Edit: { + let { body: editedOutput } = + await this.twilioClient.sendAndWaitForReply({ + name: 'human-feedback-edit', + text: 'Copy and edit the output:' + }) + editedOutput = editedOutput.replace(/```$/g, '') + editedOutput = editedOutput.replace(/^```/g, '') + metadata.feedback.editedOutput = editedOutput + break + } + + case UserActions.Decline: + metadata.feedback.accepted = false + break + + case UserActions.Exit: + throw new Error('Exiting...') + + default: + throw new Error(`Unexpected feedback: ${feedback}`) + } + + if (this._options.annotations) { + try { + const annotation = await this.twilioClient.sendAndWaitForReply({ + name: 'human-feedback-annotation', + text: 'Please leave an annotation (optional):' + }) + + if (annotation) { + metadata.feedback.annotation = annotation.body + } + } catch (e) { + // Deliberately swallow the error here as the user is not required to leave an annotation + } + } + } + + public async selectOne( + response: any[], + metadata: TaskResponseMetadata + ): Promise { + const stringified = JSON.stringify(response, null, 2) + const msg = [ + 'The following output was generated:', + '', + stringified, + '', + 'What would you like to do?' + ].join('\n') + + const choices: UserActions[] = [UserActions.Select] + + if (this._options.editing) { + choices.push(UserActions.Edit) + } + + if (this._options.bail) { + choices.push(UserActions.Exit) + } + + const feedback = + choices.length === 1 + ? UserActions.Select + : await this.askUser(msg, choices) + + metadata.feedback = {} + + switch (feedback) { + case UserActions.Edit: { + let { body: editedOutput } = + await this.twilioClient.sendAndWaitForReply({ + text: 'Copy and edit the output:', + name: 'human-feedback-edit' + }) + editedOutput = editedOutput.replace(/```$/g, '') + editedOutput = editedOutput.replace(/^```/g, '') + metadata.feedback.editedOutput = editedOutput + break + } + + case UserActions.Select: { + const { body: selectedOutput } = + await this.twilioClient.sendAndWaitForReply({ + name: 'human-feedback-select', + text: + 'Pick one output:' + + response.map((r, idx) => `\n${idx} - ${r}`).join('') + + '\n\nReply with the number of your choice.', + validate: (message) => { + const choice = parseInt(message.body) + return !isNaN(choice) && choice >= 0 && choice < response.length + } + }) + metadata.feedback.chosen = response[parseInt(selectedOutput)] + break + } + + case UserActions.Exit: + throw new Error('Exiting...') + + default: + throw new Error(`Unexpected feedback: ${feedback}`) + } + + if (this._options.annotations) { + try { + const annotation = await this.twilioClient.sendAndWaitForReply({ + text: 'Please leave an annotation (optional):', + name: 'human-feedback-annotation' + }) + + if (annotation) { + metadata.feedback.annotation = annotation.body + } + } catch (e) { + // Deliberately swallow the error here as the user is not required to leave an annotation + } + } + } + + public async selectN( + response: any[], + metadata: TaskResponseMetadata + ): Promise { + const stringified = JSON.stringify(response, null, 2) + const msg = [ + 'The following output was generated:', + '', + stringified, + '', + 'What would you like to do?' + ].join('\n') + const choices: UserActions[] = [UserActions.Select] + + if (this._options.editing) { + choices.push(UserActions.Edit) + } + + if (this._options.bail) { + choices.push(UserActions.Exit) + } + + const feedback = + choices.length === 1 + ? UserActions.Select + : await this.askUser(msg, choices) + + metadata.feedback = {} + + switch (feedback) { + case UserActions.Edit: { + let { body: editedOutput } = + await this.twilioClient.sendAndWaitForReply({ + text: 'Copy and edit the output:', + name: 'human-feedback-edit' + }) + editedOutput = editedOutput.replace(/```$/g, '') + editedOutput = editedOutput.replace(/^```/g, '') + metadata.feedback.editedOutput = editedOutput + break + } + + case UserActions.Select: { + const { body: selectedOutput } = + await this.twilioClient.sendAndWaitForReply({ + name: 'human-feedback-select', + text: + 'Select outputs:' + + response.map((r, idx) => `\n${idx} - ${r}`).join('') + + '\n\nReply with a comma-separated list of the output numbers of your choice.', + validate: (message) => { + const choices = message.body.split(',') + return choices.every((choice) => { + const choiceInt = parseInt(choice) + return ( + !isNaN(choiceInt) && + choiceInt >= 0 && + choiceInt < response.length + ) + }) + } + }) + const chosenOutputs = selectedOutput + .split(',') + .map((choice) => parseInt(choice)) + metadata.feedback.selected = response.filter((_, idx) => { + return chosenOutputs.includes(idx) + }) + break + } + + case UserActions.Exit: + throw new Error('Exiting...') + + default: + throw new Error(`Unexpected feedback: ${feedback}`) + } + + if (this._options.annotations) { + try { + const annotation = await this.twilioClient.sendAndWaitForReply({ + text: 'Please leave an annotation (optional):', + name: 'human-feedback-annotation' + }) + + if (annotation) { + metadata.feedback.annotation = annotation.body + } + } catch (e) { + // Deliberately swallow the error here as the user is not required to leave an annotation + } + } + } +} From 768a727597fc90f4fddd7de641bd6d6715f4382a Mon Sep 17 00:00:00 2001 From: Philipp Burckhardt Date: Tue, 13 Jun 2023 15:52:45 -0400 Subject: [PATCH 04/22] refactor: move annotate to own method --- src/human-feedback/cli.ts | 31 +++++++++-------- src/human-feedback/feedback.ts | 15 ++++++-- src/human-feedback/slack.ts | 59 +++++++++---------------------- src/human-feedback/twilio.ts | 63 ++++++++++------------------------ 4 files changed, 64 insertions(+), 104 deletions(-) diff --git a/src/human-feedback/cli.ts b/src/human-feedback/cli.ts index f864f52..ad5f263 100644 --- a/src/human-feedback/cli.ts +++ b/src/human-feedback/cli.ts @@ -42,7 +42,21 @@ export class HumanFeedbackMechanismCLI extends HumanFeedbackMechanism { this._options = options } - public async confirm( + protected async annotate( + response: any, + metadata: TaskResponseMetadata + ): Promise { + const annotation = await input({ + message: + 'Please leave an annotation (leave blank to skip; press enter to submit):' + }) + + if (annotation) { + metadata.feedback.annotation = annotation + } + } + + protected async confirm( response: any, metadata: TaskResponseMetadata ): Promise { @@ -90,20 +104,9 @@ export class HumanFeedbackMechanismCLI extends HumanFeedbackMechanism { default: throw new Error(`Unexpected feedback: ${feedback}`) } - - if (this._options.annotations) { - const annotation = await input({ - message: - 'Please leave an annotation (leave blank to skip; press enter to submit):' - }) - - if (annotation) { - metadata.feedback.annotation = annotation - } - } } - public async selectOne( + protected async selectOne( response: any[], metadata: TaskResponseMetadata ): Promise { @@ -155,7 +158,7 @@ export class HumanFeedbackMechanismCLI extends HumanFeedbackMechanism { } } - public async selectN( + protected async selectN( response: any[], metadata: TaskResponseMetadata ): Promise { diff --git a/src/human-feedback/feedback.ts b/src/human-feedback/feedback.ts index 41fac1b..4619a9d 100644 --- a/src/human-feedback/feedback.ts +++ b/src/human-feedback/feedback.ts @@ -82,15 +82,20 @@ export abstract class HumanFeedbackMechanism { this._options = options } - public abstract confirm( + protected abstract confirm( response: any, metadata: TaskResponseMetadata ): Promise - public abstract selectOne( + protected abstract selectOne( response: any, metadata: TaskResponseMetadata ): Promise - public abstract selectN( + protected abstract selectN( + response: any, + metadata: TaskResponseMetadata + ): Promise + + protected abstract annotate( response: any, metadata: TaskResponseMetadata ): Promise @@ -103,6 +108,10 @@ export abstract class HumanFeedbackMechanism { } else if (this._options.type === 'selectOne') { await this.selectOne(response, metadata) } + + if (this._options.annotations) { + await this.annotate(response, metadata) + } } } diff --git a/src/human-feedback/slack.ts b/src/human-feedback/slack.ts index 5f3d1ea..e8172d1 100644 --- a/src/human-feedback/slack.ts +++ b/src/human-feedback/slack.ts @@ -23,6 +23,23 @@ export class HumanFeedbackMechanismSlack extends HumanFeedbackMechanism { this.slackClient = new SlackClient() } + protected async annotate( + response: any, + metadata: TaskResponseMetadata + ): Promise { + try { + const annotation = await this.slackClient.sendAndWaitForReply({ + text: 'Please leave an annotation (optional):' + }) + + if (annotation) { + metadata.feedback.annotation = annotation.text + } + } catch (e) { + // Deliberately swallow the error here as the user is not required to leave an annotation + } + } + private async askUser( message: string, choices: UserActions[] @@ -97,20 +114,6 @@ export class HumanFeedbackMechanismSlack extends HumanFeedbackMechanism { default: throw new Error(`Unexpected feedback: ${feedback}`) } - - if (this._options.annotations) { - try { - const annotation = await this.slackClient.sendAndWaitForReply({ - text: 'Please leave an annotation (optional):' - }) - - if (annotation) { - metadata.feedback.annotation = annotation.text - } - } catch (e) { - // Deliberately swallow the error here as the user is not required to leave an annotation - } - } } public async selectOne( @@ -178,20 +181,6 @@ export class HumanFeedbackMechanismSlack extends HumanFeedbackMechanism { default: throw new Error(`Unexpected feedback: ${feedback}`) } - - if (this._options.annotations) { - try { - const annotation = await this.slackClient.sendAndWaitForReply({ - text: 'Please leave an annotation (optional):' - }) - - if (annotation) { - metadata.feedback.annotation = annotation.text - } - } catch (e) { - // Deliberately swallow the error here as the user is not required to leave an annotation - } - } } public async selectN( @@ -268,19 +257,5 @@ export class HumanFeedbackMechanismSlack extends HumanFeedbackMechanism { default: throw new Error(`Unexpected feedback: ${feedback}`) } - - if (this._options.annotations) { - try { - const annotation = await this.slackClient.sendAndWaitForReply({ - text: 'Please leave an annotation (optional):' - }) - - if (annotation) { - metadata.feedback.annotation = annotation.text - } - } catch (e) { - // Deliberately swallow the error here as the user is not required to leave an annotation - } - } } } diff --git a/src/human-feedback/twilio.ts b/src/human-feedback/twilio.ts index 383f35b..d7cae0c 100644 --- a/src/human-feedback/twilio.ts +++ b/src/human-feedback/twilio.ts @@ -23,6 +23,24 @@ export class HumanFeedbackMechanismTwilio extends HumanFeedbackMechanism { this.twilioClient = new TwilioConversationClient() } + protected async annotate( + response: any, + metadata: TaskResponseMetadata + ): Promise { + try { + const annotation = await this.twilioClient.sendAndWaitForReply({ + name: 'human-feedback-annotation', + text: 'Please leave an annotation (optional):' + }) + + if (annotation) { + metadata.feedback.annotation = annotation.body + } + } catch (e) { + // Deliberately swallow the error here as the user is not required to leave an annotation + } + } + private async askUser( message: string, choices: UserActions[] @@ -98,21 +116,6 @@ export class HumanFeedbackMechanismTwilio extends HumanFeedbackMechanism { default: throw new Error(`Unexpected feedback: ${feedback}`) } - - if (this._options.annotations) { - try { - const annotation = await this.twilioClient.sendAndWaitForReply({ - name: 'human-feedback-annotation', - text: 'Please leave an annotation (optional):' - }) - - if (annotation) { - metadata.feedback.annotation = annotation.body - } - } catch (e) { - // Deliberately swallow the error here as the user is not required to leave an annotation - } - } } public async selectOne( @@ -181,21 +184,6 @@ export class HumanFeedbackMechanismTwilio extends HumanFeedbackMechanism { default: throw new Error(`Unexpected feedback: ${feedback}`) } - - if (this._options.annotations) { - try { - const annotation = await this.twilioClient.sendAndWaitForReply({ - text: 'Please leave an annotation (optional):', - name: 'human-feedback-annotation' - }) - - if (annotation) { - metadata.feedback.annotation = annotation.body - } - } catch (e) { - // Deliberately swallow the error here as the user is not required to leave an annotation - } - } } public async selectN( @@ -275,20 +263,5 @@ export class HumanFeedbackMechanismTwilio extends HumanFeedbackMechanism { default: throw new Error(`Unexpected feedback: ${feedback}`) } - - if (this._options.annotations) { - try { - const annotation = await this.twilioClient.sendAndWaitForReply({ - text: 'Please leave an annotation (optional):', - name: 'human-feedback-annotation' - }) - - if (annotation) { - metadata.feedback.annotation = annotation.body - } - } catch (e) { - // Deliberately swallow the error here as the user is not required to leave an annotation - } - } } } From 9c41ea56076f7b53e5ed28a7ce108a9f0cd5ea2b Mon Sep 17 00:00:00 2001 From: Philipp Burckhardt Date: Tue, 13 Jun 2023 16:32:58 -0400 Subject: [PATCH 05/22] refactor: avoid code duplication --- src/human-feedback/cli.ts | 214 +++++---------------------- src/human-feedback/feedback.ts | 103 ++++++++++--- src/human-feedback/slack.ts | 250 ++++++------------------------- src/human-feedback/twilio.ts | 259 +++++++-------------------------- 4 files changed, 216 insertions(+), 610 deletions(-) diff --git a/src/human-feedback/cli.ts b/src/human-feedback/cli.ts index ad5f263..e4a23e0 100644 --- a/src/human-feedback/cli.ts +++ b/src/human-feedback/cli.ts @@ -4,7 +4,6 @@ import input from '@inquirer/input' import select from '@inquirer/select' import { Agentic } from '@/agentic' -import { TaskResponseMetadata } from '@/types' import { HumanFeedbackMechanism, @@ -13,22 +12,6 @@ import { UserActions } from './feedback' -/** - * Prompt the user to select one of a list of options. - */ -async function askUser( - message: string, - choices: UserActions[] -): Promise { - return select({ - message, - choices: choices.map((choice) => ({ - name: UserActionMessages[choice], - value: choice - })) - }) -} - export class HumanFeedbackMechanismCLI extends HumanFeedbackMechanism { constructor({ agentic, @@ -42,172 +25,49 @@ export class HumanFeedbackMechanismCLI extends HumanFeedbackMechanism { this._options = options } - protected async annotate( - response: any, - metadata: TaskResponseMetadata - ): Promise { - const annotation = await input({ + /** + * Prompt the user to select one of a list of options. + */ + protected async askUser( + message: string, + choices: UserActions[] + ): Promise { + return select({ + message, + choices: choices.map((choice) => ({ + name: UserActionMessages[choice], + value: choice + })) + }) + } + + protected async edit(output: string): Promise { + return editor({ + message: 'Edit the output:', + default: output + }) + } + + protected async annotate(): Promise { + return input({ message: 'Please leave an annotation (leave blank to skip; press enter to submit):' }) - - if (annotation) { - metadata.feedback.annotation = annotation - } } - protected async confirm( - response: any, - metadata: TaskResponseMetadata - ): Promise { - const stringified = JSON.stringify(response, null, 2) - const msg = [ - 'The following output was generated:', - stringified, - 'What would you like to do?' - ].join('\n') - const choices: UserActions[] = [UserActions.Accept, UserActions.Decline] - - if (this._options.editing) { - choices.push(UserActions.Edit) - } - - if (this._options.bail) { - choices.push(UserActions.Exit) - } - - const feedback = await askUser(msg, choices) - - metadata.feedback = {} - - switch (feedback) { - case UserActions.Accept: - metadata.feedback.accepted = true - break - - case UserActions.Edit: { - const editedOutput = await editor({ - message: 'Edit the output:', - default: stringified - }) - metadata.feedback.editedOutput = editedOutput - break - } - - case UserActions.Decline: - metadata.feedback.accepted = false - break - - case UserActions.Exit: - throw new Error('Exiting...') - - default: - throw new Error(`Unexpected feedback: ${feedback}`) - } + protected async selectOne(response: any[]): Promise { + const choices = response.map((option) => ({ + name: option, + value: option + })) + return select({ message: 'Pick one output:', choices }) } - protected async selectOne( - response: any[], - metadata: TaskResponseMetadata - ): Promise { - const stringified = JSON.stringify(response, null, 2) - const msg = [ - 'The following output was generated:', - stringified, - 'What would you like to do?' - ].join('\n') - const choices: UserActions[] = [UserActions.Select] - - if (this._options.editing) { - choices.push(UserActions.Edit) - } - - if (this._options.bail) { - choices.push(UserActions.Exit) - } - - const feedback = await askUser(msg, choices) - - metadata.feedback = {} - - switch (feedback) { - case UserActions.Edit: { - const editedOutput = await editor({ - message: 'Edit the output:', - default: stringified - }) - metadata.feedback.editedOutput = editedOutput - break - } - - case UserActions.Select: { - const choices = response.map((option) => ({ - name: option, - value: option - })) - const chosen = await select({ message: 'Pick one output:', choices }) - metadata.feedback.chosen = chosen - break - } - - case UserActions.Exit: - throw new Error('Exiting...') - - default: - throw new Error(`Unexpected feedback: ${feedback}`) - } - } - - protected async selectN( - response: any[], - metadata: TaskResponseMetadata - ): Promise { - const stringified = JSON.stringify(response, null, 2) - const msg = [ - 'The following output was generated:', - stringified, - 'What would you like to do?' - ].join('\n') - const choices: UserActions[] = [UserActions.Select] - - if (this._options.editing) { - choices.push(UserActions.Edit) - } - - if (this._options.bail) { - choices.push(UserActions.Exit) - } - - const feedback = - choices.length === 1 ? UserActions.Select : await askUser(msg, choices) - - metadata.feedback = {} - - switch (feedback) { - case UserActions.Edit: { - const editedOutput = await editor({ - message: 'Edit the output:', - default: stringified - }) - metadata.feedback.editedOutput = editedOutput - break - } - - case UserActions.Select: { - const choices = response.map((option) => ({ - name: option, - value: option - })) - const chosen = await checkbox({ message: 'Select outputs:', choices }) - metadata.feedback.selected = chosen - break - } - - case UserActions.Exit: - throw new Error('Exiting...') - - default: - throw new Error(`Unexpected feedback: ${feedback}`) - } + protected async selectN(response: any[]): Promise { + const choices = response.map((option) => ({ + name: option, + value: option + })) + return checkbox({ message: 'Select outputs:', choices }) } } diff --git a/src/human-feedback/feedback.ts b/src/human-feedback/feedback.ts index 4619a9d..27d12f7 100644 --- a/src/human-feedback/feedback.ts +++ b/src/human-feedback/feedback.ts @@ -82,35 +82,92 @@ export abstract class HumanFeedbackMechanism { this._options = options } - protected abstract confirm( - response: any, - metadata: TaskResponseMetadata - ): Promise - protected abstract selectOne( - response: any, - metadata: TaskResponseMetadata - ): Promise - protected abstract selectN( - response: any, - metadata: TaskResponseMetadata - ): Promise + protected abstract selectOne(response: any): Promise - protected abstract annotate( - response: any, - metadata: TaskResponseMetadata - ): Promise + protected abstract selectN(response: any): Promise + + protected abstract annotate(): Promise + + protected abstract edit(output: string): Promise + + protected abstract askUser( + message: string, + choices: UserActions[] + ): Promise public async interact(response: any, metadata: TaskResponseMetadata) { - if (this._options.type === 'selectN') { - await this.selectN(response, metadata) - } else if (this._options.type === 'confirm') { - await this.confirm(response, metadata) - } else if (this._options.type === 'selectOne') { - await this.selectOne(response, metadata) + const stringified = JSON.stringify(response, null, 2) + const msg = [ + 'The following output was generated:', + '```', + stringified, + '```', + 'What would you like to do?' + ].join('\n') + + const choices: UserActions[] = [] + if ( + this._options.type === 'selectN' || + this._options.type === 'selectOne' + ) { + choices.push(UserActions.Select) + } else { + // Case: confirm + choices.push(UserActions.Accept) + choices.push(UserActions.Decline) + } + + if (this._options.editing) { + choices.push(UserActions.Edit) + } + + if (this._options.bail) { + choices.push(UserActions.Exit) + } + + const feedback = + choices.length === 1 + ? UserActions.Select + : await this.askUser(msg, choices) + + metadata.feedback = {} + + switch (feedback) { + case UserActions.Accept: + metadata.feedback.accepted = true + break + + case UserActions.Edit: { + const editedOutput = await this.edit(stringified) + metadata.feedback.editedOutput = editedOutput + break + } + + case UserActions.Decline: + metadata.feedback.accepted = false + break + + case UserActions.Select: + if (this._options.type === 'selectN') { + metadata.feedback.selected = await this.selectN(response) + } else if (this._options.type === 'selectOne') { + metadata.feedback.chosen = await this.selectOne(response) + } + + break + + case UserActions.Exit: + throw new Error('Exiting...') + + default: + throw new Error(`Unexpected feedback: ${feedback}`) } if (this._options.annotations) { - await this.annotate(response, metadata) + const annotation = await this.annotate() + if (annotation) { + metadata.feedback.annotation = annotation + } } } } diff --git a/src/human-feedback/slack.ts b/src/human-feedback/slack.ts index e8172d1..7464036 100644 --- a/src/human-feedback/slack.ts +++ b/src/human-feedback/slack.ts @@ -1,6 +1,5 @@ import { Agentic } from '@/agentic' import { SlackClient } from '@/services/slack' -import { TaskResponseMetadata } from '@/types' import { HumanFeedbackMechanism, @@ -23,24 +22,28 @@ export class HumanFeedbackMechanismSlack extends HumanFeedbackMechanism { this.slackClient = new SlackClient() } - protected async annotate( - response: any, - metadata: TaskResponseMetadata - ): Promise { + protected async annotate(): Promise { try { const annotation = await this.slackClient.sendAndWaitForReply({ text: 'Please leave an annotation (optional):' }) - - if (annotation) { - metadata.feedback.annotation = annotation.text - } + return annotation.text } catch (e) { // Deliberately swallow the error here as the user is not required to leave an annotation + return '' } } - private async askUser( + protected async edit(): Promise { + let { text: editedOutput } = await this.slackClient.sendAndWaitForReply({ + text: 'Copy and edit the output:' + }) + editedOutput = editedOutput.replace(/```$/g, '') + editedOutput = editedOutput.replace(/^```/g, '') + return editedOutput + } + + protected async askUser( message: string, choices: UserActions[] ): Promise { @@ -60,202 +63,45 @@ export class HumanFeedbackMechanismSlack extends HumanFeedbackMechanism { return choices[parseInt(response.text)] } - public async confirm( - response: any, - metadata: TaskResponseMetadata - ): Promise { - const stringified = JSON.stringify(response, null, 2) - const msg = [ - 'The following output was generated:', - '```', - stringified, - '```', - 'What would you like to do?' - ].join('\n') - - const choices: UserActions[] = [UserActions.Accept, UserActions.Decline] - - if (this._options.editing) { - choices.push(UserActions.Edit) - } - - if (this._options.bail) { - choices.push(UserActions.Exit) - } - - const feedback = await this.askUser(msg, choices) - - metadata.feedback = {} - - switch (feedback) { - case UserActions.Accept: - metadata.feedback.accepted = true - break - - case UserActions.Edit: { - let { text: editedOutput } = await this.slackClient.sendAndWaitForReply( - { - text: 'Copy and edit the output:' - } - ) - editedOutput = editedOutput.replace(/```$/g, '') - editedOutput = editedOutput.replace(/^```/g, '') - metadata.feedback.editedOutput = editedOutput - break + public async selectOne(response: any[]): Promise { + const { text: selectedOutput } = await this.slackClient.sendAndWaitForReply( + { + text: + 'Pick one output:' + + response.map((r, idx) => `\n*${idx}* - ${r}`).join('') + + '\n\nReply with the number of your choice.', + validate: (slackMessage) => { + const choice = parseInt(slackMessage.text) + return !isNaN(choice) && choice >= 0 && choice < response.length + } } - - case UserActions.Decline: - metadata.feedback.accepted = false - break - - case UserActions.Exit: - throw new Error('Exiting...') - - default: - throw new Error(`Unexpected feedback: ${feedback}`) - } + ) + return response[parseInt(selectedOutput)] } - public async selectOne( - response: any[], - metadata: TaskResponseMetadata - ): Promise { - const stringified = JSON.stringify(response, null, 2) - const msg = [ - 'The following output was generated:', - '```', - stringified, - '```', - 'What would you like to do?' - ].join('\n') - - const choices: UserActions[] = [UserActions.Select] - - if (this._options.editing) { - choices.push(UserActions.Edit) - } - - if (this._options.bail) { - choices.push(UserActions.Exit) - } - - const feedback = - choices.length === 1 - ? UserActions.Select - : await this.askUser(msg, choices) - - metadata.feedback = {} - - switch (feedback) { - case UserActions.Edit: { - let { text: editedOutput } = await this.slackClient.sendAndWaitForReply( - { - text: 'Copy and edit the output:' - } - ) - editedOutput = editedOutput.replace(/```$/g, '') - editedOutput = editedOutput.replace(/^```/g, '') - metadata.feedback.editedOutput = editedOutput - break - } - - case UserActions.Select: { - const { text: selectedOutput } = - await this.slackClient.sendAndWaitForReply({ - text: - 'Pick one output:' + - response.map((r, idx) => `\n*${idx}* - ${r}`).join('') + - '\n\nReply with the number of your choice.', - validate: (slackMessage) => { - const choice = parseInt(slackMessage.text) - return !isNaN(choice) && choice >= 0 && choice < response.length - } + public async selectN(response: any[]): Promise { + const { text: selectedOutput } = await this.slackClient.sendAndWaitForReply( + { + text: + 'Select outputs:' + + response.map((r, idx) => `\n*${idx}* - ${r}`).join('') + + '\n\nReply with a comma-separated list of the output numbers of your choice.', + validate: (slackMessage) => { + const choices = slackMessage.text.split(',') + return choices.every((choice) => { + const choiceInt = parseInt(choice) + return ( + !isNaN(choiceInt) && choiceInt >= 0 && choiceInt < response.length + ) }) - metadata.feedback.chosen = response[parseInt(selectedOutput)] - break + } } - - case UserActions.Exit: - throw new Error('Exiting...') - - default: - throw new Error(`Unexpected feedback: ${feedback}`) - } - } - - public async selectN( - response: any[], - metadata: TaskResponseMetadata - ): Promise { - const stringified = JSON.stringify(response, null, 2) - const msg = [ - 'The following output was generated:', - stringified, - 'What would you like to do?' - ].join('\n') - const choices: UserActions[] = [UserActions.Select] - - if (this._options.editing) { - choices.push(UserActions.Edit) - } - - if (this._options.bail) { - choices.push(UserActions.Exit) - } - - const feedback = - choices.length === 1 - ? UserActions.Select - : await this.askUser(msg, choices) - - metadata.feedback = {} - - switch (feedback) { - case UserActions.Edit: { - let { text: editedOutput } = await this.slackClient.sendAndWaitForReply( - { - text: 'Copy and edit the output:' - } - ) - editedOutput = editedOutput.replace(/```$/g, '') - editedOutput = editedOutput.replace(/^```/g, '') - metadata.feedback.editedOutput = editedOutput - break - } - - case UserActions.Select: { - const { text: selectedOutput } = - await this.slackClient.sendAndWaitForReply({ - text: - 'Select outputs:' + - response.map((r, idx) => `\n*${idx}* - ${r}`).join('') + - '\n\nReply with a comma-separated list of the output numbers of your choice.', - validate: (slackMessage) => { - const choices = slackMessage.text.split(',') - return choices.every((choice) => { - const choiceInt = parseInt(choice) - return ( - !isNaN(choiceInt) && - choiceInt >= 0 && - choiceInt < response.length - ) - }) - } - }) - const chosenOutputs = selectedOutput - .split(',') - .map((choice) => parseInt(choice)) - metadata.feedback.selected = response.filter((_, idx) => { - return chosenOutputs.includes(idx) - }) - break - } - - case UserActions.Exit: - throw new Error('Exiting...') - - default: - throw new Error(`Unexpected feedback: ${feedback}`) - } + ) + const chosenOutputs = selectedOutput + .split(',') + .map((choice) => parseInt(choice)) + return response.filter((_, idx) => { + return chosenOutputs.includes(idx) + }) } } diff --git a/src/human-feedback/twilio.ts b/src/human-feedback/twilio.ts index d7cae0c..753578b 100644 --- a/src/human-feedback/twilio.ts +++ b/src/human-feedback/twilio.ts @@ -1,6 +1,5 @@ import { Agentic } from '@/agentic' import { TwilioConversationClient } from '@/services/twilio-conversation' -import { TaskResponseMetadata } from '@/types' import { HumanFeedbackMechanism, @@ -23,25 +22,30 @@ export class HumanFeedbackMechanismTwilio extends HumanFeedbackMechanism { this.twilioClient = new TwilioConversationClient() } - protected async annotate( - response: any, - metadata: TaskResponseMetadata - ): Promise { + protected async annotate(): Promise { try { const annotation = await this.twilioClient.sendAndWaitForReply({ name: 'human-feedback-annotation', text: 'Please leave an annotation (optional):' }) - - if (annotation) { - metadata.feedback.annotation = annotation.body - } + return annotation.body } catch (e) { // Deliberately swallow the error here as the user is not required to leave an annotation + return '' } } - private async askUser( + protected async edit(): Promise { + let { body: editedOutput } = await this.twilioClient.sendAndWaitForReply({ + text: 'Copy and edit the output:', + name: 'human-feedback-edit' + }) + editedOutput = editedOutput.replace(/```$/g, '') + editedOutput = editedOutput.replace(/^```/g, '') + return editedOutput + } + + protected async askUser( message: string, choices: UserActions[] ): Promise { @@ -62,206 +66,45 @@ export class HumanFeedbackMechanismTwilio extends HumanFeedbackMechanism { return choices[parseInt(response.body)] } - public async confirm( - response: any, - metadata: TaskResponseMetadata - ): Promise { - const stringified = JSON.stringify(response, null, 2) - const msg = [ - 'The following output was generated:', - '', - stringified, - '', - 'What would you like to do?' - ].join('\n') - - const choices: UserActions[] = [UserActions.Accept, UserActions.Decline] - - if (this._options.editing) { - choices.push(UserActions.Edit) - } - - if (this._options.bail) { - choices.push(UserActions.Exit) - } - - const feedback = await this.askUser(msg, choices) - - metadata.feedback = {} - - switch (feedback) { - case UserActions.Accept: - metadata.feedback.accepted = true - break - - case UserActions.Edit: { - let { body: editedOutput } = - await this.twilioClient.sendAndWaitForReply({ - name: 'human-feedback-edit', - text: 'Copy and edit the output:' - }) - editedOutput = editedOutput.replace(/```$/g, '') - editedOutput = editedOutput.replace(/^```/g, '') - metadata.feedback.editedOutput = editedOutput - break - } - - case UserActions.Decline: - metadata.feedback.accepted = false - break - - case UserActions.Exit: - throw new Error('Exiting...') - - default: - throw new Error(`Unexpected feedback: ${feedback}`) - } + public async selectOne(response: any[]): Promise { + const { body: selectedOutput } = + await this.twilioClient.sendAndWaitForReply({ + name: 'human-feedback-select', + text: + 'Pick one output:' + + response.map((r, idx) => `\n${idx} - ${r}`).join('') + + '\n\nReply with the number of your choice.', + validate: (message) => { + const choice = parseInt(message.body) + return !isNaN(choice) && choice >= 0 && choice < response.length + } + }) + return response[parseInt(selectedOutput)] } - public async selectOne( - response: any[], - metadata: TaskResponseMetadata - ): Promise { - const stringified = JSON.stringify(response, null, 2) - const msg = [ - 'The following output was generated:', - '', - stringified, - '', - 'What would you like to do?' - ].join('\n') - - const choices: UserActions[] = [UserActions.Select] - - if (this._options.editing) { - choices.push(UserActions.Edit) - } - - if (this._options.bail) { - choices.push(UserActions.Exit) - } - - const feedback = - choices.length === 1 - ? UserActions.Select - : await this.askUser(msg, choices) - - metadata.feedback = {} - - switch (feedback) { - case UserActions.Edit: { - let { body: editedOutput } = - await this.twilioClient.sendAndWaitForReply({ - text: 'Copy and edit the output:', - name: 'human-feedback-edit' + public async selectN(response: any[]): Promise { + const { body: selectedOutput } = + await this.twilioClient.sendAndWaitForReply({ + name: 'human-feedback-select', + text: + 'Select outputs:' + + response.map((r, idx) => `\n${idx} - ${r}`).join('') + + '\n\nReply with a comma-separated list of the output numbers of your choice.', + validate: (message) => { + const choices = message.body.split(',') + return choices.every((choice) => { + const choiceInt = parseInt(choice) + return ( + !isNaN(choiceInt) && choiceInt >= 0 && choiceInt < response.length + ) }) - editedOutput = editedOutput.replace(/```$/g, '') - editedOutput = editedOutput.replace(/^```/g, '') - metadata.feedback.editedOutput = editedOutput - break - } - - case UserActions.Select: { - const { body: selectedOutput } = - await this.twilioClient.sendAndWaitForReply({ - name: 'human-feedback-select', - text: - 'Pick one output:' + - response.map((r, idx) => `\n${idx} - ${r}`).join('') + - '\n\nReply with the number of your choice.', - validate: (message) => { - const choice = parseInt(message.body) - return !isNaN(choice) && choice >= 0 && choice < response.length - } - }) - metadata.feedback.chosen = response[parseInt(selectedOutput)] - break - } - - case UserActions.Exit: - throw new Error('Exiting...') - - default: - throw new Error(`Unexpected feedback: ${feedback}`) - } - } - - public async selectN( - response: any[], - metadata: TaskResponseMetadata - ): Promise { - const stringified = JSON.stringify(response, null, 2) - const msg = [ - 'The following output was generated:', - '', - stringified, - '', - 'What would you like to do?' - ].join('\n') - const choices: UserActions[] = [UserActions.Select] - - if (this._options.editing) { - choices.push(UserActions.Edit) - } - - if (this._options.bail) { - choices.push(UserActions.Exit) - } - - const feedback = - choices.length === 1 - ? UserActions.Select - : await this.askUser(msg, choices) - - metadata.feedback = {} - - switch (feedback) { - case UserActions.Edit: { - let { body: editedOutput } = - await this.twilioClient.sendAndWaitForReply({ - text: 'Copy and edit the output:', - name: 'human-feedback-edit' - }) - editedOutput = editedOutput.replace(/```$/g, '') - editedOutput = editedOutput.replace(/^```/g, '') - metadata.feedback.editedOutput = editedOutput - break - } - - case UserActions.Select: { - const { body: selectedOutput } = - await this.twilioClient.sendAndWaitForReply({ - name: 'human-feedback-select', - text: - 'Select outputs:' + - response.map((r, idx) => `\n${idx} - ${r}`).join('') + - '\n\nReply with a comma-separated list of the output numbers of your choice.', - validate: (message) => { - const choices = message.body.split(',') - return choices.every((choice) => { - const choiceInt = parseInt(choice) - return ( - !isNaN(choiceInt) && - choiceInt >= 0 && - choiceInt < response.length - ) - }) - } - }) - const chosenOutputs = selectedOutput - .split(',') - .map((choice) => parseInt(choice)) - metadata.feedback.selected = response.filter((_, idx) => { - return chosenOutputs.includes(idx) - }) - break - } - - case UserActions.Exit: - throw new Error('Exiting...') - - default: - throw new Error(`Unexpected feedback: ${feedback}`) - } + } + }) + const chosenOutputs = selectedOutput + .split(',') + .map((choice) => parseInt(choice)) + return response.filter((_, idx) => { + return chosenOutputs.includes(idx) + }) } } From a84b7b191c8ad5852bfcfd2df6a9097e5ae32771 Mon Sep 17 00:00:00 2001 From: Philipp Burckhardt Date: Tue, 13 Jun 2023 17:25:13 -0400 Subject: [PATCH 06/22] fix: remove asterisks for SMS --- src/human-feedback/twilio.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/human-feedback/twilio.ts b/src/human-feedback/twilio.ts index 753578b..f69ffdc 100644 --- a/src/human-feedback/twilio.ts +++ b/src/human-feedback/twilio.ts @@ -51,7 +51,7 @@ export class HumanFeedbackMechanismTwilio extends HumanFeedbackMechanism { ): Promise { message += '\n\n' message += choices - .map((choice, idx) => `*${idx}* - ${UserActionMessages[choice]}`) + .map((choice, idx) => `${idx} - ${UserActionMessages[choice]}`) .join('\n') message += '\n\n' message += 'Reply with the number of your choice.' From e722c5b9d7756a2a8e40cd43084c07a1ba5c9281 Mon Sep 17 00:00:00 2001 From: Philipp Burckhardt Date: Tue, 13 Jun 2023 18:14:12 -0400 Subject: [PATCH 07/22] fix: improve typing --- src/human-feedback/cli.ts | 20 ++---- src/human-feedback/feedback.ts | 110 +++++++++++++++++++++++++-------- src/human-feedback/slack.ts | 7 ++- src/human-feedback/twilio.ts | 7 ++- 4 files changed, 98 insertions(+), 46 deletions(-) diff --git a/src/human-feedback/cli.ts b/src/human-feedback/cli.ts index e4a23e0..39fac15 100644 --- a/src/human-feedback/cli.ts +++ b/src/human-feedback/cli.ts @@ -3,28 +3,16 @@ import editor from '@inquirer/editor' import input from '@inquirer/input' import select from '@inquirer/select' -import { Agentic } from '@/agentic' - import { HumanFeedbackMechanism, - HumanFeedbackOptions, + HumanFeedbackType, UserActionMessages, UserActions } from './feedback' -export class HumanFeedbackMechanismCLI extends HumanFeedbackMechanism { - constructor({ - agentic, - options - }: { - agentic: Agentic - options: HumanFeedbackOptions - }) { - super({ agentic, options }) - this._agentic = agentic - this._options = options - } - +export class HumanFeedbackMechanismCLI< + T extends HumanFeedbackType +> extends HumanFeedbackMechanism { /** * Prompt the user to select one of a list of options. */ diff --git a/src/human-feedback/feedback.ts b/src/human-feedback/feedback.ts index 27d12f7..7d1e69d 100644 --- a/src/human-feedback/feedback.ts +++ b/src/human-feedback/feedback.ts @@ -1,6 +1,5 @@ import { Agentic } from '@/agentic' import { BaseTask } from '@/task' -import { TaskResponseMetadata } from '@/types' import { HumanFeedbackMechanismCLI } from './cli' @@ -30,20 +29,18 @@ export const UserActionMessages: Record = { */ export type HumanFeedbackType = 'confirm' | 'selectOne' | 'selectN' -type HumanFeedbackMechanismConstructor = new ( +type HumanFeedbackMechanismConstructor = new ( ...args: any[] -) => T +) => HumanFeedbackMechanism /** * Options for human feedback. */ -export type HumanFeedbackOptions< - T extends HumanFeedbackMechanism = HumanFeedbackMechanism -> = { +export type HumanFeedbackOptions = { /** * What type of feedback to request. */ - type?: HumanFeedbackType + type?: T /** * Whether the user can bail out of the feedback loop. @@ -66,17 +63,75 @@ export type HumanFeedbackOptions< mechanism?: HumanFeedbackMechanismConstructor } -export abstract class HumanFeedbackMechanism { +export interface BaseHumanFeedbackMetadata { + /** + * Edited output by the user (if applicable). + */ + editedOutput?: string + + /** + * Annotation left by the user (if applicable). + */ + annotation?: string +} + +export interface HumanFeedbackConfirmMetadata + extends BaseHumanFeedbackMetadata { + /** + * The type of feedback requested. + */ + type: 'confirm' + + /** + * Whether the user accepted the output. + */ + accepted: boolean +} + +export interface HumanFeedbackSelectOneMetadata + extends BaseHumanFeedbackMetadata { + /** + * The type of feedback requested. + */ + type: 'selectOne' + + /** + * The selected output. + */ + chosen: any +} + +export interface HumanFeedbackSelectNMetadata + extends BaseHumanFeedbackMetadata { + /** + * The type of feedback requested. + */ + type: 'selectN' + + /** + * The selected outputs. + */ + selected: any[] +} + +export type FeedbackTypeToMetadata = + T extends 'confirm' + ? HumanFeedbackConfirmMetadata + : T extends 'selectOne' + ? HumanFeedbackSelectOneMetadata + : HumanFeedbackSelectNMetadata + +export abstract class HumanFeedbackMechanism { protected _agentic: Agentic - protected _options: HumanFeedbackOptions + protected _options: Required> constructor({ agentic, options }: { agentic: Agentic - options: HumanFeedbackOptions + options: Required> }) { this._agentic = agentic this._options = options @@ -95,7 +150,7 @@ export abstract class HumanFeedbackMechanism { choices: UserActions[] ): Promise - public async interact(response: any, metadata: TaskResponseMetadata) { + public async interact(response: any): Promise> { const stringified = JSON.stringify(response, null, 2) const msg = [ 'The following output was generated:', @@ -125,33 +180,33 @@ export abstract class HumanFeedbackMechanism { choices.push(UserActions.Exit) } - const feedback = + const choice = choices.length === 1 ? UserActions.Select : await this.askUser(msg, choices) - metadata.feedback = {} + const feedback: Record = {} - switch (feedback) { + switch (choice) { case UserActions.Accept: - metadata.feedback.accepted = true + feedback.accepted = true break case UserActions.Edit: { const editedOutput = await this.edit(stringified) - metadata.feedback.editedOutput = editedOutput + feedback.editedOutput = editedOutput break } case UserActions.Decline: - metadata.feedback.accepted = false + feedback.accepted = false break case UserActions.Select: if (this._options.type === 'selectN') { - metadata.feedback.selected = await this.selectN(response) + feedback.selected = await this.selectN(response) } else if (this._options.type === 'selectOne') { - metadata.feedback.chosen = await this.selectOne(response) + feedback.chosen = await this.selectOne(response) } break @@ -160,21 +215,23 @@ export abstract class HumanFeedbackMechanism { throw new Error('Exiting...') default: - throw new Error(`Unexpected feedback: ${feedback}`) + throw new Error(`Unexpected choice: ${choice}`) } if (this._options.annotations) { const annotation = await this.annotate() if (annotation) { - metadata.feedback.annotation = annotation + feedback.annotation = annotation } } + + return feedback as FeedbackTypeToMetadata } } -export function withHumanFeedback( +export function withHumanFeedback( task: BaseTask, - options: HumanFeedbackOptions = {} + options: HumanFeedbackOptions = {} ) { task = task.clone() @@ -182,7 +239,7 @@ export function withHumanFeedback( const instanceDefaults = task.agentic.humanFeedbackDefaults // Use Object.assign to merge the options, instance defaults, and hard-coded defaults - const finalOptions: HumanFeedbackOptions = Object.assign( + const finalOptions: HumanFeedbackOptions = Object.assign( { type: 'confirm', bail: false, @@ -212,8 +269,9 @@ export function withHumanFeedback( task.callWithMetadata = async function (input?: T) { const response = await originalCall(input) - // Process the response and add feedback to metadata - await feedbackMechanism.interact(response.result, response.metadata) + const feedback = await feedbackMechanism.interact(response.result) + + response.metadata = { ...response.metadata, feedback } return response } diff --git a/src/human-feedback/slack.ts b/src/human-feedback/slack.ts index 7464036..d344001 100644 --- a/src/human-feedback/slack.ts +++ b/src/human-feedback/slack.ts @@ -4,11 +4,14 @@ import { SlackClient } from '@/services/slack' import { HumanFeedbackMechanism, HumanFeedbackOptions, + HumanFeedbackType, UserActionMessages, UserActions } from './feedback' -export class HumanFeedbackMechanismSlack extends HumanFeedbackMechanism { +export class HumanFeedbackMechanismSlack< + T extends HumanFeedbackType +> extends HumanFeedbackMechanism { private slackClient: SlackClient constructor({ @@ -16,7 +19,7 @@ export class HumanFeedbackMechanismSlack extends HumanFeedbackMechanism { options }: { agentic: Agentic - options: HumanFeedbackOptions + options: Required> }) { super({ agentic, options }) this.slackClient = new SlackClient() diff --git a/src/human-feedback/twilio.ts b/src/human-feedback/twilio.ts index f69ffdc..45ae8cb 100644 --- a/src/human-feedback/twilio.ts +++ b/src/human-feedback/twilio.ts @@ -4,11 +4,14 @@ import { TwilioConversationClient } from '@/services/twilio-conversation' import { HumanFeedbackMechanism, HumanFeedbackOptions, + HumanFeedbackType, UserActionMessages, UserActions } from './feedback' -export class HumanFeedbackMechanismTwilio extends HumanFeedbackMechanism { +export class HumanFeedbackMechanismTwilio< + T extends HumanFeedbackType +> extends HumanFeedbackMechanism { private twilioClient: TwilioConversationClient constructor({ @@ -16,7 +19,7 @@ export class HumanFeedbackMechanismTwilio extends HumanFeedbackMechanism { options }: { agentic: Agentic - options: HumanFeedbackOptions + options: Required> }) { super({ agentic, options }) this.twilioClient = new TwilioConversationClient() From feb95a26650e859c58bee74db0695f7fd87aeb5f Mon Sep 17 00:00:00 2001 From: Philipp Burckhardt Date: Tue, 13 Jun 2023 18:16:29 -0400 Subject: [PATCH 08/22] refactor: rename symbols --- src/human-feedback/cli.ts | 10 +++---- src/human-feedback/feedback.ts | 48 ++++++++++++++++++---------------- src/human-feedback/slack.ts | 12 +++++---- src/human-feedback/twilio.ts | 12 +++++---- 4 files changed, 45 insertions(+), 37 deletions(-) diff --git a/src/human-feedback/cli.ts b/src/human-feedback/cli.ts index 39fac15..ad795bf 100644 --- a/src/human-feedback/cli.ts +++ b/src/human-feedback/cli.ts @@ -6,8 +6,8 @@ import select from '@inquirer/select' import { HumanFeedbackMechanism, HumanFeedbackType, - UserActionMessages, - UserActions + HumanFeedbackUserActionMessages, + HumanFeedbackUserActions } from './feedback' export class HumanFeedbackMechanismCLI< @@ -18,12 +18,12 @@ export class HumanFeedbackMechanismCLI< */ protected async askUser( message: string, - choices: UserActions[] - ): Promise { + choices: HumanFeedbackUserActions[] + ): Promise { return select({ message, choices: choices.map((choice) => ({ - name: UserActionMessages[choice], + name: HumanFeedbackUserActionMessages[choice], value: choice })) }) diff --git a/src/human-feedback/feedback.ts b/src/human-feedback/feedback.ts index 7d1e69d..f53923d 100644 --- a/src/human-feedback/feedback.ts +++ b/src/human-feedback/feedback.ts @@ -6,7 +6,7 @@ import { HumanFeedbackMechanismCLI } from './cli' /** * Actions the user can take in the feedback selection prompt. */ -export const UserActions = { +export const HumanFeedbackUserActions = { Accept: 'accept', Edit: 'edit', Decline: 'decline', @@ -14,14 +14,18 @@ export const UserActions = { Exit: 'exit' } as const -export type UserActions = (typeof UserActions)[keyof typeof UserActions] +export type HumanFeedbackUserActions = + (typeof HumanFeedbackUserActions)[keyof typeof HumanFeedbackUserActions] -export const UserActionMessages: Record = { - [UserActions.Accept]: 'Accept the output', - [UserActions.Edit]: 'Edit the output', - [UserActions.Decline]: 'Decline the output', - [UserActions.Select]: 'Select outputs to keep', - [UserActions.Exit]: 'Exit' +export const HumanFeedbackUserActionMessages: Record< + HumanFeedbackUserActions, + string +> = { + [HumanFeedbackUserActions.Accept]: 'Accept the output', + [HumanFeedbackUserActions.Edit]: 'Edit the output', + [HumanFeedbackUserActions.Decline]: 'Decline the output', + [HumanFeedbackUserActions.Select]: 'Select outputs to keep', + [HumanFeedbackUserActions.Exit]: 'Exit' } /** @@ -147,8 +151,8 @@ export abstract class HumanFeedbackMechanism { protected abstract askUser( message: string, - choices: UserActions[] - ): Promise + choices: HumanFeedbackUserActions[] + ): Promise public async interact(response: any): Promise> { const stringified = JSON.stringify(response, null, 2) @@ -160,49 +164,49 @@ export abstract class HumanFeedbackMechanism { 'What would you like to do?' ].join('\n') - const choices: UserActions[] = [] + const choices: HumanFeedbackUserActions[] = [] if ( this._options.type === 'selectN' || this._options.type === 'selectOne' ) { - choices.push(UserActions.Select) + choices.push(HumanFeedbackUserActions.Select) } else { // Case: confirm - choices.push(UserActions.Accept) - choices.push(UserActions.Decline) + choices.push(HumanFeedbackUserActions.Accept) + choices.push(HumanFeedbackUserActions.Decline) } if (this._options.editing) { - choices.push(UserActions.Edit) + choices.push(HumanFeedbackUserActions.Edit) } if (this._options.bail) { - choices.push(UserActions.Exit) + choices.push(HumanFeedbackUserActions.Exit) } const choice = choices.length === 1 - ? UserActions.Select + ? HumanFeedbackUserActions.Select : await this.askUser(msg, choices) const feedback: Record = {} switch (choice) { - case UserActions.Accept: + case HumanFeedbackUserActions.Accept: feedback.accepted = true break - case UserActions.Edit: { + case HumanFeedbackUserActions.Edit: { const editedOutput = await this.edit(stringified) feedback.editedOutput = editedOutput break } - case UserActions.Decline: + case HumanFeedbackUserActions.Decline: feedback.accepted = false break - case UserActions.Select: + case HumanFeedbackUserActions.Select: if (this._options.type === 'selectN') { feedback.selected = await this.selectN(response) } else if (this._options.type === 'selectOne') { @@ -211,7 +215,7 @@ export abstract class HumanFeedbackMechanism { break - case UserActions.Exit: + case HumanFeedbackUserActions.Exit: throw new Error('Exiting...') default: diff --git a/src/human-feedback/slack.ts b/src/human-feedback/slack.ts index d344001..55e30d3 100644 --- a/src/human-feedback/slack.ts +++ b/src/human-feedback/slack.ts @@ -5,8 +5,8 @@ import { HumanFeedbackMechanism, HumanFeedbackOptions, HumanFeedbackType, - UserActionMessages, - UserActions + HumanFeedbackUserActionMessages, + HumanFeedbackUserActions } from './feedback' export class HumanFeedbackMechanismSlack< @@ -48,11 +48,13 @@ export class HumanFeedbackMechanismSlack< protected async askUser( message: string, - choices: UserActions[] - ): Promise { + choices: HumanFeedbackUserActions[] + ): Promise { message += '\n\n' message += choices - .map((choice, idx) => `*${idx}* - ${UserActionMessages[choice]}`) + .map( + (choice, idx) => `*${idx}* - ${HumanFeedbackUserActionMessages[choice]}` + ) .join('\n') message += '\n\n' message += 'Reply with the number of your choice.' diff --git a/src/human-feedback/twilio.ts b/src/human-feedback/twilio.ts index 45ae8cb..42e4085 100644 --- a/src/human-feedback/twilio.ts +++ b/src/human-feedback/twilio.ts @@ -5,8 +5,8 @@ import { HumanFeedbackMechanism, HumanFeedbackOptions, HumanFeedbackType, - UserActionMessages, - UserActions + HumanFeedbackUserActionMessages, + HumanFeedbackUserActions } from './feedback' export class HumanFeedbackMechanismTwilio< @@ -50,11 +50,13 @@ export class HumanFeedbackMechanismTwilio< protected async askUser( message: string, - choices: UserActions[] - ): Promise { + choices: HumanFeedbackUserActions[] + ): Promise { message += '\n\n' message += choices - .map((choice, idx) => `${idx} - ${UserActionMessages[choice]}`) + .map( + (choice, idx) => `${idx} - ${HumanFeedbackUserActionMessages[choice]}` + ) .join('\n') message += '\n\n' message += 'Reply with the number of your choice.' From bbf3f25d9d96aaa89106a16034c7afe039dad4d1 Mon Sep 17 00:00:00 2001 From: Philipp Burckhardt Date: Tue, 13 Jun 2023 18:31:48 -0400 Subject: [PATCH 09/22] feat: parse output after manual editing --- src/human-feedback/feedback.ts | 20 ++++++++++++++------ src/human-feedback/slack.ts | 8 ++++---- src/human-feedback/twilio.ts | 8 ++++---- 3 files changed, 22 insertions(+), 14 deletions(-) diff --git a/src/human-feedback/feedback.ts b/src/human-feedback/feedback.ts index f53923d..2e9c0b0 100644 --- a/src/human-feedback/feedback.ts +++ b/src/human-feedback/feedback.ts @@ -71,7 +71,7 @@ export interface BaseHumanFeedbackMetadata { /** * Edited output by the user (if applicable). */ - editedOutput?: string + editedOutput?: any /** * Annotation left by the user (if applicable). @@ -128,16 +128,19 @@ export type FeedbackTypeToMetadata = export abstract class HumanFeedbackMechanism { protected _agentic: Agentic + protected _task: BaseTask + protected _options: Required> constructor({ - agentic, + task, options }: { - agentic: Agentic + task: BaseTask options: Required> }) { - this._agentic = agentic + this._agentic = task.agentic + this._task = task this._options = options } @@ -154,6 +157,11 @@ export abstract class HumanFeedbackMechanism { choices: HumanFeedbackUserActions[] ): Promise + protected parseEditedOutput(editedOutput: string): any { + const parsedOutput = JSON.parse(editedOutput) + return this._task.outputSchema.parse(parsedOutput) + } + public async interact(response: any): Promise> { const stringified = JSON.stringify(response, null, 2) const msg = [ @@ -198,7 +206,7 @@ export abstract class HumanFeedbackMechanism { case HumanFeedbackUserActions.Edit: { const editedOutput = await this.edit(stringified) - feedback.editedOutput = editedOutput + feedback.editedOutput = await this.parseEditedOutput(editedOutput) break } @@ -264,7 +272,7 @@ export function withHumanFeedback( } const feedbackMechanism = new finalOptions.mechanism({ - agentic: task.agentic, + task: task, options: finalOptions }) diff --git a/src/human-feedback/slack.ts b/src/human-feedback/slack.ts index 55e30d3..724d01c 100644 --- a/src/human-feedback/slack.ts +++ b/src/human-feedback/slack.ts @@ -1,5 +1,5 @@ -import { Agentic } from '@/agentic' import { SlackClient } from '@/services/slack' +import { BaseTask } from '@/task' import { HumanFeedbackMechanism, @@ -15,13 +15,13 @@ export class HumanFeedbackMechanismSlack< private slackClient: SlackClient constructor({ - agentic, + task, options }: { - agentic: Agentic + task: BaseTask options: Required> }) { - super({ agentic, options }) + super({ task, options }) this.slackClient = new SlackClient() } diff --git a/src/human-feedback/twilio.ts b/src/human-feedback/twilio.ts index 42e4085..e41c739 100644 --- a/src/human-feedback/twilio.ts +++ b/src/human-feedback/twilio.ts @@ -1,5 +1,5 @@ -import { Agentic } from '@/agentic' import { TwilioConversationClient } from '@/services/twilio-conversation' +import { BaseTask } from '@/task' import { HumanFeedbackMechanism, @@ -15,13 +15,13 @@ export class HumanFeedbackMechanismTwilio< private twilioClient: TwilioConversationClient constructor({ - agentic, + task, options }: { - agentic: Agentic + task: BaseTask options: Required> }) { - super({ agentic, options }) + super({ task, options }) this.twilioClient = new TwilioConversationClient() } From 5bddd1f04a9d68d3f23c6f1d16219c8e0465e303 Mon Sep 17 00:00:00 2001 From: Philipp Burckhardt Date: Tue, 13 Jun 2023 20:15:15 -0400 Subject: [PATCH 10/22] fix: update to reflect usage of generics --- src/agentic.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/agentic.ts b/src/agentic.ts index 512a8dc..6a1767e 100644 --- a/src/agentic.ts +++ b/src/agentic.ts @@ -2,7 +2,7 @@ import defaultKy from 'ky' import * as types from './types' import { DEFAULT_OPENAI_MODEL } from './constants' -import { HumanFeedbackOptions } from './human-feedback' +import { HumanFeedbackOptions, HumanFeedbackType } from './human-feedback' import { HumanFeedbackMechanismCLI } from './human-feedback/cli' import { OpenAIChatCompletion } from './llms/openai' import { defaultLogger } from './logger' @@ -20,7 +20,7 @@ export class Agentic { types.BaseLLMOptions, 'provider' | 'model' | 'modelParams' | 'timeoutMs' | 'retryConfig' > - protected _humanFeedbackDefaults: HumanFeedbackOptions + protected _humanFeedbackDefaults: HumanFeedbackOptions protected _idGeneratorFn: types.IDGeneratorFunction protected _id: string @@ -31,7 +31,7 @@ export class Agentic { types.BaseLLMOptions, 'provider' | 'model' | 'modelParams' | 'timeoutMs' | 'retryConfig' > - humanFeedbackDefaults?: HumanFeedbackOptions + humanFeedbackDefaults?: HumanFeedbackOptions idGeneratorFn?: types.IDGeneratorFunction logger?: types.Logger ky?: types.KyInstance From 0798a24fdc3b24cf9f9e9acaccda7588d361caa6 Mon Sep 17 00:00:00 2001 From: Philipp Burckhardt Date: Tue, 13 Jun 2023 21:25:40 -0400 Subject: [PATCH 11/22] feat: improve type safety --- src/human-feedback/cli.ts | 27 ++++++++++++----- src/human-feedback/feedback.ts | 54 ++++++++++++++++++++++------------ src/human-feedback/slack.ts | 25 ++++++++++++---- src/human-feedback/twilio.ts | 25 ++++++++++++---- src/types.ts | 10 +++++-- 5 files changed, 101 insertions(+), 40 deletions(-) diff --git a/src/human-feedback/cli.ts b/src/human-feedback/cli.ts index ad795bf..cacd22d 100644 --- a/src/human-feedback/cli.ts +++ b/src/human-feedback/cli.ts @@ -11,8 +11,9 @@ import { } from './feedback' export class HumanFeedbackMechanismCLI< - T extends HumanFeedbackType -> extends HumanFeedbackMechanism { + T extends HumanFeedbackType, + TOutput = any +> extends HumanFeedbackMechanism { /** * Prompt the user to select one of a list of options. */ @@ -43,19 +44,31 @@ export class HumanFeedbackMechanismCLI< }) } - protected async selectOne(response: any[]): Promise { + protected async selectOne( + response: TOutput + ): Promise { + if (!Array.isArray(response)) { + throw new Error('selectOne called on non-array response') + } + const choices = response.map((option) => ({ - name: option, + name: String(option), value: option })) return select({ message: 'Pick one output:', choices }) } - protected async selectN(response: any[]): Promise { + protected async selectN( + response: TOutput + ): Promise { + if (!Array.isArray(response)) { + throw new Error('selectN called on non-array response') + } + const choices = response.map((option) => ({ - name: option, + name: String(option), value: option })) - return checkbox({ message: 'Select outputs:', choices }) + return checkbox({ message: 'Select outputs:', choices }) as any } } diff --git a/src/human-feedback/feedback.ts b/src/human-feedback/feedback.ts index 2e9c0b0..7936c18 100644 --- a/src/human-feedback/feedback.ts +++ b/src/human-feedback/feedback.ts @@ -33,14 +33,15 @@ export const HumanFeedbackUserActionMessages: Record< */ export type HumanFeedbackType = 'confirm' | 'selectOne' | 'selectN' -type HumanFeedbackMechanismConstructor = new ( - ...args: any[] -) => HumanFeedbackMechanism +type HumanFeedbackMechanismConstructor< + T extends HumanFeedbackType, + TOutput = any +> = new (...args: any[]) => HumanFeedbackMechanism /** * Options for human feedback. */ -export type HumanFeedbackOptions = { +export type HumanFeedbackOptions = { /** * What type of feedback to request. */ @@ -64,7 +65,7 @@ export type HumanFeedbackOptions = { /** * The human feedback mechanism to use for this task. */ - mechanism?: HumanFeedbackMechanismConstructor + mechanism?: HumanFeedbackMechanismConstructor } export interface BaseHumanFeedbackMetadata { @@ -125,28 +126,35 @@ export type FeedbackTypeToMetadata = ? HumanFeedbackSelectOneMetadata : HumanFeedbackSelectNMetadata -export abstract class HumanFeedbackMechanism { +export abstract class HumanFeedbackMechanism< + T extends HumanFeedbackType, + TOutput +> { protected _agentic: Agentic protected _task: BaseTask - protected _options: Required> + protected _options: Required> constructor({ task, options }: { task: BaseTask - options: Required> + options: Required> }) { this._agentic = task.agentic this._task = task this._options = options } - protected abstract selectOne(response: any): Promise + protected abstract selectOne( + output: TOutput + ): Promise - protected abstract selectN(response: any): Promise + protected abstract selectN( + response: TOutput + ): Promise protected abstract annotate(): Promise @@ -162,8 +170,8 @@ export abstract class HumanFeedbackMechanism { return this._task.outputSchema.parse(parsedOutput) } - public async interact(response: any): Promise> { - const stringified = JSON.stringify(response, null, 2) + public async interact(output: TOutput): Promise> { + const stringified = JSON.stringify(output, null, 2) const msg = [ 'The following output was generated:', '```', @@ -216,9 +224,17 @@ export abstract class HumanFeedbackMechanism { case HumanFeedbackUserActions.Select: if (this._options.type === 'selectN') { - feedback.selected = await this.selectN(response) + if (!Array.isArray(output)) { + throw new Error('Expected output to be an array') + } + + feedback.selected = await this.selectN(output) } else if (this._options.type === 'selectOne') { - feedback.chosen = await this.selectOne(response) + if (!Array.isArray(output)) { + throw new Error('Expected output to be an array') + } + + feedback.chosen = await this.selectOne(output) } break @@ -241,9 +257,9 @@ export abstract class HumanFeedbackMechanism { } } -export function withHumanFeedback( - task: BaseTask, - options: HumanFeedbackOptions = {} +export function withHumanFeedback( + task: BaseTask, + options: HumanFeedbackOptions = {} ) { task = task.clone() @@ -251,7 +267,7 @@ export function withHumanFeedback( const instanceDefaults = task.agentic.humanFeedbackDefaults // Use Object.assign to merge the options, instance defaults, and hard-coded defaults - const finalOptions: HumanFeedbackOptions = Object.assign( + const finalOptions: HumanFeedbackOptions = Object.assign( { type: 'confirm', bail: false, @@ -278,7 +294,7 @@ export function withHumanFeedback( const originalCall = task.callWithMetadata.bind(task) - task.callWithMetadata = async function (input?: T) { + task.callWithMetadata = async function (input?: TInput) { const response = await originalCall(input) const feedback = await feedbackMechanism.interact(response.result) diff --git a/src/human-feedback/slack.ts b/src/human-feedback/slack.ts index 724d01c..4700202 100644 --- a/src/human-feedback/slack.ts +++ b/src/human-feedback/slack.ts @@ -10,8 +10,9 @@ import { } from './feedback' export class HumanFeedbackMechanismSlack< - T extends HumanFeedbackType -> extends HumanFeedbackMechanism { + T extends HumanFeedbackType, + TOutput = any +> extends HumanFeedbackMechanism { private slackClient: SlackClient constructor({ @@ -19,7 +20,7 @@ export class HumanFeedbackMechanismSlack< options }: { task: BaseTask - options: Required> + options: Required> }) { super({ task, options }) this.slackClient = new SlackClient() @@ -68,7 +69,13 @@ export class HumanFeedbackMechanismSlack< return choices[parseInt(response.text)] } - public async selectOne(response: any[]): Promise { + protected async selectOne( + response: TOutput + ): Promise { + if (!Array.isArray(response)) { + throw new Error('selectOne called on non-array response') + } + const { text: selectedOutput } = await this.slackClient.sendAndWaitForReply( { text: @@ -84,7 +91,13 @@ export class HumanFeedbackMechanismSlack< return response[parseInt(selectedOutput)] } - public async selectN(response: any[]): Promise { + protected async selectN( + response: TOutput + ): Promise { + if (!Array.isArray(response)) { + throw new Error('selectN called on non-array response') + } + const { text: selectedOutput } = await this.slackClient.sendAndWaitForReply( { text: @@ -107,6 +120,6 @@ export class HumanFeedbackMechanismSlack< .map((choice) => parseInt(choice)) return response.filter((_, idx) => { return chosenOutputs.includes(idx) - }) + }) as any } } diff --git a/src/human-feedback/twilio.ts b/src/human-feedback/twilio.ts index e41c739..895d10d 100644 --- a/src/human-feedback/twilio.ts +++ b/src/human-feedback/twilio.ts @@ -10,8 +10,9 @@ import { } from './feedback' export class HumanFeedbackMechanismTwilio< - T extends HumanFeedbackType -> extends HumanFeedbackMechanism { + T extends HumanFeedbackType, + TOutput = any +> extends HumanFeedbackMechanism { private twilioClient: TwilioConversationClient constructor({ @@ -19,7 +20,7 @@ export class HumanFeedbackMechanismTwilio< options }: { task: BaseTask - options: Required> + options: Required> }) { super({ task, options }) this.twilioClient = new TwilioConversationClient() @@ -71,7 +72,13 @@ export class HumanFeedbackMechanismTwilio< return choices[parseInt(response.body)] } - public async selectOne(response: any[]): Promise { + protected async selectOne( + response: TOutput + ): Promise { + if (!Array.isArray(response)) { + throw new Error('selectOne called on non-array response') + } + const { body: selectedOutput } = await this.twilioClient.sendAndWaitForReply({ name: 'human-feedback-select', @@ -87,7 +94,13 @@ export class HumanFeedbackMechanismTwilio< return response[parseInt(selectedOutput)] } - public async selectN(response: any[]): Promise { + protected async selectN( + response: TOutput + ): Promise { + if (!Array.isArray(response)) { + throw new Error('selectN called on non-array response') + } + const { body: selectedOutput } = await this.twilioClient.sendAndWaitForReply({ name: 'human-feedback-select', @@ -110,6 +123,6 @@ export class HumanFeedbackMechanismTwilio< .map((choice) => parseInt(choice)) return response.filter((_, idx) => { return chosenOutputs.includes(idx) - }) + }) as any } } diff --git a/src/types.ts b/src/types.ts index da5e8ee..5fc4752 100644 --- a/src/types.ts +++ b/src/types.ts @@ -6,11 +6,14 @@ import type { JsonObject, JsonValue } from 'type-fest' import { SafeParseReturnType, ZodType, ZodTypeAny, output, z } from 'zod' import type { Agentic } from './agentic' +import type { + FeedbackTypeToMetadata, + HumanFeedbackType +} from './human-feedback' import type { Logger } from './logger' import type { BaseTask } from './task' -export { openai } -export { anthropic } +export { anthropic, openai } export type { Logger } export type { JsonObject, JsonValue } @@ -102,6 +105,9 @@ export interface TaskResponseMetadata extends Record { error?: Error numRetries?: number callId?: string + + // human feedback info + feedback?: FeedbackTypeToMetadata } export interface LLMTaskResponseMetadata< From e1abd8a24f3a91dec2372db07e4eae9cfbe1b7d9 Mon Sep 17 00:00:00 2001 From: Philipp Burckhardt Date: Tue, 13 Jun 2023 21:47:41 -0400 Subject: [PATCH 12/22] fix: supply all required type arguments --- src/agentic.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/agentic.ts b/src/agentic.ts index 6a1767e..3905614 100644 --- a/src/agentic.ts +++ b/src/agentic.ts @@ -20,7 +20,7 @@ export class Agentic { types.BaseLLMOptions, 'provider' | 'model' | 'modelParams' | 'timeoutMs' | 'retryConfig' > - protected _humanFeedbackDefaults: HumanFeedbackOptions + protected _humanFeedbackDefaults: HumanFeedbackOptions protected _idGeneratorFn: types.IDGeneratorFunction protected _id: string @@ -31,7 +31,7 @@ export class Agentic { types.BaseLLMOptions, 'provider' | 'model' | 'modelParams' | 'timeoutMs' | 'retryConfig' > - humanFeedbackDefaults?: HumanFeedbackOptions + humanFeedbackDefaults?: HumanFeedbackOptions idGeneratorFn?: types.IDGeneratorFunction logger?: types.Logger ky?: types.KyInstance From e6548f037b5ede6dcdae78b556768551a2ecbca4 Mon Sep 17 00:00:00 2001 From: Philipp Burckhardt Date: Tue, 13 Jun 2023 21:51:04 -0400 Subject: [PATCH 13/22] chore: delete holdover feedback file --- scratch/feedback.ts | 232 -------------------------------------------- 1 file changed, 232 deletions(-) delete mode 100644 scratch/feedback.ts diff --git a/scratch/feedback.ts b/scratch/feedback.ts deleted file mode 100644 index 51b0e16..0000000 --- a/scratch/feedback.ts +++ /dev/null @@ -1,232 +0,0 @@ -import checkbox from '@inquirer/checkbox' -import editor from '@inquirer/editor' -import select from '@inquirer/select' -import { ZodTypeAny, z } from 'zod' - -import * as types from '@/types' -import { BaseTask } from '@/task' - -/** - * Actions the user can take in the feedback selection prompt. - */ -export const UserActions = { - Accept: 'accept', - Edit: 'edit', - Decline: 'decline', - Select: 'select', - Exit: 'exit' -} as const - -export type UserActions = (typeof UserActions)[keyof typeof UserActions] - -/** - * Messages to display to the user for each action. - */ -const UserActionMessages: Record = { - [UserActions.Accept]: 'Accept inputs', - [UserActions.Edit]: 'Edit (open in editor)', - [UserActions.Decline]: 'Decline', - [UserActions.Select]: 'Select inputs to keep', - [UserActions.Exit]: 'Exit' -} - -/** - * Prompt the user to select one of a list of options. - */ -async function askUser( - message: string, - choices: UserActions[] -): Promise { - return select({ - message, - choices: choices.map((choice) => ({ - name: UserActionMessages[choice], - value: choice - })) - }) -} - -/** - * Output schema when prompting the user to accept, edit, or decline a single input. - */ -export const FeedbackSingleOutputSchema = (result: T) => - z.object({ - result: result, - accepted: z.boolean() - }) - -/** - * Prompt the user to accept, edit, or decline a single input. - */ -export class HumanFeedbackSingle extends BaseTask< - ZodTypeAny, - ZodTypeAny -> { - protected choiceSchema: T - - constructor(choiceSchema: T) { - super() - this.choiceSchema = choiceSchema - } - - public get inputSchema() { - return this.choiceSchema - } - - public get outputSchema() { - return FeedbackSingleOutputSchema(this.choiceSchema) - } - - protected actionHandlers = { - [UserActions.Accept]: ( - input: types.ParsedData - ) => ({ result: input, accepted: true }), - [UserActions.Edit]: async ( - input: types.ParsedData - ) => { - const editedInput = await editor({ - message: 'Edit the input:', - default: JSON.stringify(input) - }) - return this.outputSchema.parse({ - result: JSON.parse(editedInput), - accepted: true - }) - }, - [UserActions.Decline]: () => ({ result: null, accepted: false }), - [UserActions.Exit]: () => { - throw new Error('Exiting...') - } - } - - /** - * Prompts the user to give feedback for the given input and handles their response. - */ - public async call( - input: types.ParsedData - ): Promise> { - try { - input = this.inputSchema.parse(input) - const msg = [ - 'The following input was generated:', - JSON.stringify(input, null, 2), - 'What would you like to do?' - ].join('\n') - const feedback = await askUser(msg, [ - UserActions.Accept, - UserActions.Edit, - UserActions.Decline, - UserActions.Exit - ]) - const handler = this.actionHandlers[feedback] - if (!handler) { - throw new Error(`Unexpected feedback: ${feedback}`) - } - return handler(input) - } catch (err) { - console.error('Error parsing input:', err) - throw err - } - } -} - -/** - * Output schema when prompting the user to accept, select from, edit, or decline a list of inputs. - */ -export const FeedbackSelectOutputSchema = (result: T) => - z.object({ - results: z.array(result), - accepted: z.boolean() - }) - -/** - * Prompt the user to accept, select from, edit, or decline a list of inputs. - */ -export class HumanFeedbackSelect extends BaseTask< - ZodTypeAny, - ZodTypeAny -> { - protected choiceSchema: T - - constructor(choiceSchema: T) { - super() - this.choiceSchema = choiceSchema - } - - public get inputSchema() { - return z.array(this.choiceSchema) - } - - public get outputSchema() { - return FeedbackSelectOutputSchema(this.choiceSchema) - } - - protected actionHandlers = { - [UserActions.Accept]: ( - input: types.ParsedData - ) => ({ results: input, accepted: true }), - [UserActions.Edit]: async ( - input: types.ParsedData - ) => { - const editedInput = await editor({ - message: 'Edit the input:', - default: JSON.stringify(input, null, 2) - }) - return this.outputSchema.parse({ - results: JSON.parse(editedInput), - accepted: true - }) - }, - [UserActions.Select]: async ( - input: types.ParsedData - ) => { - const choices = input.map((completion) => ({ - name: completion, - value: completion - })) - const chosen = await checkbox({ - message: 'Pick items to keep:', - choices, - pageSize: choices.length - }) - return { results: chosen.length === 0 ? [] : chosen, accepted: true } - }, - [UserActions.Decline]: () => ({ results: [], accepted: false }), - [UserActions.Exit]: () => { - throw new Error('Exiting...') - } - } - - /** - * Prompts the user to give feedback for the given list of inputs and handles their response. - */ - public async call( - input: types.ParsedData - ): Promise> { - try { - input = this.inputSchema.parse(input) - const message = [ - 'The following inputs were generated:', - ...input.map( - (choice, index) => `${index + 1}. ${JSON.stringify(choice, null, 2)}` - ), - 'What would you like to do?' - ].join('\n') - const feedback = await askUser(message, [ - UserActions.Accept, - UserActions.Select, - UserActions.Edit, - UserActions.Decline, - UserActions.Exit - ]) - const handler = this.actionHandlers[feedback] - if (!handler) { - throw new Error(`Unexpected feedback: ${feedback}`) - } - return handler(input) - } catch (err) { - console.error('Error parsing input:', err) - throw err - } - } -} From ee0c3cf967cba421d3f80e93c3d5447d5e4306d1 Mon Sep 17 00:00:00 2001 From: Philipp Burckhardt Date: Wed, 14 Jun 2023 15:55:19 -0400 Subject: [PATCH 14/22] docs: add new feedback example --- scratch/examples/human-feedback-cli.ts | 39 +++++++++++++++++++++++ scratch/examples/human-feedback-select.ts | 32 ------------------- scratch/examples/human-feedback.ts | 33 ------------------- 3 files changed, 39 insertions(+), 65 deletions(-) create mode 100644 scratch/examples/human-feedback-cli.ts delete mode 100644 scratch/examples/human-feedback-select.ts delete mode 100644 scratch/examples/human-feedback.ts diff --git a/scratch/examples/human-feedback-cli.ts b/scratch/examples/human-feedback-cli.ts new file mode 100644 index 0000000..64c7434 --- /dev/null +++ b/scratch/examples/human-feedback-cli.ts @@ -0,0 +1,39 @@ +import 'dotenv/config' +import { OpenAIClient } from 'openai-fetch' +import { z } from 'zod' + +import { HumanFeedbackMechanismCLI } from '@/human-feedback' +import { Agentic, withHumanFeedback } from '@/index' + +async function main() { + const openai = new OpenAIClient({ apiKey: process.env.OPENAI_API_KEY! }) + const ai = new Agentic({ openai }) + + const topicFacts = ai + .gpt3(`Tell me {{num}} jokes about {{topic}}`) + .input( + z.object({ + topic: z.string(), + num: z.number().int().default(5).optional() + }) + ) + .output(z.array(z.string())) + .modelParams({ temperature: 0.9 }) + + const topicFactsFeedback = withHumanFeedback(topicFacts, { + type: 'selectN', + annotations: false, + bail: false, + editing: true, + mechanism: HumanFeedbackMechanismCLI + }) + + const out = await topicFactsFeedback.callWithMetadata({ + topic: 'politicians', + num: 5 + }) + const feedback = out.metadata.feedback + console.log(JSON.stringify(feedback, null, 2)) +} + +main() diff --git a/scratch/examples/human-feedback-select.ts b/scratch/examples/human-feedback-select.ts deleted file mode 100644 index 0e50ec0..0000000 --- a/scratch/examples/human-feedback-select.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { OpenAIClient } from '@agentic/openai-fetch' -import 'dotenv/config' -import { z } from 'zod' - -import { Agentic, HumanFeedbackSelect } from '@/index' - -async function main() { - const openai = new OpenAIClient({ apiKey: process.env.OPENAI_API_KEY! }) - const ai = new Agentic({ openai }) - - const jokes = ai - .gpt3(`Tell me {{num}} jokes about {{topic}}`) - .input( - z.object({ - topic: z.string(), - num: z.number().int().default(5).optional() - }) - ) - .output(z.array(z.string())) - .modelParams({ temperature: 0.9 }) - - const feedback = new HumanFeedbackSelect(z.string()) - let out = await jokes.call({ topic: 'statisticians' }) - let hf = await feedback.call(out) - while (!hf.accepted) { - out = await jokes.call({ topic: 'statisticians' }) - hf = await feedback.call(out) - } - console.log(hf.results) -} - -main() diff --git a/scratch/examples/human-feedback.ts b/scratch/examples/human-feedback.ts deleted file mode 100644 index c62f2b5..0000000 --- a/scratch/examples/human-feedback.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { OpenAIClient } from '@agentic/openai-fetch' -import 'dotenv/config' -import { z } from 'zod' - -import { Agentic, HumanFeedbackSingle } from '@/index' - -async function main() { - const openai = new OpenAIClient({ apiKey: process.env.OPENAI_API_KEY! }) - const ai = new Agentic({ openai }) - - const topicFacts = ai - .gpt3(`Give me {{numFacts}} random facts about {{topic}}`) - .input( - z.object({ - topic: z.string(), - numFacts: z.number().int().default(5).optional() - }) - ) - .output(z.object({ facts: z.array(z.string()) })) - .modelParams({ temperature: 0.9 }) - - const feedback = new HumanFeedbackSingle(topicFacts.outputSchema) - - let out = await topicFacts.call({ topic: 'cats' }) - let hf = await feedback.call(out) - while (!hf.accepted) { - out = await topicFacts.call({ topic: 'cats' }) - hf = await feedback.call(out) - } - console.log(hf.result) -} - -main() From 4ff22618a2b8a51520b80ec394b1d0ffe56cde1c Mon Sep 17 00:00:00 2001 From: Philipp Burckhardt Date: Wed, 14 Jun 2023 15:57:15 -0400 Subject: [PATCH 15/22] docs: update variable names --- scratch/examples/human-feedback-cli.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scratch/examples/human-feedback-cli.ts b/scratch/examples/human-feedback-cli.ts index 64c7434..85a0a20 100644 --- a/scratch/examples/human-feedback-cli.ts +++ b/scratch/examples/human-feedback-cli.ts @@ -9,7 +9,7 @@ async function main() { const openai = new OpenAIClient({ apiKey: process.env.OPENAI_API_KEY! }) const ai = new Agentic({ openai }) - const topicFacts = ai + const topicJokes = ai .gpt3(`Tell me {{num}} jokes about {{topic}}`) .input( z.object({ @@ -20,7 +20,7 @@ async function main() { .output(z.array(z.string())) .modelParams({ temperature: 0.9 }) - const topicFactsFeedback = withHumanFeedback(topicFacts, { + const topicJokesFeedback = withHumanFeedback(topicJokes, { type: 'selectN', annotations: false, bail: false, @@ -28,7 +28,7 @@ async function main() { mechanism: HumanFeedbackMechanismCLI }) - const out = await topicFactsFeedback.callWithMetadata({ + const out = await topicJokesFeedback.callWithMetadata({ topic: 'politicians', num: 5 }) From 8ba59411e749a40ab1b7525338a212e52a012e06 Mon Sep 17 00:00:00 2001 From: Philipp Burckhardt Date: Wed, 14 Jun 2023 15:59:12 -0400 Subject: [PATCH 16/22] docs: add feedback examples for Slack and Twilio --- scratch/examples/human-feedback-slack.ts | 39 +++++++++++++++++++++++ scratch/examples/human-feedback-twilio.ts | 39 +++++++++++++++++++++++ 2 files changed, 78 insertions(+) create mode 100644 scratch/examples/human-feedback-slack.ts create mode 100644 scratch/examples/human-feedback-twilio.ts diff --git a/scratch/examples/human-feedback-slack.ts b/scratch/examples/human-feedback-slack.ts new file mode 100644 index 0000000..c1b21e6 --- /dev/null +++ b/scratch/examples/human-feedback-slack.ts @@ -0,0 +1,39 @@ +import 'dotenv/config' +import { OpenAIClient } from 'openai-fetch' +import { z } from 'zod' + +import { HumanFeedbackMechanismSlack } from '@/human-feedback' +import { Agentic, withHumanFeedback } from '@/index' + +async function main() { + const openai = new OpenAIClient({ apiKey: process.env.OPENAI_API_KEY! }) + const ai = new Agentic({ openai }) + + const topicJokes = ai + .gpt3(`Tell me {{num}} jokes about {{topic}}`) + .input( + z.object({ + topic: z.string(), + num: z.number().int().default(5).optional() + }) + ) + .output(z.array(z.string())) + .modelParams({ temperature: 0.9 }) + + const topicJokesFeedback = withHumanFeedback(topicJokes, { + type: 'selectN', + annotations: false, + bail: false, + editing: true, + mechanism: HumanFeedbackMechanismSlack + }) + + const out = await topicJokesFeedback.callWithMetadata({ + topic: 'politicians', + num: 5 + }) + const feedback = out.metadata.feedback + console.log(JSON.stringify(feedback, null, 2)) +} + +main() diff --git a/scratch/examples/human-feedback-twilio.ts b/scratch/examples/human-feedback-twilio.ts new file mode 100644 index 0000000..e94807b --- /dev/null +++ b/scratch/examples/human-feedback-twilio.ts @@ -0,0 +1,39 @@ +import 'dotenv/config' +import { OpenAIClient } from 'openai-fetch' +import { z } from 'zod' + +import { HumanFeedbackMechanismTwilio } from '@/human-feedback' +import { Agentic, withHumanFeedback } from '@/index' + +async function main() { + const openai = new OpenAIClient({ apiKey: process.env.OPENAI_API_KEY! }) + const ai = new Agentic({ openai }) + + const topicJokes = ai + .gpt3(`Tell me {{num}} jokes about {{topic}}`) + .input( + z.object({ + topic: z.string(), + num: z.number().int().default(5).optional() + }) + ) + .output(z.array(z.string())) + .modelParams({ temperature: 0.9 }) + + const topicJokesFeedback = withHumanFeedback(topicJokes, { + type: 'selectN', + annotations: false, + bail: false, + editing: true, + mechanism: HumanFeedbackMechanismTwilio + }) + + const out = await topicJokesFeedback.callWithMetadata({ + topic: 'politicians', + num: 5 + }) + const feedback = out.metadata.feedback + console.log(JSON.stringify(feedback, null, 2)) +} + +main() From 0bfce67a7b18fce8454e186a83d55c689db5db72 Mon Sep 17 00:00:00 2001 From: Philipp Burckhardt Date: Wed, 14 Jun 2023 21:22:06 -0400 Subject: [PATCH 17/22] feat: apply suggestions from PR --- src/human-feedback/cli.ts | 10 ++++---- src/human-feedback/feedback.ts | 42 +++++++++++++++++----------------- src/human-feedback/slack.ts | 38 +++++++++++++++--------------- src/human-feedback/twilio.ts | 28 ++++++++++++----------- 4 files changed, 60 insertions(+), 58 deletions(-) diff --git a/src/human-feedback/cli.ts b/src/human-feedback/cli.ts index cacd22d..d67a5f7 100644 --- a/src/human-feedback/cli.ts +++ b/src/human-feedback/cli.ts @@ -17,7 +17,7 @@ export class HumanFeedbackMechanismCLI< /** * Prompt the user to select one of a list of options. */ - protected async askUser( + protected async _askUser( message: string, choices: HumanFeedbackUserActions[] ): Promise { @@ -30,21 +30,21 @@ export class HumanFeedbackMechanismCLI< }) } - protected async edit(output: string): Promise { + protected async _edit(output: string): Promise { return editor({ message: 'Edit the output:', default: output }) } - protected async annotate(): Promise { + protected async _annotate(): Promise { return input({ message: 'Please leave an annotation (leave blank to skip; press enter to submit):' }) } - protected async selectOne( + protected async _selectOne( response: TOutput ): Promise { if (!Array.isArray(response)) { @@ -58,7 +58,7 @@ export class HumanFeedbackMechanismCLI< return select({ message: 'Pick one output:', choices }) } - protected async selectN( + protected async _selectN( response: TOutput ): Promise { if (!Array.isArray(response)) { diff --git a/src/human-feedback/feedback.ts b/src/human-feedback/feedback.ts index 7936c18..5a881da 100644 --- a/src/human-feedback/feedback.ts +++ b/src/human-feedback/feedback.ts @@ -11,7 +11,7 @@ export const HumanFeedbackUserActions = { Edit: 'edit', Decline: 'decline', Select: 'select', - Exit: 'exit' + Abort: 'abort' } as const export type HumanFeedbackUserActions = @@ -25,7 +25,7 @@ export const HumanFeedbackUserActionMessages: Record< [HumanFeedbackUserActions.Edit]: 'Edit the output', [HumanFeedbackUserActions.Decline]: 'Decline the output', [HumanFeedbackUserActions.Select]: 'Select outputs to keep', - [HumanFeedbackUserActions.Exit]: 'Exit' + [HumanFeedbackUserActions.Abort]: 'Abort' } /** @@ -48,9 +48,9 @@ export type HumanFeedbackOptions = { type?: T /** - * Whether the user can bail out of the feedback loop. + * Whether the user can abort the process. */ - bail?: boolean + abort?: boolean /** * Whether the user can edit the output. @@ -148,24 +148,24 @@ export abstract class HumanFeedbackMechanism< this._options = options } - protected abstract selectOne( + protected abstract _selectOne( output: TOutput ): Promise - protected abstract selectN( + protected abstract _selectN( response: TOutput ): Promise - protected abstract annotate(): Promise + protected abstract _annotate(): Promise - protected abstract edit(output: string): Promise + protected abstract _edit(output: string): Promise - protected abstract askUser( + protected abstract _askUser( message: string, choices: HumanFeedbackUserActions[] ): Promise - protected parseEditedOutput(editedOutput: string): any { + protected _parseEditedOutput(editedOutput: string): any { const parsedOutput = JSON.parse(editedOutput) return this._task.outputSchema.parse(parsedOutput) } @@ -196,14 +196,14 @@ export abstract class HumanFeedbackMechanism< choices.push(HumanFeedbackUserActions.Edit) } - if (this._options.bail) { - choices.push(HumanFeedbackUserActions.Exit) + if (this._options.abort) { + choices.push(HumanFeedbackUserActions.Abort) } const choice = choices.length === 1 ? HumanFeedbackUserActions.Select - : await this.askUser(msg, choices) + : await this._askUser(msg, choices) const feedback: Record = {} @@ -213,8 +213,8 @@ export abstract class HumanFeedbackMechanism< break case HumanFeedbackUserActions.Edit: { - const editedOutput = await this.edit(stringified) - feedback.editedOutput = await this.parseEditedOutput(editedOutput) + const editedOutput = await this._edit(stringified) + feedback.editedOutput = await this._parseEditedOutput(editedOutput) break } @@ -228,26 +228,26 @@ export abstract class HumanFeedbackMechanism< throw new Error('Expected output to be an array') } - feedback.selected = await this.selectN(output) + feedback.selected = await this._selectN(output) } else if (this._options.type === 'selectOne') { if (!Array.isArray(output)) { throw new Error('Expected output to be an array') } - feedback.chosen = await this.selectOne(output) + feedback.chosen = await this._selectOne(output) } break - case HumanFeedbackUserActions.Exit: - throw new Error('Exiting...') + case HumanFeedbackUserActions.Abort: + throw new Error('Aborting...') default: throw new Error(`Unexpected choice: ${choice}`) } if (this._options.annotations) { - const annotation = await this.annotate() + const annotation = await this._annotate() if (annotation) { feedback.annotation = annotation } @@ -270,7 +270,7 @@ export function withHumanFeedback( const finalOptions: HumanFeedbackOptions = Object.assign( { type: 'confirm', - bail: false, + abort: false, editing: false, annotations: false, mechanism: HumanFeedbackMechanismCLI diff --git a/src/human-feedback/slack.ts b/src/human-feedback/slack.ts index 4700202..25b7f9a 100644 --- a/src/human-feedback/slack.ts +++ b/src/human-feedback/slack.ts @@ -13,22 +13,24 @@ export class HumanFeedbackMechanismSlack< T extends HumanFeedbackType, TOutput = any > extends HumanFeedbackMechanism { - private slackClient: SlackClient + protected _slackClient: SlackClient constructor({ task, - options + options, + slackClient = new SlackClient() }: { task: BaseTask options: Required> + slackClient: SlackClient }) { super({ task, options }) - this.slackClient = new SlackClient() + this._slackClient = slackClient } - protected async annotate(): Promise { + protected async _annotate(): Promise { try { - const annotation = await this.slackClient.sendAndWaitForReply({ + const annotation = await this._slackClient.sendAndWaitForReply({ text: 'Please leave an annotation (optional):' }) return annotation.text @@ -38,8 +40,8 @@ export class HumanFeedbackMechanismSlack< } } - protected async edit(): Promise { - let { text: editedOutput } = await this.slackClient.sendAndWaitForReply({ + protected async _edit(): Promise { + let { text: editedOutput } = await this._slackClient.sendAndWaitForReply({ text: 'Copy and edit the output:' }) editedOutput = editedOutput.replace(/```$/g, '') @@ -47,7 +49,7 @@ export class HumanFeedbackMechanismSlack< return editedOutput } - protected async askUser( + protected async _askUser( message: string, choices: HumanFeedbackUserActions[] ): Promise { @@ -59,7 +61,7 @@ export class HumanFeedbackMechanismSlack< .join('\n') message += '\n\n' message += 'Reply with the number of your choice.' - const response = await this.slackClient.sendAndWaitForReply({ + const response = await this._slackClient.sendAndWaitForReply({ text: message, validate: (slackMessage) => { const choice = parseInt(slackMessage.text) @@ -69,15 +71,15 @@ export class HumanFeedbackMechanismSlack< return choices[parseInt(response.text)] } - protected async selectOne( + protected async _selectOne( response: TOutput ): Promise { if (!Array.isArray(response)) { throw new Error('selectOne called on non-array response') } - const { text: selectedOutput } = await this.slackClient.sendAndWaitForReply( - { + const { text: selectedOutput } = + await this._slackClient.sendAndWaitForReply({ text: 'Pick one output:' + response.map((r, idx) => `\n*${idx}* - ${r}`).join('') + @@ -86,20 +88,19 @@ export class HumanFeedbackMechanismSlack< const choice = parseInt(slackMessage.text) return !isNaN(choice) && choice >= 0 && choice < response.length } - } - ) + }) return response[parseInt(selectedOutput)] } - protected async selectN( + protected async _selectN( response: TOutput ): Promise { if (!Array.isArray(response)) { throw new Error('selectN called on non-array response') } - const { text: selectedOutput } = await this.slackClient.sendAndWaitForReply( - { + const { text: selectedOutput } = + await this._slackClient.sendAndWaitForReply({ text: 'Select outputs:' + response.map((r, idx) => `\n*${idx}* - ${r}`).join('') + @@ -113,8 +114,7 @@ export class HumanFeedbackMechanismSlack< ) }) } - } - ) + }) const chosenOutputs = selectedOutput .split(',') .map((choice) => parseInt(choice)) diff --git a/src/human-feedback/twilio.ts b/src/human-feedback/twilio.ts index 895d10d..72f8386 100644 --- a/src/human-feedback/twilio.ts +++ b/src/human-feedback/twilio.ts @@ -13,22 +13,24 @@ export class HumanFeedbackMechanismTwilio< T extends HumanFeedbackType, TOutput = any > extends HumanFeedbackMechanism { - private twilioClient: TwilioConversationClient + protected _twilioClient: TwilioConversationClient constructor({ task, - options + options, + twilioClient = new TwilioConversationClient() }: { task: BaseTask options: Required> + twilioClient: TwilioConversationClient }) { super({ task, options }) - this.twilioClient = new TwilioConversationClient() + this._twilioClient = twilioClient } - protected async annotate(): Promise { + protected async _annotate(): Promise { try { - const annotation = await this.twilioClient.sendAndWaitForReply({ + const annotation = await this._twilioClient.sendAndWaitForReply({ name: 'human-feedback-annotation', text: 'Please leave an annotation (optional):' }) @@ -39,8 +41,8 @@ export class HumanFeedbackMechanismTwilio< } } - protected async edit(): Promise { - let { body: editedOutput } = await this.twilioClient.sendAndWaitForReply({ + protected async _edit(): Promise { + let { body: editedOutput } = await this._twilioClient.sendAndWaitForReply({ text: 'Copy and edit the output:', name: 'human-feedback-edit' }) @@ -49,7 +51,7 @@ export class HumanFeedbackMechanismTwilio< return editedOutput } - protected async askUser( + protected async _askUser( message: string, choices: HumanFeedbackUserActions[] ): Promise { @@ -61,7 +63,7 @@ export class HumanFeedbackMechanismTwilio< .join('\n') message += '\n\n' message += 'Reply with the number of your choice.' - const response = await this.twilioClient.sendAndWaitForReply({ + const response = await this._twilioClient.sendAndWaitForReply({ name: 'human-feedback-ask', text: message, validate: (message) => { @@ -72,7 +74,7 @@ export class HumanFeedbackMechanismTwilio< return choices[parseInt(response.body)] } - protected async selectOne( + protected async _selectOne( response: TOutput ): Promise { if (!Array.isArray(response)) { @@ -80,7 +82,7 @@ export class HumanFeedbackMechanismTwilio< } const { body: selectedOutput } = - await this.twilioClient.sendAndWaitForReply({ + await this._twilioClient.sendAndWaitForReply({ name: 'human-feedback-select', text: 'Pick one output:' + @@ -94,7 +96,7 @@ export class HumanFeedbackMechanismTwilio< return response[parseInt(selectedOutput)] } - protected async selectN( + protected async _selectN( response: TOutput ): Promise { if (!Array.isArray(response)) { @@ -102,7 +104,7 @@ export class HumanFeedbackMechanismTwilio< } const { body: selectedOutput } = - await this.twilioClient.sendAndWaitForReply({ + await this._twilioClient.sendAndWaitForReply({ name: 'human-feedback-select', text: 'Select outputs:' + From 5b510116e4fe7f9d13c01ab06808451a307d6de4 Mon Sep 17 00:00:00 2001 From: Philipp Burckhardt Date: Wed, 14 Jun 2023 23:00:02 -0400 Subject: [PATCH 18/22] feat: handle message chunking in Twilio service --- src/services/twilio-conversation.ts | 34 +++++++++++++++++++++-- src/utils.ts | 33 ++++++++++++++++++++++ test/services/twilio-conversation.test.ts | 23 +++++++++++++++ 3 files changed, 88 insertions(+), 2 deletions(-) diff --git a/src/services/twilio-conversation.ts b/src/services/twilio-conversation.ts index 31e6541..f6addd5 100644 --- a/src/services/twilio-conversation.ts +++ b/src/services/twilio-conversation.ts @@ -1,7 +1,7 @@ import defaultKy from 'ky' import { DEFAULT_BOT_NAME } from '@/constants' -import { sleep } from '@/utils' +import { chunkString, sleep } from '@/utils' export const TWILIO_CONVERSATION_API_BASE_URL = 'https://conversations.twilio.com/v1' @@ -9,6 +9,14 @@ export const TWILIO_CONVERSATION_API_BASE_URL = export const DEFAULT_TWILIO_TIMEOUT_MS = 120_000 export const DEFAULT_TWILIO_INTERVAL_MS = 5_000 +/** + * Twilio recommends keeping SMS messages to a length of 320 characters or less, so we'll use that as the maximum. + * + * @see {@link https://support.twilio.com/hc/en-us/articles/360033806753-Maximum-Message-Length-with-Twilio-Programmable-Messaging} + */ +const TWILIO_SMS_LENGTH_SOFT_LIMIT = 320 +const TWILIO_SMS_LENGTH_HARD_LIMIT = 1600 + export interface TwilioConversation { unique_name?: string date_updated: Date @@ -229,6 +237,23 @@ export class TwilioConversationClient { .json() } + /** + * Chunks a long text message into smaller parts and sends them as separate messages. + */ + async sendTextWithChunking({ + conversationSid, + text + }: { + conversationSid: string + text: string + maxChunkLength?: number + }) { + const chunks = chunkString(text, TWILIO_SMS_LENGTH_SOFT_LIMIT) + return Promise.all( + chunks.map((chunk) => this.sendMessage({ conversationSid, text: chunk })) + ) + } + /** * Posts a message to a conversation. */ @@ -239,6 +264,11 @@ export class TwilioConversationClient { conversationSid: string text: string }) { + // Truncate the text if it exceeds the hard limit and add an ellipsis: + if (text.length > TWILIO_SMS_LENGTH_HARD_LIMIT) { + text = text.substring(0, TWILIO_SMS_LENGTH_HARD_LIMIT - 3) + '...' + } + const params = new URLSearchParams() params.set('Body', text) params.set('Author', this.botName) @@ -291,7 +321,7 @@ export class TwilioConversationClient { const { sid: conversationSid } = await this.createConversation(name) await this.addParticipant({ conversationSid, recipientPhoneNumber }) - await this.sendMessage({ conversationSid, text }) + await this.sendTextWithChunking({ conversationSid, text }) const start = Date.now() let nUserMessages = 0 diff --git a/src/utils.ts b/src/utils.ts index d3e8d17..61fd39f 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -21,3 +21,36 @@ const taskNameRegex = /^[a-zA-Z_][a-zA-Z0-9_-]{0,63}$/ export function isValidTaskIdentifier(id: string): boolean { return !!id && taskNameRegex.test(id) } + +/** + * Chunk a string into an array of strings of a given length + * + * @param text - string to chunk + * @param length - maximum length of each chunk + * @returns array of strings + */ +export const chunkString = (text: string, length: number) => { + const words = text.split(' ') + const chunks: string[] = [] + let chunk = '' + + for (const word of words) { + if (word.length > length) { + // Truncate the word if it's too long and indicate that it was truncated: + chunks.push(word.substring(0, length - 3) + '...') + } + + if ((chunk + word).length > length) { + chunks.push(chunk.trim()) + chunk = word + } else { + chunk += ' ' + word + } + } + + if (chunk) { + chunks.push(chunk.trim()) + } + + return chunks +} diff --git a/test/services/twilio-conversation.test.ts b/test/services/twilio-conversation.test.ts index d763335..571d9d8 100644 --- a/test/services/twilio-conversation.test.ts +++ b/test/services/twilio-conversation.test.ts @@ -58,6 +58,29 @@ test.serial('TwilioConversationClient.sendMessage', async (t) => { await client.deleteConversation(conversationSid) }) +test.serial('TwilioConversationClient.sendTextWithChunking', async (t) => { + if (!process.env.TWILIO_ACCOUNT_SID || !process.env.TWILIO_AUTH_TOKEN) { + return t.pass() + } + + const client = new TwilioConversationClient() + + const { sid: conversationSid } = await client.createConversation( + 'send-message-test' + ) + + const text = + 'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.' + + const messages = await client.sendTextWithChunking({ conversationSid, text }) + + // Text should be sent in two messages: + t.true(text.startsWith(messages[0].body)) + t.true(text.endsWith(messages[1].body)) + + await client.deleteConversation(conversationSid) +}) + test.serial('TwilioConversationClient.fetchMessages', async (t) => { if (!process.env.TWILIO_ACCOUNT_SID || !process.env.TWILIO_AUTH_TOKEN) { return t.pass() From 07138c9ca703ab3b84b10663a15135f450b3cd27 Mon Sep 17 00:00:00 2001 From: Philipp Burckhardt Date: Wed, 14 Jun 2023 23:30:16 -0400 Subject: [PATCH 19/22] fix: make quantifier lazy to not span multiple JSON blocks, tests + docs --- src/utils.ts | 40 +++++++++++++++++++++++-------- test/utils.test.ts | 59 +++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 88 insertions(+), 11 deletions(-) diff --git a/src/utils.ts b/src/utils.ts index 61fd39f..075c996 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -2,18 +2,39 @@ import { customAlphabet, urlAlphabet } from 'nanoid' import * as types from './types' +/** + * Extracts the first JSON object string from a given string. + * + * @param text - string from which to extract the JSON object + * @returns extracted JSON object string, or `undefined` if no JSON object is found + */ export function extractJSONObjectFromString(text: string): string | undefined { return text.match(/\{(.|\n)*\}/gm)?.[0] } +/** + * Extracts the first JSON array string from a given string. + * + * @param text - string from which to extract the JSON array + * @returns extracted JSON array string, or `undefined` if no JSON array is found + */ export function extractJSONArrayFromString(text: string): string | undefined { return text.match(/\[(.|\n)*\]/gm)?.[0] } +/** + * Pauses the execution of a function for a specified time. + * + * @param ms - number of milliseconds to pause + * @returns promise that resolves after the specified number of milliseconds + */ export function sleep(ms: number) { return new Promise((resolve) => setTimeout(resolve, ms)) } +/** + * A default ID generator function that uses a custom alphabet based on URL safe symbols. + */ export const defaultIDGeneratorFn: types.IDGeneratorFunction = customAlphabet(urlAlphabet) @@ -23,28 +44,27 @@ export function isValidTaskIdentifier(id: string): boolean { } /** - * Chunk a string into an array of strings of a given length + * Chunks a string into an array of chunks. * * @param text - string to chunk - * @param length - maximum length of each chunk - * @returns array of strings + * @param maxLength - maximum length of each chunk + * @returns array of chunks */ -export const chunkString = (text: string, length: number) => { +export const chunkString = (text: string, maxLength: number) => { const words = text.split(' ') const chunks: string[] = [] let chunk = '' for (const word of words) { - if (word.length > length) { + if (word.length > maxLength) { // Truncate the word if it's too long and indicate that it was truncated: - chunks.push(word.substring(0, length - 3) + '...') - } - - if ((chunk + word).length > length) { + chunks.push(word.substring(0, maxLength - 3) + '...') + } else if ((chunk + word + 1).length > maxLength) { + // +1 accounts for the space between words chunks.push(chunk.trim()) chunk = word } else { - chunk += ' ' + word + chunk += (chunk ? ' ' : '') + word } } diff --git a/test/utils.test.ts b/test/utils.test.ts index dbf4c2f..65413d6 100644 --- a/test/utils.test.ts +++ b/test/utils.test.ts @@ -1,6 +1,13 @@ import test from 'ava' -import { isValidTaskIdentifier } from '@/utils' +import { + chunkString, + defaultIDGeneratorFn, + extractJSONArrayFromString, + extractJSONObjectFromString, + isValidTaskIdentifier, + sleep +} from '@/utils' test('isValidTaskIdentifier - valid', async (t) => { t.true(isValidTaskIdentifier('foo')) @@ -18,3 +25,53 @@ test('isValidTaskIdentifier - invalid', async (t) => { t.false(isValidTaskIdentifier('x'.repeat(65))) t.false(isValidTaskIdentifier('-foo')) }) + +test('extractJSONObjectFromString should extract first JSON object from string', (t) => { + const jsonString = '{"name":"John"} Some other text {"name":"Doe"}' + const result = extractJSONObjectFromString(jsonString) + t.is(result, '{"name":"John"}') +}) + +test('extractJSONArrayFromString should extract first JSON array from string', (t) => { + const jsonString = '[1,2,3] Some other text [4,5,6]' + const result = extractJSONArrayFromString(jsonString) + t.is(result, '[1,2,3]') +}) + +test('extractJSONObjectFromString should return undefined if no JSON object is found', (t) => { + const jsonString = 'Some text' + const result = extractJSONObjectFromString(jsonString) + t.is(result, undefined) +}) + +test('extractJSONArrayFromString should return undefined if no JSON array is found', (t) => { + const jsonString = 'Some text' + const result = extractJSONArrayFromString(jsonString) + t.is(result, undefined) +}) + +test('sleep should delay execution', async (t) => { + const start = Date.now() + await sleep(1000) // for example, 1000ms / 1sec + const end = Date.now() + t.true(end - start >= 1000) +}) + +test('defaultIDGeneratorFn should generate URL-safe string', (t) => { + const result = defaultIDGeneratorFn() + + // Check if generated string matches URL-safe characters: + t.regex(result, /^[A-Za-z0-9\-_]+$/) +}) + +test('chunkString should split string into chunks', (t) => { + const text = 'Hello, this is a test string for chunkString function.' + const chunks = chunkString(text, 12) + t.deepEqual(chunks, [ + 'Hello, this', + 'is a test', + 'string for', + 'chunkString', + 'function.' + ]) +}) From ff5a878cc92f7dc3e92dfcb5047c17984c97eb41 Mon Sep 17 00:00:00 2001 From: Philipp Burckhardt Date: Wed, 14 Jun 2023 23:31:13 -0400 Subject: [PATCH 20/22] chore: fix example code and simplify code --- scratch/examples/human-feedback-cli.ts | 2 +- scratch/examples/human-feedback-slack.ts | 2 +- scratch/examples/human-feedback-twilio.ts | 2 +- src/human-feedback/feedback.ts | 9 +++------ 4 files changed, 6 insertions(+), 9 deletions(-) diff --git a/scratch/examples/human-feedback-cli.ts b/scratch/examples/human-feedback-cli.ts index 85a0a20..56312d7 100644 --- a/scratch/examples/human-feedback-cli.ts +++ b/scratch/examples/human-feedback-cli.ts @@ -23,7 +23,7 @@ async function main() { const topicJokesFeedback = withHumanFeedback(topicJokes, { type: 'selectN', annotations: false, - bail: false, + abort: false, editing: true, mechanism: HumanFeedbackMechanismCLI }) diff --git a/scratch/examples/human-feedback-slack.ts b/scratch/examples/human-feedback-slack.ts index c1b21e6..57444b9 100644 --- a/scratch/examples/human-feedback-slack.ts +++ b/scratch/examples/human-feedback-slack.ts @@ -23,7 +23,7 @@ async function main() { const topicJokesFeedback = withHumanFeedback(topicJokes, { type: 'selectN', annotations: false, - bail: false, + abort: false, editing: true, mechanism: HumanFeedbackMechanismSlack }) diff --git a/scratch/examples/human-feedback-twilio.ts b/scratch/examples/human-feedback-twilio.ts index e94807b..ec80b80 100644 --- a/scratch/examples/human-feedback-twilio.ts +++ b/scratch/examples/human-feedback-twilio.ts @@ -23,7 +23,7 @@ async function main() { const topicJokesFeedback = withHumanFeedback(topicJokes, { type: 'selectN', annotations: false, - bail: false, + abort: false, editing: true, mechanism: HumanFeedbackMechanismTwilio }) diff --git a/src/human-feedback/feedback.ts b/src/human-feedback/feedback.ts index 5a881da..c82bf5c 100644 --- a/src/human-feedback/feedback.ts +++ b/src/human-feedback/feedback.ts @@ -263,10 +263,7 @@ export function withHumanFeedback( ) { task = task.clone() - // Default options defined at the instance level - const instanceDefaults = task.agentic.humanFeedbackDefaults - - // Use Object.assign to merge the options, instance defaults, and hard-coded defaults + // Use Object.assign to merge the options, instance defaults, and hard-coded defaults: const finalOptions: HumanFeedbackOptions = Object.assign( { type: 'confirm', @@ -275,8 +272,8 @@ export function withHumanFeedback( annotations: false, mechanism: HumanFeedbackMechanismCLI }, - // Defaults from the instance: - instanceDefaults, + // Default options from the instance: + task.agentic.humanFeedbackDefaults, // User-provided options (override instance defaults): options ) From 4a2560432fb4ab5dea430fb6e9101174b11f6e0d Mon Sep 17 00:00:00 2001 From: Philipp Burckhardt Date: Wed, 14 Jun 2023 23:33:57 -0400 Subject: [PATCH 21/22] fix: update default to match type after renaming --- src/agentic.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/agentic.ts b/src/agentic.ts index 3905614..1f88a1a 100644 --- a/src/agentic.ts +++ b/src/agentic.ts @@ -67,7 +67,7 @@ export class Agentic { this._humanFeedbackDefaults = { type: 'confirm', - bail: false, + abort: false, editing: false, annotations: false, mechanism: HumanFeedbackMechanismCLI, From cb63718743d43a7967a1a7d78c7b6e5bf1f5d464 Mon Sep 17 00:00:00 2001 From: Travis Fischer Date: Wed, 14 Jun 2023 22:59:58 -0700 Subject: [PATCH 22/22] fix: JSON utils unit tests --- src/human-feedback/feedback.ts | 7 ++++++- src/task.ts | 1 - src/utils.ts | 4 ++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/human-feedback/feedback.ts b/src/human-feedback/feedback.ts index c82bf5c..3ae3c55 100644 --- a/src/human-feedback/feedback.ts +++ b/src/human-feedback/feedback.ts @@ -1,3 +1,4 @@ +import * as types from '@/types' import { Agentic } from '@/agentic' import { BaseTask } from '@/task' @@ -257,7 +258,11 @@ export abstract class HumanFeedbackMechanism< } } -export function withHumanFeedback( +export function withHumanFeedback< + TInput extends void | types.JsonObject, + TOutput extends types.JsonObject, + V extends HumanFeedbackType +>( task: BaseTask, options: HumanFeedbackOptions = {} ) { diff --git a/src/task.ts b/src/task.ts index 864f6b0..de8b9a0 100644 --- a/src/task.ts +++ b/src/task.ts @@ -4,7 +4,6 @@ import { ZodType } from 'zod' import * as errors from './errors' import * as types from './types' import type { Agentic } from './agentic' -import { defaultLogger } from './logger' import { defaultIDGeneratorFn, isValidTaskIdentifier } from './utils' /** diff --git a/src/utils.ts b/src/utils.ts index 075c996..d5954a7 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -9,7 +9,7 @@ import * as types from './types' * @returns extracted JSON object string, or `undefined` if no JSON object is found */ export function extractJSONObjectFromString(text: string): string | undefined { - return text.match(/\{(.|\n)*\}/gm)?.[0] + return text.match(/\{([^}]|\n)*\}/gm)?.[0] } /** @@ -19,7 +19,7 @@ export function extractJSONObjectFromString(text: string): string | undefined { * @returns extracted JSON array string, or `undefined` if no JSON array is found */ export function extractJSONArrayFromString(text: string): string | undefined { - return text.match(/\[(.|\n)*\]/gm)?.[0] + return text.match(/\[([^\]]|\n)*\]/gm)?.[0] } /**