From da9adf587e79f3e22bb5caa32ecdf55b83eb3b8c Mon Sep 17 00:00:00 2001 From: Philipp Burckhardt Date: Tue, 13 Jun 2023 12:36:45 -0400 Subject: [PATCH] feat: add initial Slack and Twilio human feedback classes --- legacy/src/human-feedback/cli.ts | 28 +-- legacy/src/human-feedback/feedback.ts | 156 ++++++++++++++ legacy/src/human-feedback/index.ts | 139 +----------- legacy/src/human-feedback/slack.ts | 286 +++++++++++++++++++++++++ legacy/src/human-feedback/twilio.ts | 294 ++++++++++++++++++++++++++ 5 files changed, 746 insertions(+), 157 deletions(-) create mode 100644 legacy/src/human-feedback/feedback.ts create mode 100644 legacy/src/human-feedback/slack.ts create mode 100644 legacy/src/human-feedback/twilio.ts diff --git a/legacy/src/human-feedback/cli.ts b/legacy/src/human-feedback/cli.ts index a503d0d5..f864f526 100644 --- a/legacy/src/human-feedback/cli.ts +++ b/legacy/src/human-feedback/cli.ts @@ -6,28 +6,12 @@ import select from '@inquirer/select' import { Agentic } from '@/agentic' import { TaskResponseMetadata } from '@/types' -import { HumanFeedbackMechanism, HumanFeedbackOptions } from './index' - -/** - * Actions the user can take in the feedback selection prompt. - */ -export const UserActions = { - Accept: 'accept', - Edit: 'edit', - Decline: 'decline', - Select: 'select', - Exit: 'exit' -} as const - -export type UserActions = (typeof UserActions)[keyof typeof UserActions] - -const UserActionMessages: Record = { - [UserActions.Accept]: 'Accept the output', - [UserActions.Edit]: 'Edit the output (open in editor)', - [UserActions.Decline]: 'Decline the output', - [UserActions.Select]: 'Select outputs to keep', - [UserActions.Exit]: 'Exit' -} +import { + HumanFeedbackMechanism, + HumanFeedbackOptions, + UserActionMessages, + UserActions +} from './feedback' /** * Prompt the user to select one of a list of options. diff --git a/legacy/src/human-feedback/feedback.ts b/legacy/src/human-feedback/feedback.ts new file mode 100644 index 00000000..41fac1bf --- /dev/null +++ b/legacy/src/human-feedback/feedback.ts @@ -0,0 +1,156 @@ +import { Agentic } from '@/agentic' +import { BaseTask } from '@/task' +import { TaskResponseMetadata } from '@/types' + +import { HumanFeedbackMechanismCLI } from './cli' + +/** + * Actions the user can take in the feedback selection prompt. + */ +export const UserActions = { + Accept: 'accept', + Edit: 'edit', + Decline: 'decline', + Select: 'select', + Exit: 'exit' +} as const + +export type UserActions = (typeof UserActions)[keyof typeof UserActions] + +export const UserActionMessages: Record = { + [UserActions.Accept]: 'Accept the output', + [UserActions.Edit]: 'Edit the output', + [UserActions.Decline]: 'Decline the output', + [UserActions.Select]: 'Select outputs to keep', + [UserActions.Exit]: 'Exit' +} + +/** + * Available types of human feedback. + */ +export type HumanFeedbackType = 'confirm' | 'selectOne' | 'selectN' + +type HumanFeedbackMechanismConstructor = new ( + ...args: any[] +) => T + +/** + * Options for human feedback. + */ +export type HumanFeedbackOptions< + T extends HumanFeedbackMechanism = HumanFeedbackMechanism +> = { + /** + * What type of feedback to request. + */ + type?: HumanFeedbackType + + /** + * Whether the user can bail out of the feedback loop. + */ + bail?: boolean + + /** + * Whether the user can edit the output. + */ + editing?: boolean + + /** + * Whether the user can add free-form text annotations. + */ + annotations?: boolean + + /** + * The human feedback mechanism to use for this task. + */ + mechanism?: HumanFeedbackMechanismConstructor +} + +export abstract class HumanFeedbackMechanism { + protected _agentic: Agentic + + protected _options: HumanFeedbackOptions + + constructor({ + agentic, + options + }: { + agentic: Agentic + options: HumanFeedbackOptions + }) { + this._agentic = agentic + this._options = options + } + + public abstract confirm( + response: any, + metadata: TaskResponseMetadata + ): Promise + public abstract selectOne( + response: any, + metadata: TaskResponseMetadata + ): Promise + public abstract selectN( + response: any, + metadata: TaskResponseMetadata + ): Promise + + public async interact(response: any, metadata: TaskResponseMetadata) { + if (this._options.type === 'selectN') { + await this.selectN(response, metadata) + } else if (this._options.type === 'confirm') { + await this.confirm(response, metadata) + } else if (this._options.type === 'selectOne') { + await this.selectOne(response, metadata) + } + } +} + +export function withHumanFeedback( + task: BaseTask, + options: HumanFeedbackOptions = {} +) { + task = task.clone() + + // Default options defined at the instance level + const instanceDefaults = task.agentic.humanFeedbackDefaults + + // Use Object.assign to merge the options, instance defaults, and hard-coded defaults + const finalOptions: HumanFeedbackOptions = Object.assign( + { + type: 'confirm', + bail: false, + editing: false, + annotations: false, + mechanism: HumanFeedbackMechanismCLI + }, + // Defaults from the instance: + instanceDefaults, + // User-provided options (override instance defaults): + options + ) + + if (!finalOptions.mechanism) { + throw new Error( + 'No feedback mechanism provided. Please provide a feedback mechanism to use.' + ) + } + + const feedbackMechanism = new finalOptions.mechanism({ + agentic: task.agentic, + options: finalOptions + }) + + const originalCall = task.callWithMetadata.bind(task) + + task.callWithMetadata = async function (input?: T) { + const response = await originalCall(input) + + // Process the response and add feedback to metadata + await feedbackMechanism.interact(response.result, response.metadata) + + return response + } + + return task +} diff --git a/legacy/src/human-feedback/index.ts b/legacy/src/human-feedback/index.ts index e2af21dc..105c8765 100644 --- a/legacy/src/human-feedback/index.ts +++ b/legacy/src/human-feedback/index.ts @@ -1,135 +1,4 @@ -import { Agentic } from '@/agentic' -import { BaseTask } from '@/task' -import { TaskResponseMetadata } from '@/types' - -import { HumanFeedbackMechanismCLI } from './cli' - -/** - * Available types of human feedback. - */ -export type HumanFeedbackType = 'confirm' | 'selectOne' | 'selectN' - -type HumanFeedbackMechanismConstructor = new ( - ...args: any[] -) => T - -/** - * Options for human feedback. - */ -export type HumanFeedbackOptions< - T extends HumanFeedbackMechanism = HumanFeedbackMechanism -> = { - /** - * What type of feedback to request. - */ - type?: HumanFeedbackType - - /** - * Whether the user can bail out of the feedback loop. - */ - bail?: boolean - - /** - * Whether the user can edit the output. - */ - editing?: boolean - - /** - * Whether the user can add free-form text annotations. - */ - annotations?: boolean - - /** - * The human feedback mechanism to use for this task. - */ - mechanism?: HumanFeedbackMechanismConstructor -} - -export abstract class HumanFeedbackMechanism { - protected _agentic: Agentic - - protected _options: HumanFeedbackOptions - - constructor({ - agentic, - options - }: { - agentic: Agentic - options: HumanFeedbackOptions - }) { - this._agentic = agentic - this._options = options - } - - public abstract confirm( - response: any, - metadata: TaskResponseMetadata - ): Promise - public abstract selectOne( - response: any, - metadata: TaskResponseMetadata - ): Promise - public abstract selectN( - response: any, - metadata: TaskResponseMetadata - ): Promise - - public async interact(response: any, metadata: TaskResponseMetadata) { - if (this._options.type === 'selectN') { - await this.selectN(response, metadata) - } else if (this._options.type === 'confirm') { - await this.confirm(response, metadata) - } else if (this._options.type === 'selectOne') { - await this.selectOne(response, metadata) - } - } -} - -export function withHumanFeedback( - task: BaseTask, - options: HumanFeedbackOptions = {} -) { - task = task.clone() - - // Default options defined at the instance level - const instanceDefaults = task.agentic.humanFeedbackDefaults - - // Use Object.assign to merge the options, instance defaults, and hard-coded defaults - const finalOptions: HumanFeedbackOptions = Object.assign( - { - type: 'confirm', - bail: false, - editing: false, - annotations: false, - mechanism: HumanFeedbackMechanismCLI - }, - // Defaults from the instance: - instanceDefaults, - // User-provided options (override instance defaults): - options - ) - - if (!finalOptions.mechanism) { - throw new Error( - 'No feedback mechanism provided. Please provide a feedback mechanism to use.' - ) - } - - const feedbackMechanism = new finalOptions.mechanism({ - agentic: task.agentic, - options: finalOptions - }) - - const originalCall = task.callWithMetadata.bind(task) - - task.callWithMetadata = async function (input?: T) { - const response = await originalCall(input) - - // Process the response and add feedback to metadata - await feedbackMechanism.interact(response.result, response.metadata) - - return response - } - - return task -} +export * from './cli' +export * from './feedback' +export * from './slack' +export * from './twilio' diff --git a/legacy/src/human-feedback/slack.ts b/legacy/src/human-feedback/slack.ts new file mode 100644 index 00000000..5f3d1ea2 --- /dev/null +++ b/legacy/src/human-feedback/slack.ts @@ -0,0 +1,286 @@ +import { Agentic } from '@/agentic' +import { SlackClient } from '@/services/slack' +import { TaskResponseMetadata } from '@/types' + +import { + HumanFeedbackMechanism, + HumanFeedbackOptions, + UserActionMessages, + UserActions +} from './feedback' + +export class HumanFeedbackMechanismSlack extends HumanFeedbackMechanism { + private slackClient: SlackClient + + constructor({ + agentic, + options + }: { + agentic: Agentic + options: HumanFeedbackOptions + }) { + super({ agentic, options }) + this.slackClient = new SlackClient() + } + + private async askUser( + message: string, + choices: UserActions[] + ): Promise { + message += '\n\n' + message += choices + .map((choice, idx) => `*${idx}* - ${UserActionMessages[choice]}`) + .join('\n') + message += '\n\n' + message += 'Reply with the number of your choice.' + const response = await this.slackClient.sendAndWaitForReply({ + text: message, + validate: (slackMessage) => { + const choice = parseInt(slackMessage.text) + return !isNaN(choice) && choice >= 0 && choice < choices.length + } + }) + return choices[parseInt(response.text)] + } + + public async confirm( + response: any, + metadata: TaskResponseMetadata + ): Promise { + const stringified = JSON.stringify(response, null, 2) + const msg = [ + 'The following output was generated:', + '```', + stringified, + '```', + 'What would you like to do?' + ].join('\n') + + const choices: UserActions[] = [UserActions.Accept, UserActions.Decline] + + if (this._options.editing) { + choices.push(UserActions.Edit) + } + + if (this._options.bail) { + choices.push(UserActions.Exit) + } + + const feedback = await this.askUser(msg, choices) + + metadata.feedback = {} + + switch (feedback) { + case UserActions.Accept: + metadata.feedback.accepted = true + break + + case UserActions.Edit: { + let { text: editedOutput } = await this.slackClient.sendAndWaitForReply( + { + text: 'Copy and edit the output:' + } + ) + editedOutput = editedOutput.replace(/```$/g, '') + editedOutput = editedOutput.replace(/^```/g, '') + metadata.feedback.editedOutput = editedOutput + break + } + + case UserActions.Decline: + metadata.feedback.accepted = false + break + + case UserActions.Exit: + throw new Error('Exiting...') + + default: + throw new Error(`Unexpected feedback: ${feedback}`) + } + + if (this._options.annotations) { + try { + const annotation = await this.slackClient.sendAndWaitForReply({ + text: 'Please leave an annotation (optional):' + }) + + if (annotation) { + metadata.feedback.annotation = annotation.text + } + } catch (e) { + // Deliberately swallow the error here as the user is not required to leave an annotation + } + } + } + + public async selectOne( + response: any[], + metadata: TaskResponseMetadata + ): Promise { + const stringified = JSON.stringify(response, null, 2) + const msg = [ + 'The following output was generated:', + '```', + stringified, + '```', + 'What would you like to do?' + ].join('\n') + + const choices: UserActions[] = [UserActions.Select] + + if (this._options.editing) { + choices.push(UserActions.Edit) + } + + if (this._options.bail) { + choices.push(UserActions.Exit) + } + + const feedback = + choices.length === 1 + ? UserActions.Select + : await this.askUser(msg, choices) + + metadata.feedback = {} + + switch (feedback) { + case UserActions.Edit: { + let { text: editedOutput } = await this.slackClient.sendAndWaitForReply( + { + text: 'Copy and edit the output:' + } + ) + editedOutput = editedOutput.replace(/```$/g, '') + editedOutput = editedOutput.replace(/^```/g, '') + metadata.feedback.editedOutput = editedOutput + break + } + + case UserActions.Select: { + const { text: selectedOutput } = + await this.slackClient.sendAndWaitForReply({ + text: + 'Pick one output:' + + response.map((r, idx) => `\n*${idx}* - ${r}`).join('') + + '\n\nReply with the number of your choice.', + validate: (slackMessage) => { + const choice = parseInt(slackMessage.text) + return !isNaN(choice) && choice >= 0 && choice < response.length + } + }) + metadata.feedback.chosen = response[parseInt(selectedOutput)] + break + } + + case UserActions.Exit: + throw new Error('Exiting...') + + default: + throw new Error(`Unexpected feedback: ${feedback}`) + } + + if (this._options.annotations) { + try { + const annotation = await this.slackClient.sendAndWaitForReply({ + text: 'Please leave an annotation (optional):' + }) + + if (annotation) { + metadata.feedback.annotation = annotation.text + } + } catch (e) { + // Deliberately swallow the error here as the user is not required to leave an annotation + } + } + } + + public async selectN( + response: any[], + metadata: TaskResponseMetadata + ): Promise { + const stringified = JSON.stringify(response, null, 2) + const msg = [ + 'The following output was generated:', + stringified, + 'What would you like to do?' + ].join('\n') + const choices: UserActions[] = [UserActions.Select] + + if (this._options.editing) { + choices.push(UserActions.Edit) + } + + if (this._options.bail) { + choices.push(UserActions.Exit) + } + + const feedback = + choices.length === 1 + ? UserActions.Select + : await this.askUser(msg, choices) + + metadata.feedback = {} + + switch (feedback) { + case UserActions.Edit: { + let { text: editedOutput } = await this.slackClient.sendAndWaitForReply( + { + text: 'Copy and edit the output:' + } + ) + editedOutput = editedOutput.replace(/```$/g, '') + editedOutput = editedOutput.replace(/^```/g, '') + metadata.feedback.editedOutput = editedOutput + break + } + + case UserActions.Select: { + const { text: selectedOutput } = + await this.slackClient.sendAndWaitForReply({ + text: + 'Select outputs:' + + response.map((r, idx) => `\n*${idx}* - ${r}`).join('') + + '\n\nReply with a comma-separated list of the output numbers of your choice.', + validate: (slackMessage) => { + const choices = slackMessage.text.split(',') + return choices.every((choice) => { + const choiceInt = parseInt(choice) + return ( + !isNaN(choiceInt) && + choiceInt >= 0 && + choiceInt < response.length + ) + }) + } + }) + const chosenOutputs = selectedOutput + .split(',') + .map((choice) => parseInt(choice)) + metadata.feedback.selected = response.filter((_, idx) => { + return chosenOutputs.includes(idx) + }) + break + } + + case UserActions.Exit: + throw new Error('Exiting...') + + default: + throw new Error(`Unexpected feedback: ${feedback}`) + } + + if (this._options.annotations) { + try { + const annotation = await this.slackClient.sendAndWaitForReply({ + text: 'Please leave an annotation (optional):' + }) + + if (annotation) { + metadata.feedback.annotation = annotation.text + } + } catch (e) { + // Deliberately swallow the error here as the user is not required to leave an annotation + } + } + } +} diff --git a/legacy/src/human-feedback/twilio.ts b/legacy/src/human-feedback/twilio.ts new file mode 100644 index 00000000..383f35b1 --- /dev/null +++ b/legacy/src/human-feedback/twilio.ts @@ -0,0 +1,294 @@ +import { Agentic } from '@/agentic' +import { TwilioConversationClient } from '@/services/twilio-conversation' +import { TaskResponseMetadata } from '@/types' + +import { + HumanFeedbackMechanism, + HumanFeedbackOptions, + UserActionMessages, + UserActions +} from './feedback' + +export class HumanFeedbackMechanismTwilio extends HumanFeedbackMechanism { + private twilioClient: TwilioConversationClient + + constructor({ + agentic, + options + }: { + agentic: Agentic + options: HumanFeedbackOptions + }) { + super({ agentic, options }) + this.twilioClient = new TwilioConversationClient() + } + + private async askUser( + message: string, + choices: UserActions[] + ): Promise { + message += '\n\n' + message += choices + .map((choice, idx) => `*${idx}* - ${UserActionMessages[choice]}`) + .join('\n') + message += '\n\n' + message += 'Reply with the number of your choice.' + const response = await this.twilioClient.sendAndWaitForReply({ + name: 'human-feedback-ask', + text: message, + validate: (message) => { + const choice = parseInt(message.body) + return !isNaN(choice) && choice >= 0 && choice < choices.length + } + }) + return choices[parseInt(response.body)] + } + + public async confirm( + response: any, + metadata: TaskResponseMetadata + ): Promise { + const stringified = JSON.stringify(response, null, 2) + const msg = [ + 'The following output was generated:', + '', + stringified, + '', + 'What would you like to do?' + ].join('\n') + + const choices: UserActions[] = [UserActions.Accept, UserActions.Decline] + + if (this._options.editing) { + choices.push(UserActions.Edit) + } + + if (this._options.bail) { + choices.push(UserActions.Exit) + } + + const feedback = await this.askUser(msg, choices) + + metadata.feedback = {} + + switch (feedback) { + case UserActions.Accept: + metadata.feedback.accepted = true + break + + case UserActions.Edit: { + let { body: editedOutput } = + await this.twilioClient.sendAndWaitForReply({ + name: 'human-feedback-edit', + text: 'Copy and edit the output:' + }) + editedOutput = editedOutput.replace(/```$/g, '') + editedOutput = editedOutput.replace(/^```/g, '') + metadata.feedback.editedOutput = editedOutput + break + } + + case UserActions.Decline: + metadata.feedback.accepted = false + break + + case UserActions.Exit: + throw new Error('Exiting...') + + default: + throw new Error(`Unexpected feedback: ${feedback}`) + } + + if (this._options.annotations) { + try { + const annotation = await this.twilioClient.sendAndWaitForReply({ + name: 'human-feedback-annotation', + text: 'Please leave an annotation (optional):' + }) + + if (annotation) { + metadata.feedback.annotation = annotation.body + } + } catch (e) { + // Deliberately swallow the error here as the user is not required to leave an annotation + } + } + } + + public async selectOne( + response: any[], + metadata: TaskResponseMetadata + ): Promise { + const stringified = JSON.stringify(response, null, 2) + const msg = [ + 'The following output was generated:', + '', + stringified, + '', + 'What would you like to do?' + ].join('\n') + + const choices: UserActions[] = [UserActions.Select] + + if (this._options.editing) { + choices.push(UserActions.Edit) + } + + if (this._options.bail) { + choices.push(UserActions.Exit) + } + + const feedback = + choices.length === 1 + ? UserActions.Select + : await this.askUser(msg, choices) + + metadata.feedback = {} + + switch (feedback) { + case UserActions.Edit: { + let { body: editedOutput } = + await this.twilioClient.sendAndWaitForReply({ + text: 'Copy and edit the output:', + name: 'human-feedback-edit' + }) + editedOutput = editedOutput.replace(/```$/g, '') + editedOutput = editedOutput.replace(/^```/g, '') + metadata.feedback.editedOutput = editedOutput + break + } + + case UserActions.Select: { + const { body: selectedOutput } = + await this.twilioClient.sendAndWaitForReply({ + name: 'human-feedback-select', + text: + 'Pick one output:' + + response.map((r, idx) => `\n${idx} - ${r}`).join('') + + '\n\nReply with the number of your choice.', + validate: (message) => { + const choice = parseInt(message.body) + return !isNaN(choice) && choice >= 0 && choice < response.length + } + }) + metadata.feedback.chosen = response[parseInt(selectedOutput)] + break + } + + case UserActions.Exit: + throw new Error('Exiting...') + + default: + throw new Error(`Unexpected feedback: ${feedback}`) + } + + if (this._options.annotations) { + try { + const annotation = await this.twilioClient.sendAndWaitForReply({ + text: 'Please leave an annotation (optional):', + name: 'human-feedback-annotation' + }) + + if (annotation) { + metadata.feedback.annotation = annotation.body + } + } catch (e) { + // Deliberately swallow the error here as the user is not required to leave an annotation + } + } + } + + public async selectN( + response: any[], + metadata: TaskResponseMetadata + ): Promise { + const stringified = JSON.stringify(response, null, 2) + const msg = [ + 'The following output was generated:', + '', + stringified, + '', + 'What would you like to do?' + ].join('\n') + const choices: UserActions[] = [UserActions.Select] + + if (this._options.editing) { + choices.push(UserActions.Edit) + } + + if (this._options.bail) { + choices.push(UserActions.Exit) + } + + const feedback = + choices.length === 1 + ? UserActions.Select + : await this.askUser(msg, choices) + + metadata.feedback = {} + + switch (feedback) { + case UserActions.Edit: { + let { body: editedOutput } = + await this.twilioClient.sendAndWaitForReply({ + text: 'Copy and edit the output:', + name: 'human-feedback-edit' + }) + editedOutput = editedOutput.replace(/```$/g, '') + editedOutput = editedOutput.replace(/^```/g, '') + metadata.feedback.editedOutput = editedOutput + break + } + + case UserActions.Select: { + const { body: selectedOutput } = + await this.twilioClient.sendAndWaitForReply({ + name: 'human-feedback-select', + text: + 'Select outputs:' + + response.map((r, idx) => `\n${idx} - ${r}`).join('') + + '\n\nReply with a comma-separated list of the output numbers of your choice.', + validate: (message) => { + const choices = message.body.split(',') + return choices.every((choice) => { + const choiceInt = parseInt(choice) + return ( + !isNaN(choiceInt) && + choiceInt >= 0 && + choiceInt < response.length + ) + }) + } + }) + const chosenOutputs = selectedOutput + .split(',') + .map((choice) => parseInt(choice)) + metadata.feedback.selected = response.filter((_, idx) => { + return chosenOutputs.includes(idx) + }) + break + } + + case UserActions.Exit: + throw new Error('Exiting...') + + default: + throw new Error(`Unexpected feedback: ${feedback}`) + } + + if (this._options.annotations) { + try { + const annotation = await this.twilioClient.sendAndWaitForReply({ + text: 'Please leave an annotation (optional):', + name: 'human-feedback-annotation' + }) + + if (annotation) { + metadata.feedback.annotation = annotation.body + } + } catch (e) { + // Deliberately swallow the error here as the user is not required to leave an annotation + } + } + } +}