From 9c41ea56076f7b53e5ed28a7ce108a9f0cd5ea2b Mon Sep 17 00:00:00 2001 From: Philipp Burckhardt Date: Tue, 13 Jun 2023 16:32:58 -0400 Subject: [PATCH] refactor: avoid code duplication --- src/human-feedback/cli.ts | 214 +++++---------------------- src/human-feedback/feedback.ts | 103 ++++++++++--- src/human-feedback/slack.ts | 250 ++++++------------------------- src/human-feedback/twilio.ts | 259 +++++++-------------------------- 4 files changed, 216 insertions(+), 610 deletions(-) diff --git a/src/human-feedback/cli.ts b/src/human-feedback/cli.ts index ad5f263..e4a23e0 100644 --- a/src/human-feedback/cli.ts +++ b/src/human-feedback/cli.ts @@ -4,7 +4,6 @@ import input from '@inquirer/input' import select from '@inquirer/select' import { Agentic } from '@/agentic' -import { TaskResponseMetadata } from '@/types' import { HumanFeedbackMechanism, @@ -13,22 +12,6 @@ import { UserActions } from './feedback' -/** - * Prompt the user to select one of a list of options. - */ -async function askUser( - message: string, - choices: UserActions[] -): Promise { - return select({ - message, - choices: choices.map((choice) => ({ - name: UserActionMessages[choice], - value: choice - })) - }) -} - export class HumanFeedbackMechanismCLI extends HumanFeedbackMechanism { constructor({ agentic, @@ -42,172 +25,49 @@ export class HumanFeedbackMechanismCLI extends HumanFeedbackMechanism { this._options = options } - protected async annotate( - response: any, - metadata: TaskResponseMetadata - ): Promise { - const annotation = await input({ + /** + * Prompt the user to select one of a list of options. + */ + protected async askUser( + message: string, + choices: UserActions[] + ): Promise { + return select({ + message, + choices: choices.map((choice) => ({ + name: UserActionMessages[choice], + value: choice + })) + }) + } + + protected async edit(output: string): Promise { + return editor({ + message: 'Edit the output:', + default: output + }) + } + + protected async annotate(): Promise { + return input({ message: 'Please leave an annotation (leave blank to skip; press enter to submit):' }) - - if (annotation) { - metadata.feedback.annotation = annotation - } } - protected async confirm( - response: any, - metadata: TaskResponseMetadata - ): Promise { - const stringified = JSON.stringify(response, null, 2) - const msg = [ - 'The following output was generated:', - stringified, - 'What would you like to do?' - ].join('\n') - const choices: UserActions[] = [UserActions.Accept, UserActions.Decline] - - if (this._options.editing) { - choices.push(UserActions.Edit) - } - - if (this._options.bail) { - choices.push(UserActions.Exit) - } - - const feedback = await askUser(msg, choices) - - metadata.feedback = {} - - switch (feedback) { - case UserActions.Accept: - metadata.feedback.accepted = true - break - - case UserActions.Edit: { - const editedOutput = await editor({ - message: 'Edit the output:', - default: stringified - }) - metadata.feedback.editedOutput = editedOutput - break - } - - case UserActions.Decline: - metadata.feedback.accepted = false - break - - case UserActions.Exit: - throw new Error('Exiting...') - - default: - throw new Error(`Unexpected feedback: ${feedback}`) - } + protected async selectOne(response: any[]): Promise { + const choices = response.map((option) => ({ + name: option, + value: option + })) + return select({ message: 'Pick one output:', choices }) } - protected async selectOne( - response: any[], - metadata: TaskResponseMetadata - ): Promise { - const stringified = JSON.stringify(response, null, 2) - const msg = [ - 'The following output was generated:', - stringified, - 'What would you like to do?' - ].join('\n') - const choices: UserActions[] = [UserActions.Select] - - if (this._options.editing) { - choices.push(UserActions.Edit) - } - - if (this._options.bail) { - choices.push(UserActions.Exit) - } - - const feedback = await askUser(msg, choices) - - metadata.feedback = {} - - switch (feedback) { - case UserActions.Edit: { - const editedOutput = await editor({ - message: 'Edit the output:', - default: stringified - }) - metadata.feedback.editedOutput = editedOutput - break - } - - case UserActions.Select: { - const choices = response.map((option) => ({ - name: option, - value: option - })) - const chosen = await select({ message: 'Pick one output:', choices }) - metadata.feedback.chosen = chosen - break - } - - case UserActions.Exit: - throw new Error('Exiting...') - - default: - throw new Error(`Unexpected feedback: ${feedback}`) - } - } - - protected async selectN( - response: any[], - metadata: TaskResponseMetadata - ): Promise { - const stringified = JSON.stringify(response, null, 2) - const msg = [ - 'The following output was generated:', - stringified, - 'What would you like to do?' - ].join('\n') - const choices: UserActions[] = [UserActions.Select] - - if (this._options.editing) { - choices.push(UserActions.Edit) - } - - if (this._options.bail) { - choices.push(UserActions.Exit) - } - - const feedback = - choices.length === 1 ? UserActions.Select : await askUser(msg, choices) - - metadata.feedback = {} - - switch (feedback) { - case UserActions.Edit: { - const editedOutput = await editor({ - message: 'Edit the output:', - default: stringified - }) - metadata.feedback.editedOutput = editedOutput - break - } - - case UserActions.Select: { - const choices = response.map((option) => ({ - name: option, - value: option - })) - const chosen = await checkbox({ message: 'Select outputs:', choices }) - metadata.feedback.selected = chosen - break - } - - case UserActions.Exit: - throw new Error('Exiting...') - - default: - throw new Error(`Unexpected feedback: ${feedback}`) - } + protected async selectN(response: any[]): Promise { + const choices = response.map((option) => ({ + name: option, + value: option + })) + return checkbox({ message: 'Select outputs:', choices }) } } diff --git a/src/human-feedback/feedback.ts b/src/human-feedback/feedback.ts index 4619a9d..27d12f7 100644 --- a/src/human-feedback/feedback.ts +++ b/src/human-feedback/feedback.ts @@ -82,35 +82,92 @@ export abstract class HumanFeedbackMechanism { this._options = options } - protected abstract confirm( - response: any, - metadata: TaskResponseMetadata - ): Promise - protected abstract selectOne( - response: any, - metadata: TaskResponseMetadata - ): Promise - protected abstract selectN( - response: any, - metadata: TaskResponseMetadata - ): Promise + protected abstract selectOne(response: any): Promise - protected abstract annotate( - response: any, - metadata: TaskResponseMetadata - ): Promise + protected abstract selectN(response: any): Promise + + protected abstract annotate(): Promise + + protected abstract edit(output: string): Promise + + protected abstract askUser( + message: string, + choices: UserActions[] + ): Promise public async interact(response: any, metadata: TaskResponseMetadata) { - if (this._options.type === 'selectN') { - await this.selectN(response, metadata) - } else if (this._options.type === 'confirm') { - await this.confirm(response, metadata) - } else if (this._options.type === 'selectOne') { - await this.selectOne(response, metadata) + const stringified = JSON.stringify(response, null, 2) + const msg = [ + 'The following output was generated:', + '```', + stringified, + '```', + 'What would you like to do?' + ].join('\n') + + const choices: UserActions[] = [] + if ( + this._options.type === 'selectN' || + this._options.type === 'selectOne' + ) { + choices.push(UserActions.Select) + } else { + // Case: confirm + choices.push(UserActions.Accept) + choices.push(UserActions.Decline) + } + + if (this._options.editing) { + choices.push(UserActions.Edit) + } + + if (this._options.bail) { + choices.push(UserActions.Exit) + } + + const feedback = + choices.length === 1 + ? UserActions.Select + : await this.askUser(msg, choices) + + metadata.feedback = {} + + switch (feedback) { + case UserActions.Accept: + metadata.feedback.accepted = true + break + + case UserActions.Edit: { + const editedOutput = await this.edit(stringified) + metadata.feedback.editedOutput = editedOutput + break + } + + case UserActions.Decline: + metadata.feedback.accepted = false + break + + case UserActions.Select: + if (this._options.type === 'selectN') { + metadata.feedback.selected = await this.selectN(response) + } else if (this._options.type === 'selectOne') { + metadata.feedback.chosen = await this.selectOne(response) + } + + break + + case UserActions.Exit: + throw new Error('Exiting...') + + default: + throw new Error(`Unexpected feedback: ${feedback}`) } if (this._options.annotations) { - await this.annotate(response, metadata) + const annotation = await this.annotate() + if (annotation) { + metadata.feedback.annotation = annotation + } } } } diff --git a/src/human-feedback/slack.ts b/src/human-feedback/slack.ts index e8172d1..7464036 100644 --- a/src/human-feedback/slack.ts +++ b/src/human-feedback/slack.ts @@ -1,6 +1,5 @@ import { Agentic } from '@/agentic' import { SlackClient } from '@/services/slack' -import { TaskResponseMetadata } from '@/types' import { HumanFeedbackMechanism, @@ -23,24 +22,28 @@ export class HumanFeedbackMechanismSlack extends HumanFeedbackMechanism { this.slackClient = new SlackClient() } - protected async annotate( - response: any, - metadata: TaskResponseMetadata - ): Promise { + protected async annotate(): Promise { try { const annotation = await this.slackClient.sendAndWaitForReply({ text: 'Please leave an annotation (optional):' }) - - if (annotation) { - metadata.feedback.annotation = annotation.text - } + return annotation.text } catch (e) { // Deliberately swallow the error here as the user is not required to leave an annotation + return '' } } - private async askUser( + protected async edit(): Promise { + let { text: editedOutput } = await this.slackClient.sendAndWaitForReply({ + text: 'Copy and edit the output:' + }) + editedOutput = editedOutput.replace(/```$/g, '') + editedOutput = editedOutput.replace(/^```/g, '') + return editedOutput + } + + protected async askUser( message: string, choices: UserActions[] ): Promise { @@ -60,202 +63,45 @@ export class HumanFeedbackMechanismSlack extends HumanFeedbackMechanism { return choices[parseInt(response.text)] } - public async confirm( - response: any, - metadata: TaskResponseMetadata - ): Promise { - const stringified = JSON.stringify(response, null, 2) - const msg = [ - 'The following output was generated:', - '```', - stringified, - '```', - 'What would you like to do?' - ].join('\n') - - const choices: UserActions[] = [UserActions.Accept, UserActions.Decline] - - if (this._options.editing) { - choices.push(UserActions.Edit) - } - - if (this._options.bail) { - choices.push(UserActions.Exit) - } - - const feedback = await this.askUser(msg, choices) - - metadata.feedback = {} - - switch (feedback) { - case UserActions.Accept: - metadata.feedback.accepted = true - break - - case UserActions.Edit: { - let { text: editedOutput } = await this.slackClient.sendAndWaitForReply( - { - text: 'Copy and edit the output:' - } - ) - editedOutput = editedOutput.replace(/```$/g, '') - editedOutput = editedOutput.replace(/^```/g, '') - metadata.feedback.editedOutput = editedOutput - break + public async selectOne(response: any[]): Promise { + const { text: selectedOutput } = await this.slackClient.sendAndWaitForReply( + { + text: + 'Pick one output:' + + response.map((r, idx) => `\n*${idx}* - ${r}`).join('') + + '\n\nReply with the number of your choice.', + validate: (slackMessage) => { + const choice = parseInt(slackMessage.text) + return !isNaN(choice) && choice >= 0 && choice < response.length + } } - - case UserActions.Decline: - metadata.feedback.accepted = false - break - - case UserActions.Exit: - throw new Error('Exiting...') - - default: - throw new Error(`Unexpected feedback: ${feedback}`) - } + ) + return response[parseInt(selectedOutput)] } - public async selectOne( - response: any[], - metadata: TaskResponseMetadata - ): Promise { - const stringified = JSON.stringify(response, null, 2) - const msg = [ - 'The following output was generated:', - '```', - stringified, - '```', - 'What would you like to do?' - ].join('\n') - - const choices: UserActions[] = [UserActions.Select] - - if (this._options.editing) { - choices.push(UserActions.Edit) - } - - if (this._options.bail) { - choices.push(UserActions.Exit) - } - - const feedback = - choices.length === 1 - ? UserActions.Select - : await this.askUser(msg, choices) - - metadata.feedback = {} - - switch (feedback) { - case UserActions.Edit: { - let { text: editedOutput } = await this.slackClient.sendAndWaitForReply( - { - text: 'Copy and edit the output:' - } - ) - editedOutput = editedOutput.replace(/```$/g, '') - editedOutput = editedOutput.replace(/^```/g, '') - metadata.feedback.editedOutput = editedOutput - break - } - - case UserActions.Select: { - const { text: selectedOutput } = - await this.slackClient.sendAndWaitForReply({ - text: - 'Pick one output:' + - response.map((r, idx) => `\n*${idx}* - ${r}`).join('') + - '\n\nReply with the number of your choice.', - validate: (slackMessage) => { - const choice = parseInt(slackMessage.text) - return !isNaN(choice) && choice >= 0 && choice < response.length - } + public async selectN(response: any[]): Promise { + const { text: selectedOutput } = await this.slackClient.sendAndWaitForReply( + { + text: + 'Select outputs:' + + response.map((r, idx) => `\n*${idx}* - ${r}`).join('') + + '\n\nReply with a comma-separated list of the output numbers of your choice.', + validate: (slackMessage) => { + const choices = slackMessage.text.split(',') + return choices.every((choice) => { + const choiceInt = parseInt(choice) + return ( + !isNaN(choiceInt) && choiceInt >= 0 && choiceInt < response.length + ) }) - metadata.feedback.chosen = response[parseInt(selectedOutput)] - break + } } - - case UserActions.Exit: - throw new Error('Exiting...') - - default: - throw new Error(`Unexpected feedback: ${feedback}`) - } - } - - public async selectN( - response: any[], - metadata: TaskResponseMetadata - ): Promise { - const stringified = JSON.stringify(response, null, 2) - const msg = [ - 'The following output was generated:', - stringified, - 'What would you like to do?' - ].join('\n') - const choices: UserActions[] = [UserActions.Select] - - if (this._options.editing) { - choices.push(UserActions.Edit) - } - - if (this._options.bail) { - choices.push(UserActions.Exit) - } - - const feedback = - choices.length === 1 - ? UserActions.Select - : await this.askUser(msg, choices) - - metadata.feedback = {} - - switch (feedback) { - case UserActions.Edit: { - let { text: editedOutput } = await this.slackClient.sendAndWaitForReply( - { - text: 'Copy and edit the output:' - } - ) - editedOutput = editedOutput.replace(/```$/g, '') - editedOutput = editedOutput.replace(/^```/g, '') - metadata.feedback.editedOutput = editedOutput - break - } - - case UserActions.Select: { - const { text: selectedOutput } = - await this.slackClient.sendAndWaitForReply({ - text: - 'Select outputs:' + - response.map((r, idx) => `\n*${idx}* - ${r}`).join('') + - '\n\nReply with a comma-separated list of the output numbers of your choice.', - validate: (slackMessage) => { - const choices = slackMessage.text.split(',') - return choices.every((choice) => { - const choiceInt = parseInt(choice) - return ( - !isNaN(choiceInt) && - choiceInt >= 0 && - choiceInt < response.length - ) - }) - } - }) - const chosenOutputs = selectedOutput - .split(',') - .map((choice) => parseInt(choice)) - metadata.feedback.selected = response.filter((_, idx) => { - return chosenOutputs.includes(idx) - }) - break - } - - case UserActions.Exit: - throw new Error('Exiting...') - - default: - throw new Error(`Unexpected feedback: ${feedback}`) - } + ) + const chosenOutputs = selectedOutput + .split(',') + .map((choice) => parseInt(choice)) + return response.filter((_, idx) => { + return chosenOutputs.includes(idx) + }) } } diff --git a/src/human-feedback/twilio.ts b/src/human-feedback/twilio.ts index d7cae0c..753578b 100644 --- a/src/human-feedback/twilio.ts +++ b/src/human-feedback/twilio.ts @@ -1,6 +1,5 @@ import { Agentic } from '@/agentic' import { TwilioConversationClient } from '@/services/twilio-conversation' -import { TaskResponseMetadata } from '@/types' import { HumanFeedbackMechanism, @@ -23,25 +22,30 @@ export class HumanFeedbackMechanismTwilio extends HumanFeedbackMechanism { this.twilioClient = new TwilioConversationClient() } - protected async annotate( - response: any, - metadata: TaskResponseMetadata - ): Promise { + protected async annotate(): Promise { try { const annotation = await this.twilioClient.sendAndWaitForReply({ name: 'human-feedback-annotation', text: 'Please leave an annotation (optional):' }) - - if (annotation) { - metadata.feedback.annotation = annotation.body - } + return annotation.body } catch (e) { // Deliberately swallow the error here as the user is not required to leave an annotation + return '' } } - private async askUser( + protected async edit(): Promise { + let { body: editedOutput } = await this.twilioClient.sendAndWaitForReply({ + text: 'Copy and edit the output:', + name: 'human-feedback-edit' + }) + editedOutput = editedOutput.replace(/```$/g, '') + editedOutput = editedOutput.replace(/^```/g, '') + return editedOutput + } + + protected async askUser( message: string, choices: UserActions[] ): Promise { @@ -62,206 +66,45 @@ export class HumanFeedbackMechanismTwilio extends HumanFeedbackMechanism { return choices[parseInt(response.body)] } - public async confirm( - response: any, - metadata: TaskResponseMetadata - ): Promise { - const stringified = JSON.stringify(response, null, 2) - const msg = [ - 'The following output was generated:', - '', - stringified, - '', - 'What would you like to do?' - ].join('\n') - - const choices: UserActions[] = [UserActions.Accept, UserActions.Decline] - - if (this._options.editing) { - choices.push(UserActions.Edit) - } - - if (this._options.bail) { - choices.push(UserActions.Exit) - } - - const feedback = await this.askUser(msg, choices) - - metadata.feedback = {} - - switch (feedback) { - case UserActions.Accept: - metadata.feedback.accepted = true - break - - case UserActions.Edit: { - let { body: editedOutput } = - await this.twilioClient.sendAndWaitForReply({ - name: 'human-feedback-edit', - text: 'Copy and edit the output:' - }) - editedOutput = editedOutput.replace(/```$/g, '') - editedOutput = editedOutput.replace(/^```/g, '') - metadata.feedback.editedOutput = editedOutput - break - } - - case UserActions.Decline: - metadata.feedback.accepted = false - break - - case UserActions.Exit: - throw new Error('Exiting...') - - default: - throw new Error(`Unexpected feedback: ${feedback}`) - } + public async selectOne(response: any[]): Promise { + const { body: selectedOutput } = + await this.twilioClient.sendAndWaitForReply({ + name: 'human-feedback-select', + text: + 'Pick one output:' + + response.map((r, idx) => `\n${idx} - ${r}`).join('') + + '\n\nReply with the number of your choice.', + validate: (message) => { + const choice = parseInt(message.body) + return !isNaN(choice) && choice >= 0 && choice < response.length + } + }) + return response[parseInt(selectedOutput)] } - public async selectOne( - response: any[], - metadata: TaskResponseMetadata - ): Promise { - const stringified = JSON.stringify(response, null, 2) - const msg = [ - 'The following output was generated:', - '', - stringified, - '', - 'What would you like to do?' - ].join('\n') - - const choices: UserActions[] = [UserActions.Select] - - if (this._options.editing) { - choices.push(UserActions.Edit) - } - - if (this._options.bail) { - choices.push(UserActions.Exit) - } - - const feedback = - choices.length === 1 - ? UserActions.Select - : await this.askUser(msg, choices) - - metadata.feedback = {} - - switch (feedback) { - case UserActions.Edit: { - let { body: editedOutput } = - await this.twilioClient.sendAndWaitForReply({ - text: 'Copy and edit the output:', - name: 'human-feedback-edit' + public async selectN(response: any[]): Promise { + const { body: selectedOutput } = + await this.twilioClient.sendAndWaitForReply({ + name: 'human-feedback-select', + text: + 'Select outputs:' + + response.map((r, idx) => `\n${idx} - ${r}`).join('') + + '\n\nReply with a comma-separated list of the output numbers of your choice.', + validate: (message) => { + const choices = message.body.split(',') + return choices.every((choice) => { + const choiceInt = parseInt(choice) + return ( + !isNaN(choiceInt) && choiceInt >= 0 && choiceInt < response.length + ) }) - editedOutput = editedOutput.replace(/```$/g, '') - editedOutput = editedOutput.replace(/^```/g, '') - metadata.feedback.editedOutput = editedOutput - break - } - - case UserActions.Select: { - const { body: selectedOutput } = - await this.twilioClient.sendAndWaitForReply({ - name: 'human-feedback-select', - text: - 'Pick one output:' + - response.map((r, idx) => `\n${idx} - ${r}`).join('') + - '\n\nReply with the number of your choice.', - validate: (message) => { - const choice = parseInt(message.body) - return !isNaN(choice) && choice >= 0 && choice < response.length - } - }) - metadata.feedback.chosen = response[parseInt(selectedOutput)] - break - } - - case UserActions.Exit: - throw new Error('Exiting...') - - default: - throw new Error(`Unexpected feedback: ${feedback}`) - } - } - - public async selectN( - response: any[], - metadata: TaskResponseMetadata - ): Promise { - const stringified = JSON.stringify(response, null, 2) - const msg = [ - 'The following output was generated:', - '', - stringified, - '', - 'What would you like to do?' - ].join('\n') - const choices: UserActions[] = [UserActions.Select] - - if (this._options.editing) { - choices.push(UserActions.Edit) - } - - if (this._options.bail) { - choices.push(UserActions.Exit) - } - - const feedback = - choices.length === 1 - ? UserActions.Select - : await this.askUser(msg, choices) - - metadata.feedback = {} - - switch (feedback) { - case UserActions.Edit: { - let { body: editedOutput } = - await this.twilioClient.sendAndWaitForReply({ - text: 'Copy and edit the output:', - name: 'human-feedback-edit' - }) - editedOutput = editedOutput.replace(/```$/g, '') - editedOutput = editedOutput.replace(/^```/g, '') - metadata.feedback.editedOutput = editedOutput - break - } - - case UserActions.Select: { - const { body: selectedOutput } = - await this.twilioClient.sendAndWaitForReply({ - name: 'human-feedback-select', - text: - 'Select outputs:' + - response.map((r, idx) => `\n${idx} - ${r}`).join('') + - '\n\nReply with a comma-separated list of the output numbers of your choice.', - validate: (message) => { - const choices = message.body.split(',') - return choices.every((choice) => { - const choiceInt = parseInt(choice) - return ( - !isNaN(choiceInt) && - choiceInt >= 0 && - choiceInt < response.length - ) - }) - } - }) - const chosenOutputs = selectedOutput - .split(',') - .map((choice) => parseInt(choice)) - metadata.feedback.selected = response.filter((_, idx) => { - return chosenOutputs.includes(idx) - }) - break - } - - case UserActions.Exit: - throw new Error('Exiting...') - - default: - throw new Error(`Unexpected feedback: ${feedback}`) - } + } + }) + const chosenOutputs = selectedOutput + .split(',') + .map((choice) => parseInt(choice)) + return response.filter((_, idx) => { + return chosenOutputs.includes(idx) + }) } }