diff --git a/src/human-feedback/cli.ts b/src/human-feedback/cli.ts index 134766b..aa00ece 100644 --- a/src/human-feedback/cli.ts +++ b/src/human-feedback/cli.ts @@ -2,6 +2,9 @@ import checkbox from '@inquirer/checkbox' import editor from '@inquirer/editor' import input from '@inquirer/input' import select from '@inquirer/select' +import { setTimeout } from 'timers/promises' + +import { CancelablePromise } from '@/types' import { HumanFeedbackMechanism, @@ -21,27 +24,62 @@ export class HumanFeedbackMechanismCLI< message: string, choices: HumanFeedbackUserActions[] ): Promise { - return select({ - message, - choices: choices.map((choice) => ({ - name: HumanFeedbackUserActionMessages[choice], - value: choice - })) + return this._errorAfterTimeout( + select({ + message, + choices: choices.map((choice) => ({ + name: HumanFeedbackUserActionMessages[choice], + value: choice + })) + }) + ) + } + + protected _defaultAfterTimeout( + promise: CancelablePromise, + defaultValue: any + ) { + if (!isFinite(this._options.timeoutMs)) { + return promise + } + + const resolveDefault = setTimeout(this._options.timeoutMs).then(() => { + promise.cancel() + return defaultValue }) + return Promise.race([resolveDefault, promise]) + } + + protected async _errorAfterTimeout(promise: CancelablePromise) { + if (!isFinite(this._options.timeoutMs)) { + return promise + } + + const rejectError = setTimeout(this._options.timeoutMs).then(() => { + promise.cancel() + throw new Error('Timeout waiting for user input') + }) + return Promise.race([rejectError, promise]) } protected async _edit(output: string): Promise { - return editor({ - message: 'Edit the output:', - default: output - }) + return this._defaultAfterTimeout( + editor({ + message: 'Edit the output:', + default: output + }), + output + ) } protected async _annotate(): Promise { - return input({ - message: - 'Please leave an annotation (leave blank to skip; press enter to submit):' - }) + return this._defaultAfterTimeout( + input({ + message: + 'Please leave an annotation (leave blank to skip; press enter to submit):' + }), + '' + ) } protected async _selectOne( @@ -55,7 +93,9 @@ export class HumanFeedbackMechanismCLI< name: JSON.stringify(option), value: option })) - return select({ message: 'Pick one output:', choices }) + return this._errorAfterTimeout( + select({ message: 'Pick one output:', choices }) + ) } protected async _selectN( @@ -69,6 +109,8 @@ export class HumanFeedbackMechanismCLI< name: JSON.stringify(option), value: option })) - return checkbox({ message: 'Select outputs:', choices }) as any + return this._errorAfterTimeout( + checkbox({ message: 'Select outputs:', choices }) + ) } } diff --git a/src/human-feedback/feedback.ts b/src/human-feedback/feedback.ts index f3e6acd..af20800 100644 --- a/src/human-feedback/feedback.ts +++ b/src/human-feedback/feedback.ts @@ -72,6 +72,11 @@ export type HumanFeedbackOptions = { * Custom label to be displayed along with the output when requesting feedback. */ outputLabel?: string + + /** + * Timeout in milliseconds after which waiting for any user input is aborted (default: +Infinity, i.e. no timeout.) + */ + timeoutMs?: number } export interface BaseHumanFeedbackMetadata { @@ -284,6 +289,7 @@ export function withHumanFeedback< abort: false, editing: false, annotations: false, + timeoutMs: Number.POSITIVE_INFINITY, mechanism: HumanFeedbackMechanismCLI }, // Default options from the instance: diff --git a/src/human-feedback/slack.ts b/src/human-feedback/slack.ts index 81b7d75..10ee38a 100644 --- a/src/human-feedback/slack.ts +++ b/src/human-feedback/slack.ts @@ -31,7 +31,8 @@ export class HumanFeedbackMechanismSlack< protected async _annotate(): Promise { try { const annotation = await this._slackClient.sendAndWaitForReply({ - text: 'Please leave an annotation (optional):' + text: 'Please leave an annotation (optional):', + timeoutMs: this._options.timeoutMs }) return annotation.text } catch (e) { @@ -42,7 +43,8 @@ export class HumanFeedbackMechanismSlack< protected async _edit(): Promise { let { text: editedOutput } = await this._slackClient.sendAndWaitForReply({ - text: 'Copy and edit the output:' + text: 'Copy and edit the output:', + timeoutMs: this._options.timeoutMs }) editedOutput = editedOutput.replace(/```$/g, '') editedOutput = editedOutput.replace(/^```/g, '') @@ -63,6 +65,7 @@ export class HumanFeedbackMechanismSlack< message += 'Reply with the number of your choice.' const response = await this._slackClient.sendAndWaitForReply({ text: message, + timeoutMs: this._options.timeoutMs, validate: (slackMessage) => { const choice = parseInt(slackMessage.text) return !isNaN(choice) && choice >= 0 && choice < choices.length @@ -86,6 +89,7 @@ export class HumanFeedbackMechanismSlack< .map((r, idx) => `\n*${idx}* - ${JSON.stringify(r)}`) .join('') + '\n\nReply with the number of your choice.', + timeoutMs: this._options.timeoutMs, validate: (slackMessage) => { const choice = parseInt(slackMessage.text) return !isNaN(choice) && choice >= 0 && choice < response.length @@ -109,6 +113,7 @@ export class HumanFeedbackMechanismSlack< .map((r, idx) => `\n*${idx}* - ${JSON.stringify(r)}`) .join('') + '\n\nReply with a comma-separated list of the output numbers of your choice.', + timeoutMs: this._options.timeoutMs, validate: (slackMessage) => { const choices = slackMessage.text.split(',') return choices.every((choice) => { diff --git a/src/human-feedback/twilio.ts b/src/human-feedback/twilio.ts index 401ec8a..017952d 100644 --- a/src/human-feedback/twilio.ts +++ b/src/human-feedback/twilio.ts @@ -32,7 +32,8 @@ export class HumanFeedbackMechanismTwilio< try { const annotation = await this._twilioClient.sendAndWaitForReply({ name: 'human-feedback-annotation', - text: 'Please leave an annotation (optional):' + text: 'Please leave an annotation (optional):', + timeoutMs: this._options.timeoutMs }) return annotation.body } catch (e) { @@ -44,7 +45,8 @@ export class HumanFeedbackMechanismTwilio< protected async _edit(): Promise { let { body: editedOutput } = await this._twilioClient.sendAndWaitForReply({ text: 'Copy and edit the output:', - name: 'human-feedback-edit' + name: 'human-feedback-edit', + timeoutMs: this._options.timeoutMs }) editedOutput = editedOutput.replace(/```$/g, '') editedOutput = editedOutput.replace(/^```/g, '') @@ -66,6 +68,7 @@ export class HumanFeedbackMechanismTwilio< const response = await this._twilioClient.sendAndWaitForReply({ name: 'human-feedback-ask', text: message, + timeoutMs: this._options.timeoutMs, validate: (message) => { const choice = parseInt(message.body) return !isNaN(choice) && choice >= 0 && choice < choices.length @@ -88,6 +91,7 @@ export class HumanFeedbackMechanismTwilio< 'Pick one output:' + response.map((r, idx) => `\n${idx} - ${JSON.stringify(r)}`).join('') + '\n\nReply with the number of your choice.', + timeoutMs: this._options.timeoutMs, validate: (message) => { const choice = parseInt(message.body) return !isNaN(choice) && choice >= 0 && choice < response.length @@ -110,6 +114,7 @@ export class HumanFeedbackMechanismTwilio< 'Select outputs:' + response.map((r, idx) => `\n${idx} - ${JSON.stringify(r)}`).join('') + '\n\nReply with a comma-separated list of the output numbers of your choice.', + timeoutMs: this._options.timeoutMs, validate: (message) => { const choices = message.body.split(',') return choices.every((choice) => { diff --git a/src/services/twilio-conversation.ts b/src/services/twilio-conversation.ts index b03cf26..41f90b4 100644 --- a/src/services/twilio-conversation.ts +++ b/src/services/twilio-conversation.ts @@ -6,7 +6,7 @@ import { chunkString, sleep } from '@/utils' export const TWILIO_CONVERSATION_API_BASE_URL = 'https://conversations.twilio.com/v1' -export const DEFAULT_TWILIO_TIMEOUT_MS = 120_000 +export const DEFAULT_TWILIO_TIMEOUT_MS = 1_800_000 export const DEFAULT_TWILIO_INTERVAL_MS = 5_000 /** diff --git a/src/types.ts b/src/types.ts index 5fc4752..6474b6b 100644 --- a/src/types.ts +++ b/src/types.ts @@ -15,8 +15,7 @@ import type { BaseTask } from './task' export { anthropic, openai } -export type { Logger } -export type { JsonObject, JsonValue } +export type { JsonObject, JsonValue, Logger } export type KyInstance = typeof ky export type ParsedData = T extends ZodTypeAny @@ -141,4 +140,8 @@ export interface SerializedTask extends JsonObject { _taskName: string } +export declare class CancelablePromise extends Promise { + cancel: () => void +} + // export type ProgressFunction = (partialResponse: ChatMessage) => void