feat: add timeout control to human feedback

src/human-feedback/cli.ts

@@ -2,6 +2,9 @@ import checkbox from '@inquirer/checkbox'
 import editor from '@inquirer/editor'
 import input from '@inquirer/input'
 import select from '@inquirer/select'
+import { setTimeout } from 'timers/promises'
+
+import { CancelablePromise } from '@/types'
 
 import {
   HumanFeedbackMechanism,
@@ -21,27 +24,62 @@ export class HumanFeedbackMechanismCLI<
     message: string,
     choices: HumanFeedbackUserActions[]
   ): Promise<HumanFeedbackUserActions> {
-    return select({
-      message,
-      choices: choices.map((choice) => ({
-        name: HumanFeedbackUserActionMessages[choice],
-        value: choice
-      }))
+    return this._errorAfterTimeout(
+      select({
+        message,
+        choices: choices.map((choice) => ({
+          name: HumanFeedbackUserActionMessages[choice],
+          value: choice
+        }))
+      })
+    )
+  }
+
+  protected _defaultAfterTimeout(
+    promise: CancelablePromise<any>,
+    defaultValue: any
+  ) {
+    if (!isFinite(this._options.timeoutMs)) {
+      return promise
+    }
+
+    const resolveDefault = setTimeout(this._options.timeoutMs).then(() => {
+      promise.cancel()
+      return defaultValue
     })
+    return Promise.race([resolveDefault, promise])
+  }
+
+  protected async _errorAfterTimeout(promise: CancelablePromise<any>) {
+    if (!isFinite(this._options.timeoutMs)) {
+      return promise
+    }
+
+    const rejectError = setTimeout(this._options.timeoutMs).then(() => {
+      promise.cancel()
+      throw new Error('Timeout waiting for user input')
+    })
+    return Promise.race([rejectError, promise])
   }
 
   protected async _edit(output: string): Promise<string> {
-    return editor({
-      message: 'Edit the output:',
-      default: output
-    })
+    return this._defaultAfterTimeout(
+      editor({
+        message: 'Edit the output:',
+        default: output
+      }),
+      output
+    )
   }
 
   protected async _annotate(): Promise<string> {
-    return input({
-      message:
-        'Please leave an annotation (leave blank to skip; press enter to submit):'
-    })
+    return this._defaultAfterTimeout(
+      input({
+        message:
+          'Please leave an annotation (leave blank to skip; press enter to submit):'
+      }),
+      ''
+    )
   }
 
   protected async _selectOne(
@@ -55,7 +93,9 @@ export class HumanFeedbackMechanismCLI<
       name: JSON.stringify(option),
       value: option
     }))
-    return select({ message: 'Pick one output:', choices })
+    return this._errorAfterTimeout(
+      select({ message: 'Pick one output:', choices })
+    )
   }
 
   protected async _selectN(
@@ -69,6 +109,8 @@ export class HumanFeedbackMechanismCLI<
       name: JSON.stringify(option),
       value: option
     }))
-    return checkbox({ message: 'Select outputs:', choices }) as any
+    return this._errorAfterTimeout(
+      checkbox({ message: 'Select outputs:', choices })
+    )
   }
 }

src/human-feedback/feedback.ts

@@ -72,6 +72,11 @@ export type HumanFeedbackOptions<T extends HumanFeedbackType, TOutput> = {
    * Custom label to be displayed along with the output when requesting feedback.
    */
   outputLabel?: string
+
+  /**
+   * Timeout in milliseconds after which waiting for any user input is aborted (default: +Infinity, i.e. no timeout.)
+   */
+  timeoutMs?: number
 }
 
 export interface BaseHumanFeedbackMetadata {
@@ -284,6 +289,7 @@ export function withHumanFeedback<
       abort: false,
       editing: false,
       annotations: false,
+      timeoutMs: Number.POSITIVE_INFINITY,
       mechanism: HumanFeedbackMechanismCLI
     },
     // Default options from the instance:

src/human-feedback/slack.ts

@@ -31,7 +31,8 @@ export class HumanFeedbackMechanismSlack<
   protected async _annotate(): Promise<string> {
     try {
       const annotation = await this._slackClient.sendAndWaitForReply({
-        text: 'Please leave an annotation (optional):'
+        text: 'Please leave an annotation (optional):',
+        timeoutMs: this._options.timeoutMs
       })
       return annotation.text
     } catch (e) {
@@ -42,7 +43,8 @@ export class HumanFeedbackMechanismSlack<
 
   protected async _edit(): Promise<string> {
     let { text: editedOutput } = await this._slackClient.sendAndWaitForReply({
-      text: 'Copy and edit the output:'
+      text: 'Copy and edit the output:',
+      timeoutMs: this._options.timeoutMs
     })
     editedOutput = editedOutput.replace(/```$/g, '')
     editedOutput = editedOutput.replace(/^```/g, '')
@@ -63,6 +65,7 @@ export class HumanFeedbackMechanismSlack<
     message += 'Reply with the number of your choice.'
     const response = await this._slackClient.sendAndWaitForReply({
       text: message,
+      timeoutMs: this._options.timeoutMs,
       validate: (slackMessage) => {
         const choice = parseInt(slackMessage.text)
         return !isNaN(choice) && choice >= 0 && choice < choices.length
@@ -86,6 +89,7 @@ export class HumanFeedbackMechanismSlack<
             .map((r, idx) => `\n*${idx}* - ${JSON.stringify(r)}`)
             .join('') +
           '\n\nReply with the number of your choice.',
+        timeoutMs: this._options.timeoutMs,
         validate: (slackMessage) => {
           const choice = parseInt(slackMessage.text)
           return !isNaN(choice) && choice >= 0 && choice < response.length
@@ -109,6 +113,7 @@ export class HumanFeedbackMechanismSlack<
             .map((r, idx) => `\n*${idx}* - ${JSON.stringify(r)}`)
             .join('') +
           '\n\nReply with a comma-separated list of the output numbers of your choice.',
+        timeoutMs: this._options.timeoutMs,
         validate: (slackMessage) => {
           const choices = slackMessage.text.split(',')
           return choices.every((choice) => {

src/human-feedback/twilio.ts

@@ -32,7 +32,8 @@ export class HumanFeedbackMechanismTwilio<
     try {
       const annotation = await this._twilioClient.sendAndWaitForReply({
         name: 'human-feedback-annotation',
-        text: 'Please leave an annotation (optional):'
+        text: 'Please leave an annotation (optional):',
+        timeoutMs: this._options.timeoutMs
       })
       return annotation.body
     } catch (e) {
@@ -44,7 +45,8 @@ export class HumanFeedbackMechanismTwilio<
   protected async _edit(): Promise<string> {
     let { body: editedOutput } = await this._twilioClient.sendAndWaitForReply({
       text: 'Copy and edit the output:',
-      name: 'human-feedback-edit'
+      name: 'human-feedback-edit',
+      timeoutMs: this._options.timeoutMs
     })
     editedOutput = editedOutput.replace(/```$/g, '')
     editedOutput = editedOutput.replace(/^```/g, '')
@@ -66,6 +68,7 @@ export class HumanFeedbackMechanismTwilio<
     const response = await this._twilioClient.sendAndWaitForReply({
       name: 'human-feedback-ask',
       text: message,
+      timeoutMs: this._options.timeoutMs,
       validate: (message) => {
         const choice = parseInt(message.body)
         return !isNaN(choice) && choice >= 0 && choice < choices.length
@@ -88,6 +91,7 @@ export class HumanFeedbackMechanismTwilio<
           'Pick one output:' +
           response.map((r, idx) => `\n${idx} - ${JSON.stringify(r)}`).join('') +
           '\n\nReply with the number of your choice.',
+        timeoutMs: this._options.timeoutMs,
         validate: (message) => {
           const choice = parseInt(message.body)
           return !isNaN(choice) && choice >= 0 && choice < response.length
@@ -110,6 +114,7 @@ export class HumanFeedbackMechanismTwilio<
           'Select outputs:' +
           response.map((r, idx) => `\n${idx} - ${JSON.stringify(r)}`).join('') +
           '\n\nReply with a comma-separated list of the output numbers of your choice.',
+        timeoutMs: this._options.timeoutMs,
         validate: (message) => {
           const choices = message.body.split(',')
           return choices.every((choice) => {

src/services/twilio-conversation.ts

@@ -6,7 +6,7 @@ import { chunkString, sleep } from '@/utils'
 export const TWILIO_CONVERSATION_API_BASE_URL =
   'https://conversations.twilio.com/v1'
 
-export const DEFAULT_TWILIO_TIMEOUT_MS = 120_000
+export const DEFAULT_TWILIO_TIMEOUT_MS = 1_800_000
 export const DEFAULT_TWILIO_INTERVAL_MS = 5_000
 
 /**

src/types.ts

@@ -15,8 +15,7 @@ import type { BaseTask } from './task'
 
 export { anthropic, openai }
 
-export type { Logger }
-export type { JsonObject, JsonValue }
+export type { JsonObject, JsonValue, Logger }
 export type KyInstance = typeof ky
 
 export type ParsedData<T extends ZodTypeAny> = T extends ZodTypeAny
@@ -141,4 +140,8 @@ export interface SerializedTask extends JsonObject {
   _taskName: string
 }
 
+export declare class CancelablePromise<T> extends Promise<T> {
+  cancel: () => void
+}
+
 // export type ProgressFunction = (partialResponse: ChatMessage) => void