kopia lustrzana https://github.com/transitive-bullshit/chatgpt-api
feat: add human feedback functionality
Philipp Burckhardt
GitHub
commit
0cc76b0f3d
|
|
@ -0,0 +1,39 @@
|
|||
import 'dotenv/config'
|
||||
import { OpenAIClient } from 'openai-fetch'
|
||||
import { z } from 'zod'
|
||||
|
||||
import { HumanFeedbackMechanismCLI } from '@/human-feedback'
|
||||
import { Agentic, withHumanFeedback } from '@/index'
|
||||
|
||||
async function main() {
|
||||
const openai = new OpenAIClient({ apiKey: process.env.OPENAI_API_KEY! })
|
||||
const ai = new Agentic({ openai })
|
||||
|
||||
const topicJokes = ai
|
||||
.gpt3(`Tell me {{num}} jokes about {{topic}}`)
|
||||
.input(
|
||||
z.object({
|
||||
topic: z.string(),
|
||||
num: z.number().int().default(5).optional()
|
||||
})
|
||||
)
|
||||
.output(z.array(z.string()))
|
||||
.modelParams({ temperature: 0.9 })
|
||||
|
||||
const topicJokesFeedback = withHumanFeedback(topicJokes, {
|
||||
type: 'selectN',
|
||||
annotations: false,
|
||||
abort: false,
|
||||
editing: true,
|
||||
mechanism: HumanFeedbackMechanismCLI
|
||||
})
|
||||
|
||||
const out = await topicJokesFeedback.callWithMetadata({
|
||||
topic: 'politicians',
|
||||
num: 5
|
||||
})
|
||||
const feedback = out.metadata.feedback
|
||||
console.log(JSON.stringify(feedback, null, 2))
|
||||
}
|
||||
|
||||
main()
|
||||
|
|
@ -1,32 +0,0 @@
|
|||
import { OpenAIClient } from '@agentic/openai-fetch'
|
||||
import 'dotenv/config'
|
||||
import { z } from 'zod'
|
||||
|
||||
import { Agentic, HumanFeedbackSelect } from '@/index'
|
||||
|
||||
async function main() {
|
||||
const openai = new OpenAIClient({ apiKey: process.env.OPENAI_API_KEY! })
|
||||
const ai = new Agentic({ openai })
|
||||
|
||||
const jokes = ai
|
||||
.gpt3(`Tell me {{num}} jokes about {{topic}}`)
|
||||
.input(
|
||||
z.object({
|
||||
topic: z.string(),
|
||||
num: z.number().int().default(5).optional()
|
||||
})
|
||||
)
|
||||
.output(z.array(z.string()))
|
||||
.modelParams({ temperature: 0.9 })
|
||||
|
||||
const feedback = new HumanFeedbackSelect(z.string())
|
||||
let out = await jokes.call({ topic: 'statisticians' })
|
||||
let hf = await feedback.call(out)
|
||||
while (!hf.accepted) {
|
||||
out = await jokes.call({ topic: 'statisticians' })
|
||||
hf = await feedback.call(out)
|
||||
}
|
||||
console.log(hf.results)
|
||||
}
|
||||
|
||||
main()
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
import 'dotenv/config'
|
||||
import { OpenAIClient } from 'openai-fetch'
|
||||
import { z } from 'zod'
|
||||
|
||||
import { HumanFeedbackMechanismSlack } from '@/human-feedback'
|
||||
import { Agentic, withHumanFeedback } from '@/index'
|
||||
|
||||
async function main() {
|
||||
const openai = new OpenAIClient({ apiKey: process.env.OPENAI_API_KEY! })
|
||||
const ai = new Agentic({ openai })
|
||||
|
||||
const topicJokes = ai
|
||||
.gpt3(`Tell me {{num}} jokes about {{topic}}`)
|
||||
.input(
|
||||
z.object({
|
||||
topic: z.string(),
|
||||
num: z.number().int().default(5).optional()
|
||||
})
|
||||
)
|
||||
.output(z.array(z.string()))
|
||||
.modelParams({ temperature: 0.9 })
|
||||
|
||||
const topicJokesFeedback = withHumanFeedback(topicJokes, {
|
||||
type: 'selectN',
|
||||
annotations: false,
|
||||
abort: false,
|
||||
editing: true,
|
||||
mechanism: HumanFeedbackMechanismSlack
|
||||
})
|
||||
|
||||
const out = await topicJokesFeedback.callWithMetadata({
|
||||
topic: 'politicians',
|
||||
num: 5
|
||||
})
|
||||
const feedback = out.metadata.feedback
|
||||
console.log(JSON.stringify(feedback, null, 2))
|
||||
}
|
||||
|
||||
main()
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
import 'dotenv/config'
|
||||
import { OpenAIClient } from 'openai-fetch'
|
||||
import { z } from 'zod'
|
||||
|
||||
import { HumanFeedbackMechanismTwilio } from '@/human-feedback'
|
||||
import { Agentic, withHumanFeedback } from '@/index'
|
||||
|
||||
async function main() {
|
||||
const openai = new OpenAIClient({ apiKey: process.env.OPENAI_API_KEY! })
|
||||
const ai = new Agentic({ openai })
|
||||
|
||||
const topicJokes = ai
|
||||
.gpt3(`Tell me {{num}} jokes about {{topic}}`)
|
||||
.input(
|
||||
z.object({
|
||||
topic: z.string(),
|
||||
num: z.number().int().default(5).optional()
|
||||
})
|
||||
)
|
||||
.output(z.array(z.string()))
|
||||
.modelParams({ temperature: 0.9 })
|
||||
|
||||
const topicJokesFeedback = withHumanFeedback(topicJokes, {
|
||||
type: 'selectN',
|
||||
annotations: false,
|
||||
abort: false,
|
||||
editing: true,
|
||||
mechanism: HumanFeedbackMechanismTwilio
|
||||
})
|
||||
|
||||
const out = await topicJokesFeedback.callWithMetadata({
|
||||
topic: 'politicians',
|
||||
num: 5
|
||||
})
|
||||
const feedback = out.metadata.feedback
|
||||
console.log(JSON.stringify(feedback, null, 2))
|
||||
}
|
||||
|
||||
main()
|
||||
|
|
@ -1,33 +0,0 @@
|
|||
import { OpenAIClient } from '@agentic/openai-fetch'
|
||||
import 'dotenv/config'
|
||||
import { z } from 'zod'
|
||||
|
||||
import { Agentic, HumanFeedbackSingle } from '@/index'
|
||||
|
||||
async function main() {
|
||||
const openai = new OpenAIClient({ apiKey: process.env.OPENAI_API_KEY! })
|
||||
const ai = new Agentic({ openai })
|
||||
|
||||
const topicFacts = ai
|
||||
.gpt3(`Give me {{numFacts}} random facts about {{topic}}`)
|
||||
.input(
|
||||
z.object({
|
||||
topic: z.string(),
|
||||
numFacts: z.number().int().default(5).optional()
|
||||
})
|
||||
)
|
||||
.output(z.object({ facts: z.array(z.string()) }))
|
||||
.modelParams({ temperature: 0.9 })
|
||||
|
||||
const feedback = new HumanFeedbackSingle(topicFacts.outputSchema)
|
||||
|
||||
let out = await topicFacts.call({ topic: 'cats' })
|
||||
let hf = await feedback.call(out)
|
||||
while (!hf.accepted) {
|
||||
out = await topicFacts.call({ topic: 'cats' })
|
||||
hf = await feedback.call(out)
|
||||
}
|
||||
console.log(hf.result)
|
||||
}
|
||||
|
||||
main()
|
||||
|
|
@ -1,232 +0,0 @@
|
|||
import checkbox from '@inquirer/checkbox'
|
||||
import editor from '@inquirer/editor'
|
||||
import select from '@inquirer/select'
|
||||
import { ZodTypeAny, z } from 'zod'
|
||||
|
||||
import * as types from '@/types'
|
||||
import { BaseTask } from '@/task'
|
||||
|
||||
/**
|
||||
* Actions the user can take in the feedback selection prompt.
|
||||
*/
|
||||
export const UserActions = {
|
||||
Accept: 'accept',
|
||||
Edit: 'edit',
|
||||
Decline: 'decline',
|
||||
Select: 'select',
|
||||
Exit: 'exit'
|
||||
} as const
|
||||
|
||||
export type UserActions = (typeof UserActions)[keyof typeof UserActions]
|
||||
|
||||
/**
|
||||
* Messages to display to the user for each action.
|
||||
*/
|
||||
const UserActionMessages: Record<UserActions, string> = {
|
||||
[UserActions.Accept]: 'Accept inputs',
|
||||
[UserActions.Edit]: 'Edit (open in editor)',
|
||||
[UserActions.Decline]: 'Decline',
|
||||
[UserActions.Select]: 'Select inputs to keep',
|
||||
[UserActions.Exit]: 'Exit'
|
||||
}
|
||||
|
||||
/**
|
||||
* Prompt the user to select one of a list of options.
|
||||
*/
|
||||
async function askUser(
|
||||
message: string,
|
||||
choices: UserActions[]
|
||||
): Promise<UserActions> {
|
||||
return select({
|
||||
message,
|
||||
choices: choices.map((choice) => ({
|
||||
name: UserActionMessages[choice],
|
||||
value: choice
|
||||
}))
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Output schema when prompting the user to accept, edit, or decline a single input.
|
||||
*/
|
||||
export const FeedbackSingleOutputSchema = <T extends ZodTypeAny>(result: T) =>
|
||||
z.object({
|
||||
result: result,
|
||||
accepted: z.boolean()
|
||||
})
|
||||
|
||||
/**
|
||||
* Prompt the user to accept, edit, or decline a single input.
|
||||
*/
|
||||
export class HumanFeedbackSingle<T extends ZodTypeAny> extends BaseTask<
|
||||
ZodTypeAny,
|
||||
ZodTypeAny
|
||||
> {
|
||||
protected choiceSchema: T
|
||||
|
||||
constructor(choiceSchema: T) {
|
||||
super()
|
||||
this.choiceSchema = choiceSchema
|
||||
}
|
||||
|
||||
public get inputSchema() {
|
||||
return this.choiceSchema
|
||||
}
|
||||
|
||||
public get outputSchema() {
|
||||
return FeedbackSingleOutputSchema(this.choiceSchema)
|
||||
}
|
||||
|
||||
protected actionHandlers = {
|
||||
[UserActions.Accept]: (
|
||||
input: types.ParsedData<typeof this.inputSchema>
|
||||
) => ({ result: input, accepted: true }),
|
||||
[UserActions.Edit]: async (
|
||||
input: types.ParsedData<typeof this.inputSchema>
|
||||
) => {
|
||||
const editedInput = await editor({
|
||||
message: 'Edit the input:',
|
||||
default: JSON.stringify(input)
|
||||
})
|
||||
return this.outputSchema.parse({
|
||||
result: JSON.parse(editedInput),
|
||||
accepted: true
|
||||
})
|
||||
},
|
||||
[UserActions.Decline]: () => ({ result: null, accepted: false }),
|
||||
[UserActions.Exit]: () => {
|
||||
throw new Error('Exiting...')
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Prompts the user to give feedback for the given input and handles their response.
|
||||
*/
|
||||
public async call(
|
||||
input: types.ParsedData<typeof this.inputSchema>
|
||||
): Promise<types.ParsedData<typeof this.outputSchema>> {
|
||||
try {
|
||||
input = this.inputSchema.parse(input)
|
||||
const msg = [
|
||||
'The following input was generated:',
|
||||
JSON.stringify(input, null, 2),
|
||||
'What would you like to do?'
|
||||
].join('\n')
|
||||
const feedback = await askUser(msg, [
|
||||
UserActions.Accept,
|
||||
UserActions.Edit,
|
||||
UserActions.Decline,
|
||||
UserActions.Exit
|
||||
])
|
||||
const handler = this.actionHandlers[feedback]
|
||||
if (!handler) {
|
||||
throw new Error(`Unexpected feedback: ${feedback}`)
|
||||
}
|
||||
return handler(input)
|
||||
} catch (err) {
|
||||
console.error('Error parsing input:', err)
|
||||
throw err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Output schema when prompting the user to accept, select from, edit, or decline a list of inputs.
|
||||
*/
|
||||
export const FeedbackSelectOutputSchema = <T extends ZodTypeAny>(result: T) =>
|
||||
z.object({
|
||||
results: z.array(result),
|
||||
accepted: z.boolean()
|
||||
})
|
||||
|
||||
/**
|
||||
* Prompt the user to accept, select from, edit, or decline a list of inputs.
|
||||
*/
|
||||
export class HumanFeedbackSelect<T extends ZodTypeAny> extends BaseTask<
|
||||
ZodTypeAny,
|
||||
ZodTypeAny
|
||||
> {
|
||||
protected choiceSchema: T
|
||||
|
||||
constructor(choiceSchema: T) {
|
||||
super()
|
||||
this.choiceSchema = choiceSchema
|
||||
}
|
||||
|
||||
public get inputSchema() {
|
||||
return z.array(this.choiceSchema)
|
||||
}
|
||||
|
||||
public get outputSchema() {
|
||||
return FeedbackSelectOutputSchema(this.choiceSchema)
|
||||
}
|
||||
|
||||
protected actionHandlers = {
|
||||
[UserActions.Accept]: (
|
||||
input: types.ParsedData<typeof this.inputSchema>
|
||||
) => ({ results: input, accepted: true }),
|
||||
[UserActions.Edit]: async (
|
||||
input: types.ParsedData<typeof this.inputSchema>
|
||||
) => {
|
||||
const editedInput = await editor({
|
||||
message: 'Edit the input:',
|
||||
default: JSON.stringify(input, null, 2)
|
||||
})
|
||||
return this.outputSchema.parse({
|
||||
results: JSON.parse(editedInput),
|
||||
accepted: true
|
||||
})
|
||||
},
|
||||
[UserActions.Select]: async (
|
||||
input: types.ParsedData<typeof this.inputSchema>
|
||||
) => {
|
||||
const choices = input.map((completion) => ({
|
||||
name: completion,
|
||||
value: completion
|
||||
}))
|
||||
const chosen = await checkbox({
|
||||
message: 'Pick items to keep:',
|
||||
choices,
|
||||
pageSize: choices.length
|
||||
})
|
||||
return { results: chosen.length === 0 ? [] : chosen, accepted: true }
|
||||
},
|
||||
[UserActions.Decline]: () => ({ results: [], accepted: false }),
|
||||
[UserActions.Exit]: () => {
|
||||
throw new Error('Exiting...')
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Prompts the user to give feedback for the given list of inputs and handles their response.
|
||||
*/
|
||||
public async call(
|
||||
input: types.ParsedData<typeof this.inputSchema>
|
||||
): Promise<types.ParsedData<typeof this.outputSchema>> {
|
||||
try {
|
||||
input = this.inputSchema.parse(input)
|
||||
const message = [
|
||||
'The following inputs were generated:',
|
||||
...input.map(
|
||||
(choice, index) => `${index + 1}. ${JSON.stringify(choice, null, 2)}`
|
||||
),
|
||||
'What would you like to do?'
|
||||
].join('\n')
|
||||
const feedback = await askUser(message, [
|
||||
UserActions.Accept,
|
||||
UserActions.Select,
|
||||
UserActions.Edit,
|
||||
UserActions.Decline,
|
||||
UserActions.Exit
|
||||
])
|
||||
const handler = this.actionHandlers[feedback]
|
||||
if (!handler) {
|
||||
throw new Error(`Unexpected feedback: ${feedback}`)
|
||||
}
|
||||
return handler(input)
|
||||
} catch (err) {
|
||||
console.error('Error parsing input:', err)
|
||||
throw err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -2,10 +2,8 @@ import defaultKy from 'ky'
|
|||
|
||||
import * as types from './types'
|
||||
import { DEFAULT_OPENAI_MODEL } from './constants'
|
||||
import {
|
||||
HumanFeedbackMechanism,
|
||||
HumanFeedbackMechanismCLI
|
||||
} from './human-feedback'
|
||||
import { HumanFeedbackOptions, HumanFeedbackType } from './human-feedback'
|
||||
import { HumanFeedbackMechanismCLI } from './human-feedback/cli'
|
||||
import { OpenAIChatCompletion } from './llms/openai'
|
||||
import { defaultLogger } from './logger'
|
||||
import { defaultIDGeneratorFn } from './utils'
|
||||
|
|
@ -22,8 +20,7 @@ export class Agentic {
|
|||
types.BaseLLMOptions,
|
||||
'provider' | 'model' | 'modelParams' | 'timeoutMs' | 'retryConfig'
|
||||
>
|
||||
protected _defaultHumanFeedbackMechamism?: HumanFeedbackMechanism
|
||||
|
||||
protected _humanFeedbackDefaults: HumanFeedbackOptions<HumanFeedbackType, any>
|
||||
protected _idGeneratorFn: types.IDGeneratorFunction
|
||||
protected _id: string
|
||||
|
||||
|
|
@ -34,7 +31,7 @@ export class Agentic {
|
|||
types.BaseLLMOptions,
|
||||
'provider' | 'model' | 'modelParams' | 'timeoutMs' | 'retryConfig'
|
||||
>
|
||||
defaultHumanFeedbackMechanism?: HumanFeedbackMechanism
|
||||
humanFeedbackDefaults?: HumanFeedbackOptions<HumanFeedbackType, any>
|
||||
idGeneratorFn?: types.IDGeneratorFunction
|
||||
logger?: types.Logger
|
||||
ky?: types.KyInstance
|
||||
|
|
@ -68,9 +65,14 @@ export class Agentic {
|
|||
// TODO
|
||||
// this._anthropicModelDefaults = {}
|
||||
|
||||
this._defaultHumanFeedbackMechamism =
|
||||
opts.defaultHumanFeedbackMechanism ??
|
||||
new HumanFeedbackMechanismCLI({ agentic: this })
|
||||
this._humanFeedbackDefaults = {
|
||||
type: 'confirm',
|
||||
abort: false,
|
||||
editing: false,
|
||||
annotations: false,
|
||||
mechanism: HumanFeedbackMechanismCLI,
|
||||
...opts.humanFeedbackDefaults
|
||||
}
|
||||
|
||||
this._idGeneratorFn = opts.idGeneratorFn ?? defaultIDGeneratorFn
|
||||
this._id = this._idGeneratorFn()
|
||||
|
|
@ -92,8 +94,8 @@ export class Agentic {
|
|||
return this._logger
|
||||
}
|
||||
|
||||
public get defaultHumanFeedbackMechamism() {
|
||||
return this._defaultHumanFeedbackMechamism
|
||||
public get humanFeedbackDefaults() {
|
||||
return this._humanFeedbackDefaults
|
||||
}
|
||||
|
||||
public get idGeneratorFn(): types.IDGeneratorFunction {
|
||||
|
|
|
|||
|
|
@ -1,55 +0,0 @@
|
|||
import * as types from './types'
|
||||
import { Agentic } from './agentic'
|
||||
import { BaseTask } from './task'
|
||||
|
||||
export type HumanFeedbackType = 'confirm' | 'selectOne' | 'selectN'
|
||||
|
||||
export type HumanFeedbackOptions = {
|
||||
type: HumanFeedbackType
|
||||
|
||||
/**
|
||||
* Whether to allow exiting
|
||||
*/
|
||||
bail?: boolean
|
||||
|
||||
editing?: boolean
|
||||
|
||||
annotations?: boolean
|
||||
|
||||
feedbackMechanism?: HumanFeedbackMechanism
|
||||
}
|
||||
|
||||
export abstract class HumanFeedbackMechanism {
|
||||
protected _agentic: Agentic
|
||||
|
||||
constructor({ agentic }: { agentic: Agentic }) {
|
||||
this._agentic = agentic
|
||||
}
|
||||
// TODO
|
||||
}
|
||||
|
||||
export class HumanFeedbackMechanismCLI extends HumanFeedbackMechanism {
|
||||
// TODO
|
||||
constructor(opts: { agentic: Agentic }) {
|
||||
super(opts)
|
||||
}
|
||||
}
|
||||
|
||||
export function withHumanFeedback<
|
||||
TInput extends void | types.JsonObject = void,
|
||||
TOutput extends types.JsonValue = string
|
||||
>(
|
||||
task: BaseTask<TInput, TOutput>,
|
||||
options: HumanFeedbackOptions = {
|
||||
type: 'confirm',
|
||||
bail: false,
|
||||
editing: false,
|
||||
annotations: false
|
||||
}
|
||||
) {
|
||||
const { feedbackMechanism = task.agentic.defaultHumanFeedbackMechamism } =
|
||||
options
|
||||
|
||||
// TODO
|
||||
return task
|
||||
}
|
||||
|
|
@ -0,0 +1,74 @@
|
|||
import checkbox from '@inquirer/checkbox'
|
||||
import editor from '@inquirer/editor'
|
||||
import input from '@inquirer/input'
|
||||
import select from '@inquirer/select'
|
||||
|
||||
import {
|
||||
HumanFeedbackMechanism,
|
||||
HumanFeedbackType,
|
||||
HumanFeedbackUserActionMessages,
|
||||
HumanFeedbackUserActions
|
||||
} from './feedback'
|
||||
|
||||
export class HumanFeedbackMechanismCLI<
|
||||
T extends HumanFeedbackType,
|
||||
TOutput = any
|
||||
> extends HumanFeedbackMechanism<T, TOutput> {
|
||||
/**
|
||||
* Prompt the user to select one of a list of options.
|
||||
*/
|
||||
protected async _askUser(
|
||||
message: string,
|
||||
choices: HumanFeedbackUserActions[]
|
||||
): Promise<HumanFeedbackUserActions> {
|
||||
return select({
|
||||
message,
|
||||
choices: choices.map((choice) => ({
|
||||
name: HumanFeedbackUserActionMessages[choice],
|
||||
value: choice
|
||||
}))
|
||||
})
|
||||
}
|
||||
|
||||
protected async _edit(output: string): Promise<string> {
|
||||
return editor({
|
||||
message: 'Edit the output:',
|
||||
default: output
|
||||
})
|
||||
}
|
||||
|
||||
protected async _annotate(): Promise<string> {
|
||||
return input({
|
||||
message:
|
||||
'Please leave an annotation (leave blank to skip; press enter to submit):'
|
||||
})
|
||||
}
|
||||
|
||||
protected async _selectOne(
|
||||
response: TOutput
|
||||
): Promise<TOutput extends (infer U)[] ? U : never> {
|
||||
if (!Array.isArray(response)) {
|
||||
throw new Error('selectOne called on non-array response')
|
||||
}
|
||||
|
||||
const choices = response.map((option) => ({
|
||||
name: String(option),
|
||||
value: option
|
||||
}))
|
||||
return select({ message: 'Pick one output:', choices })
|
||||
}
|
||||
|
||||
protected async _selectN(
|
||||
response: TOutput
|
||||
): Promise<TOutput extends any[] ? TOutput : never> {
|
||||
if (!Array.isArray(response)) {
|
||||
throw new Error('selectN called on non-array response')
|
||||
}
|
||||
|
||||
const choices = response.map((option) => ({
|
||||
name: String(option),
|
||||
value: option
|
||||
}))
|
||||
return checkbox({ message: 'Select outputs:', choices }) as any
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,310 @@
|
|||
import * as types from '@/types'
|
||||
import { Agentic } from '@/agentic'
|
||||
import { BaseTask } from '@/task'
|
||||
|
||||
import { HumanFeedbackMechanismCLI } from './cli'
|
||||
|
||||
/**
|
||||
* Actions the user can take in the feedback selection prompt.
|
||||
*/
|
||||
export const HumanFeedbackUserActions = {
|
||||
Accept: 'accept',
|
||||
Edit: 'edit',
|
||||
Decline: 'decline',
|
||||
Select: 'select',
|
||||
Abort: 'abort'
|
||||
} as const
|
||||
|
||||
export type HumanFeedbackUserActions =
|
||||
(typeof HumanFeedbackUserActions)[keyof typeof HumanFeedbackUserActions]
|
||||
|
||||
export const HumanFeedbackUserActionMessages: Record<
|
||||
HumanFeedbackUserActions,
|
||||
string
|
||||
> = {
|
||||
[HumanFeedbackUserActions.Accept]: 'Accept the output',
|
||||
[HumanFeedbackUserActions.Edit]: 'Edit the output',
|
||||
[HumanFeedbackUserActions.Decline]: 'Decline the output',
|
||||
[HumanFeedbackUserActions.Select]: 'Select outputs to keep',
|
||||
[HumanFeedbackUserActions.Abort]: 'Abort'
|
||||
}
|
||||
|
||||
/**
|
||||
* Available types of human feedback.
|
||||
*/
|
||||
export type HumanFeedbackType = 'confirm' | 'selectOne' | 'selectN'
|
||||
|
||||
type HumanFeedbackMechanismConstructor<
|
||||
T extends HumanFeedbackType,
|
||||
TOutput = any
|
||||
> = new (...args: any[]) => HumanFeedbackMechanism<T, TOutput>
|
||||
|
||||
/**
|
||||
* Options for human feedback.
|
||||
*/
|
||||
export type HumanFeedbackOptions<T extends HumanFeedbackType, TOutput> = {
|
||||
/**
|
||||
* What type of feedback to request.
|
||||
*/
|
||||
type?: T
|
||||
|
||||
/**
|
||||
* Whether the user can abort the process.
|
||||
*/
|
||||
abort?: boolean
|
||||
|
||||
/**
|
||||
* Whether the user can edit the output.
|
||||
*/
|
||||
editing?: boolean
|
||||
|
||||
/**
|
||||
* Whether the user can add free-form text annotations.
|
||||
*/
|
||||
annotations?: boolean
|
||||
|
||||
/**
|
||||
* The human feedback mechanism to use for this task.
|
||||
*/
|
||||
mechanism?: HumanFeedbackMechanismConstructor<T, TOutput>
|
||||
}
|
||||
|
||||
export interface BaseHumanFeedbackMetadata {
|
||||
/**
|
||||
* Edited output by the user (if applicable).
|
||||
*/
|
||||
editedOutput?: any
|
||||
|
||||
/**
|
||||
* Annotation left by the user (if applicable).
|
||||
*/
|
||||
annotation?: string
|
||||
}
|
||||
|
||||
export interface HumanFeedbackConfirmMetadata
|
||||
extends BaseHumanFeedbackMetadata {
|
||||
/**
|
||||
* The type of feedback requested.
|
||||
*/
|
||||
type: 'confirm'
|
||||
|
||||
/**
|
||||
* Whether the user accepted the output.
|
||||
*/
|
||||
accepted: boolean
|
||||
}
|
||||
|
||||
export interface HumanFeedbackSelectOneMetadata
|
||||
extends BaseHumanFeedbackMetadata {
|
||||
/**
|
||||
* The type of feedback requested.
|
||||
*/
|
||||
type: 'selectOne'
|
||||
|
||||
/**
|
||||
* The selected output.
|
||||
*/
|
||||
chosen: any
|
||||
}
|
||||
|
||||
export interface HumanFeedbackSelectNMetadata
|
||||
extends BaseHumanFeedbackMetadata {
|
||||
/**
|
||||
* The type of feedback requested.
|
||||
*/
|
||||
type: 'selectN'
|
||||
|
||||
/**
|
||||
* The selected outputs.
|
||||
*/
|
||||
selected: any[]
|
||||
}
|
||||
|
||||
export type FeedbackTypeToMetadata<T extends HumanFeedbackType> =
|
||||
T extends 'confirm'
|
||||
? HumanFeedbackConfirmMetadata
|
||||
: T extends 'selectOne'
|
||||
? HumanFeedbackSelectOneMetadata
|
||||
: HumanFeedbackSelectNMetadata
|
||||
|
||||
export abstract class HumanFeedbackMechanism<
|
||||
T extends HumanFeedbackType,
|
||||
TOutput
|
||||
> {
|
||||
protected _agentic: Agentic
|
||||
|
||||
protected _task: BaseTask
|
||||
|
||||
protected _options: Required<HumanFeedbackOptions<T, TOutput>>
|
||||
|
||||
constructor({
|
||||
task,
|
||||
options
|
||||
}: {
|
||||
task: BaseTask
|
||||
options: Required<HumanFeedbackOptions<T, TOutput>>
|
||||
}) {
|
||||
this._agentic = task.agentic
|
||||
this._task = task
|
||||
this._options = options
|
||||
}
|
||||
|
||||
protected abstract _selectOne(
|
||||
output: TOutput
|
||||
): Promise<TOutput extends any[] ? TOutput[0] : never>
|
||||
|
||||
protected abstract _selectN(
|
||||
response: TOutput
|
||||
): Promise<TOutput extends any[] ? TOutput : never>
|
||||
|
||||
protected abstract _annotate(): Promise<string>
|
||||
|
||||
protected abstract _edit(output: string): Promise<string>
|
||||
|
||||
protected abstract _askUser(
|
||||
message: string,
|
||||
choices: HumanFeedbackUserActions[]
|
||||
): Promise<HumanFeedbackUserActions>
|
||||
|
||||
protected _parseEditedOutput(editedOutput: string): any {
|
||||
const parsedOutput = JSON.parse(editedOutput)
|
||||
return this._task.outputSchema.parse(parsedOutput)
|
||||
}
|
||||
|
||||
public async interact(output: TOutput): Promise<FeedbackTypeToMetadata<T>> {
|
||||
const stringified = JSON.stringify(output, null, 2)
|
||||
const msg = [
|
||||
'The following output was generated:',
|
||||
'```',
|
||||
stringified,
|
||||
'```',
|
||||
'What would you like to do?'
|
||||
].join('\n')
|
||||
|
||||
const choices: HumanFeedbackUserActions[] = []
|
||||
if (
|
||||
this._options.type === 'selectN' ||
|
||||
this._options.type === 'selectOne'
|
||||
) {
|
||||
choices.push(HumanFeedbackUserActions.Select)
|
||||
} else {
|
||||
// Case: confirm
|
||||
choices.push(HumanFeedbackUserActions.Accept)
|
||||
choices.push(HumanFeedbackUserActions.Decline)
|
||||
}
|
||||
|
||||
if (this._options.editing) {
|
||||
choices.push(HumanFeedbackUserActions.Edit)
|
||||
}
|
||||
|
||||
if (this._options.abort) {
|
||||
choices.push(HumanFeedbackUserActions.Abort)
|
||||
}
|
||||
|
||||
const choice =
|
||||
choices.length === 1
|
||||
? HumanFeedbackUserActions.Select
|
||||
: await this._askUser(msg, choices)
|
||||
|
||||
const feedback: Record<string, any> = {}
|
||||
|
||||
switch (choice) {
|
||||
case HumanFeedbackUserActions.Accept:
|
||||
feedback.accepted = true
|
||||
break
|
||||
|
||||
case HumanFeedbackUserActions.Edit: {
|
||||
const editedOutput = await this._edit(stringified)
|
||||
feedback.editedOutput = await this._parseEditedOutput(editedOutput)
|
||||
break
|
||||
}
|
||||
|
||||
case HumanFeedbackUserActions.Decline:
|
||||
feedback.accepted = false
|
||||
break
|
||||
|
||||
case HumanFeedbackUserActions.Select:
|
||||
if (this._options.type === 'selectN') {
|
||||
if (!Array.isArray(output)) {
|
||||
throw new Error('Expected output to be an array')
|
||||
}
|
||||
|
||||
feedback.selected = await this._selectN(output)
|
||||
} else if (this._options.type === 'selectOne') {
|
||||
if (!Array.isArray(output)) {
|
||||
throw new Error('Expected output to be an array')
|
||||
}
|
||||
|
||||
feedback.chosen = await this._selectOne(output)
|
||||
}
|
||||
|
||||
break
|
||||
|
||||
case HumanFeedbackUserActions.Abort:
|
||||
throw new Error('Aborting...')
|
||||
|
||||
default:
|
||||
throw new Error(`Unexpected choice: ${choice}`)
|
||||
}
|
||||
|
||||
if (this._options.annotations) {
|
||||
const annotation = await this._annotate()
|
||||
if (annotation) {
|
||||
feedback.annotation = annotation
|
||||
}
|
||||
}
|
||||
|
||||
return feedback as FeedbackTypeToMetadata<T>
|
||||
}
|
||||
}
|
||||
|
||||
export function withHumanFeedback<
|
||||
TInput extends void | types.JsonObject,
|
||||
TOutput extends types.JsonObject,
|
||||
V extends HumanFeedbackType
|
||||
>(
|
||||
task: BaseTask<TInput, TOutput>,
|
||||
options: HumanFeedbackOptions<V, TOutput> = {}
|
||||
) {
|
||||
task = task.clone()
|
||||
|
||||
// Use Object.assign to merge the options, instance defaults, and hard-coded defaults:
|
||||
const finalOptions: HumanFeedbackOptions<V, TOutput> = Object.assign(
|
||||
{
|
||||
type: 'confirm',
|
||||
abort: false,
|
||||
editing: false,
|
||||
annotations: false,
|
||||
mechanism: HumanFeedbackMechanismCLI
|
||||
},
|
||||
// Default options from the instance:
|
||||
task.agentic.humanFeedbackDefaults,
|
||||
// User-provided options (override instance defaults):
|
||||
options
|
||||
)
|
||||
|
||||
if (!finalOptions.mechanism) {
|
||||
throw new Error(
|
||||
'No feedback mechanism provided. Please provide a feedback mechanism to use.'
|
||||
)
|
||||
}
|
||||
|
||||
const feedbackMechanism = new finalOptions.mechanism({
|
||||
task: task,
|
||||
options: finalOptions
|
||||
})
|
||||
|
||||
const originalCall = task.callWithMetadata.bind(task)
|
||||
|
||||
task.callWithMetadata = async function (input?: TInput) {
|
||||
const response = await originalCall(input)
|
||||
|
||||
const feedback = await feedbackMechanism.interact(response.result)
|
||||
|
||||
response.metadata = { ...response.metadata, feedback }
|
||||
|
||||
return response
|
||||
}
|
||||
|
||||
return task
|
||||
}
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
export * from './cli'
|
||||
export * from './feedback'
|
||||
export * from './slack'
|
||||
export * from './twilio'
|
||||
|
|
@ -0,0 +1,125 @@
|
|||
import { SlackClient } from '@/services/slack'
|
||||
import { BaseTask } from '@/task'
|
||||
|
||||
import {
|
||||
HumanFeedbackMechanism,
|
||||
HumanFeedbackOptions,
|
||||
HumanFeedbackType,
|
||||
HumanFeedbackUserActionMessages,
|
||||
HumanFeedbackUserActions
|
||||
} from './feedback'
|
||||
|
||||
export class HumanFeedbackMechanismSlack<
|
||||
T extends HumanFeedbackType,
|
||||
TOutput = any
|
||||
> extends HumanFeedbackMechanism<T, TOutput> {
|
||||
protected _slackClient: SlackClient
|
||||
|
||||
constructor({
|
||||
task,
|
||||
options,
|
||||
slackClient = new SlackClient()
|
||||
}: {
|
||||
task: BaseTask
|
||||
options: Required<HumanFeedbackOptions<T, TOutput>>
|
||||
slackClient: SlackClient
|
||||
}) {
|
||||
super({ task, options })
|
||||
this._slackClient = slackClient
|
||||
}
|
||||
|
||||
protected async _annotate(): Promise<string> {
|
||||
try {
|
||||
const annotation = await this._slackClient.sendAndWaitForReply({
|
||||
text: 'Please leave an annotation (optional):'
|
||||
})
|
||||
return annotation.text
|
||||
} catch (e) {
|
||||
// Deliberately swallow the error here as the user is not required to leave an annotation
|
||||
return ''
|
||||
}
|
||||
}
|
||||
|
||||
protected async _edit(): Promise<string> {
|
||||
let { text: editedOutput } = await this._slackClient.sendAndWaitForReply({
|
||||
text: 'Copy and edit the output:'
|
||||
})
|
||||
editedOutput = editedOutput.replace(/```$/g, '')
|
||||
editedOutput = editedOutput.replace(/^```/g, '')
|
||||
return editedOutput
|
||||
}
|
||||
|
||||
protected async _askUser(
|
||||
message: string,
|
||||
choices: HumanFeedbackUserActions[]
|
||||
): Promise<HumanFeedbackUserActions> {
|
||||
message += '\n\n'
|
||||
message += choices
|
||||
.map(
|
||||
(choice, idx) => `*${idx}* - ${HumanFeedbackUserActionMessages[choice]}`
|
||||
)
|
||||
.join('\n')
|
||||
message += '\n\n'
|
||||
message += 'Reply with the number of your choice.'
|
||||
const response = await this._slackClient.sendAndWaitForReply({
|
||||
text: message,
|
||||
validate: (slackMessage) => {
|
||||
const choice = parseInt(slackMessage.text)
|
||||
return !isNaN(choice) && choice >= 0 && choice < choices.length
|
||||
}
|
||||
})
|
||||
return choices[parseInt(response.text)]
|
||||
}
|
||||
|
||||
protected async _selectOne(
|
||||
response: TOutput
|
||||
): Promise<TOutput extends (infer U)[] ? U : never> {
|
||||
if (!Array.isArray(response)) {
|
||||
throw new Error('selectOne called on non-array response')
|
||||
}
|
||||
|
||||
const { text: selectedOutput } =
|
||||
await this._slackClient.sendAndWaitForReply({
|
||||
text:
|
||||
'Pick one output:' +
|
||||
response.map((r, idx) => `\n*${idx}* - ${r}`).join('') +
|
||||
'\n\nReply with the number of your choice.',
|
||||
validate: (slackMessage) => {
|
||||
const choice = parseInt(slackMessage.text)
|
||||
return !isNaN(choice) && choice >= 0 && choice < response.length
|
||||
}
|
||||
})
|
||||
return response[parseInt(selectedOutput)]
|
||||
}
|
||||
|
||||
protected async _selectN(
|
||||
response: TOutput
|
||||
): Promise<TOutput extends any[] ? TOutput : never> {
|
||||
if (!Array.isArray(response)) {
|
||||
throw new Error('selectN called on non-array response')
|
||||
}
|
||||
|
||||
const { text: selectedOutput } =
|
||||
await this._slackClient.sendAndWaitForReply({
|
||||
text:
|
||||
'Select outputs:' +
|
||||
response.map((r, idx) => `\n*${idx}* - ${r}`).join('') +
|
||||
'\n\nReply with a comma-separated list of the output numbers of your choice.',
|
||||
validate: (slackMessage) => {
|
||||
const choices = slackMessage.text.split(',')
|
||||
return choices.every((choice) => {
|
||||
const choiceInt = parseInt(choice)
|
||||
return (
|
||||
!isNaN(choiceInt) && choiceInt >= 0 && choiceInt < response.length
|
||||
)
|
||||
})
|
||||
}
|
||||
})
|
||||
const chosenOutputs = selectedOutput
|
||||
.split(',')
|
||||
.map((choice) => parseInt(choice))
|
||||
return response.filter((_, idx) => {
|
||||
return chosenOutputs.includes(idx)
|
||||
}) as any
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,130 @@
|
|||
import { TwilioConversationClient } from '@/services/twilio-conversation'
|
||||
import { BaseTask } from '@/task'
|
||||
|
||||
import {
|
||||
HumanFeedbackMechanism,
|
||||
HumanFeedbackOptions,
|
||||
HumanFeedbackType,
|
||||
HumanFeedbackUserActionMessages,
|
||||
HumanFeedbackUserActions
|
||||
} from './feedback'
|
||||
|
||||
export class HumanFeedbackMechanismTwilio<
|
||||
T extends HumanFeedbackType,
|
||||
TOutput = any
|
||||
> extends HumanFeedbackMechanism<T, TOutput> {
|
||||
protected _twilioClient: TwilioConversationClient
|
||||
|
||||
constructor({
|
||||
task,
|
||||
options,
|
||||
twilioClient = new TwilioConversationClient()
|
||||
}: {
|
||||
task: BaseTask
|
||||
options: Required<HumanFeedbackOptions<T, TOutput>>
|
||||
twilioClient: TwilioConversationClient
|
||||
}) {
|
||||
super({ task, options })
|
||||
this._twilioClient = twilioClient
|
||||
}
|
||||
|
||||
protected async _annotate(): Promise<string> {
|
||||
try {
|
||||
const annotation = await this._twilioClient.sendAndWaitForReply({
|
||||
name: 'human-feedback-annotation',
|
||||
text: 'Please leave an annotation (optional):'
|
||||
})
|
||||
return annotation.body
|
||||
} catch (e) {
|
||||
// Deliberately swallow the error here as the user is not required to leave an annotation
|
||||
return ''
|
||||
}
|
||||
}
|
||||
|
||||
protected async _edit(): Promise<string> {
|
||||
let { body: editedOutput } = await this._twilioClient.sendAndWaitForReply({
|
||||
text: 'Copy and edit the output:',
|
||||
name: 'human-feedback-edit'
|
||||
})
|
||||
editedOutput = editedOutput.replace(/```$/g, '')
|
||||
editedOutput = editedOutput.replace(/^```/g, '')
|
||||
return editedOutput
|
||||
}
|
||||
|
||||
protected async _askUser(
|
||||
message: string,
|
||||
choices: HumanFeedbackUserActions[]
|
||||
): Promise<HumanFeedbackUserActions> {
|
||||
message += '\n\n'
|
||||
message += choices
|
||||
.map(
|
||||
(choice, idx) => `${idx} - ${HumanFeedbackUserActionMessages[choice]}`
|
||||
)
|
||||
.join('\n')
|
||||
message += '\n\n'
|
||||
message += 'Reply with the number of your choice.'
|
||||
const response = await this._twilioClient.sendAndWaitForReply({
|
||||
name: 'human-feedback-ask',
|
||||
text: message,
|
||||
validate: (message) => {
|
||||
const choice = parseInt(message.body)
|
||||
return !isNaN(choice) && choice >= 0 && choice < choices.length
|
||||
}
|
||||
})
|
||||
return choices[parseInt(response.body)]
|
||||
}
|
||||
|
||||
protected async _selectOne(
|
||||
response: TOutput
|
||||
): Promise<TOutput extends (infer U)[] ? U : never> {
|
||||
if (!Array.isArray(response)) {
|
||||
throw new Error('selectOne called on non-array response')
|
||||
}
|
||||
|
||||
const { body: selectedOutput } =
|
||||
await this._twilioClient.sendAndWaitForReply({
|
||||
name: 'human-feedback-select',
|
||||
text:
|
||||
'Pick one output:' +
|
||||
response.map((r, idx) => `\n${idx} - ${r}`).join('') +
|
||||
'\n\nReply with the number of your choice.',
|
||||
validate: (message) => {
|
||||
const choice = parseInt(message.body)
|
||||
return !isNaN(choice) && choice >= 0 && choice < response.length
|
||||
}
|
||||
})
|
||||
return response[parseInt(selectedOutput)]
|
||||
}
|
||||
|
||||
protected async _selectN(
|
||||
response: TOutput
|
||||
): Promise<TOutput extends any[] ? TOutput : never> {
|
||||
if (!Array.isArray(response)) {
|
||||
throw new Error('selectN called on non-array response')
|
||||
}
|
||||
|
||||
const { body: selectedOutput } =
|
||||
await this._twilioClient.sendAndWaitForReply({
|
||||
name: 'human-feedback-select',
|
||||
text:
|
||||
'Select outputs:' +
|
||||
response.map((r, idx) => `\n${idx} - ${r}`).join('') +
|
||||
'\n\nReply with a comma-separated list of the output numbers of your choice.',
|
||||
validate: (message) => {
|
||||
const choices = message.body.split(',')
|
||||
return choices.every((choice) => {
|
||||
const choiceInt = parseInt(choice)
|
||||
return (
|
||||
!isNaN(choiceInt) && choiceInt >= 0 && choiceInt < response.length
|
||||
)
|
||||
})
|
||||
}
|
||||
})
|
||||
const chosenOutputs = selectedOutput
|
||||
.split(',')
|
||||
.map((choice) => parseInt(choice))
|
||||
return response.filter((_, idx) => {
|
||||
return chosenOutputs.includes(idx)
|
||||
}) as any
|
||||
}
|
||||
}
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
import defaultKy from 'ky'
|
||||
|
||||
import { DEFAULT_BOT_NAME } from '@/constants'
|
||||
import { sleep } from '@/utils'
|
||||
import { chunkString, sleep } from '@/utils'
|
||||
|
||||
export const TWILIO_CONVERSATION_API_BASE_URL =
|
||||
'https://conversations.twilio.com/v1'
|
||||
|
|
@ -9,6 +9,14 @@ export const TWILIO_CONVERSATION_API_BASE_URL =
|
|||
export const DEFAULT_TWILIO_TIMEOUT_MS = 120_000
|
||||
export const DEFAULT_TWILIO_INTERVAL_MS = 5_000
|
||||
|
||||
/**
|
||||
* Twilio recommends keeping SMS messages to a length of 320 characters or less, so we'll use that as the maximum.
|
||||
*
|
||||
* @see {@link https://support.twilio.com/hc/en-us/articles/360033806753-Maximum-Message-Length-with-Twilio-Programmable-Messaging}
|
||||
*/
|
||||
const TWILIO_SMS_LENGTH_SOFT_LIMIT = 320
|
||||
const TWILIO_SMS_LENGTH_HARD_LIMIT = 1600
|
||||
|
||||
export interface TwilioConversation {
|
||||
unique_name?: string
|
||||
date_updated: Date
|
||||
|
|
@ -229,6 +237,23 @@ export class TwilioConversationClient {
|
|||
.json<TwilioConversationParticipant>()
|
||||
}
|
||||
|
||||
/**
|
||||
* Chunks a long text message into smaller parts and sends them as separate messages.
|
||||
*/
|
||||
async sendTextWithChunking({
|
||||
conversationSid,
|
||||
text
|
||||
}: {
|
||||
conversationSid: string
|
||||
text: string
|
||||
maxChunkLength?: number
|
||||
}) {
|
||||
const chunks = chunkString(text, TWILIO_SMS_LENGTH_SOFT_LIMIT)
|
||||
return Promise.all(
|
||||
chunks.map((chunk) => this.sendMessage({ conversationSid, text: chunk }))
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Posts a message to a conversation.
|
||||
*/
|
||||
|
|
@ -239,6 +264,11 @@ export class TwilioConversationClient {
|
|||
conversationSid: string
|
||||
text: string
|
||||
}) {
|
||||
// Truncate the text if it exceeds the hard limit and add an ellipsis:
|
||||
if (text.length > TWILIO_SMS_LENGTH_HARD_LIMIT) {
|
||||
text = text.substring(0, TWILIO_SMS_LENGTH_HARD_LIMIT - 3) + '...'
|
||||
}
|
||||
|
||||
const params = new URLSearchParams()
|
||||
params.set('Body', text)
|
||||
params.set('Author', this.botName)
|
||||
|
|
@ -291,7 +321,7 @@ export class TwilioConversationClient {
|
|||
|
||||
const { sid: conversationSid } = await this.createConversation(name)
|
||||
await this.addParticipant({ conversationSid, recipientPhoneNumber })
|
||||
await this.sendMessage({ conversationSid, text })
|
||||
await this.sendTextWithChunking({ conversationSid, text })
|
||||
|
||||
const start = Date.now()
|
||||
let nUserMessages = 0
|
||||
|
|
|
|||
|
|
@ -4,7 +4,6 @@ import { ZodType } from 'zod'
|
|||
import * as errors from './errors'
|
||||
import * as types from './types'
|
||||
import type { Agentic } from './agentic'
|
||||
import { defaultLogger } from './logger'
|
||||
import { defaultIDGeneratorFn, isValidTaskIdentifier } from './utils'
|
||||
|
||||
/**
|
||||
|
|
|
|||
10
src/types.ts
10
src/types.ts
|
|
@ -6,11 +6,14 @@ import type { JsonObject, JsonValue } from 'type-fest'
|
|||
import { SafeParseReturnType, ZodType, ZodTypeAny, output, z } from 'zod'
|
||||
|
||||
import type { Agentic } from './agentic'
|
||||
import type {
|
||||
FeedbackTypeToMetadata,
|
||||
HumanFeedbackType
|
||||
} from './human-feedback'
|
||||
import type { Logger } from './logger'
|
||||
import type { BaseTask } from './task'
|
||||
|
||||
export { openai }
|
||||
export { anthropic }
|
||||
export { anthropic, openai }
|
||||
|
||||
export type { Logger }
|
||||
export type { JsonObject, JsonValue }
|
||||
|
|
@ -102,6 +105,9 @@ export interface TaskResponseMetadata extends Record<string, any> {
|
|||
error?: Error
|
||||
numRetries?: number
|
||||
callId?: string
|
||||
|
||||
// human feedback info
|
||||
feedback?: FeedbackTypeToMetadata<HumanFeedbackType>
|
||||
}
|
||||
|
||||
export interface LLMTaskResponseMetadata<
|
||||
|
|
|
|||
57
src/utils.ts
57
src/utils.ts
|
|
@ -2,18 +2,39 @@ import { customAlphabet, urlAlphabet } from 'nanoid'
|
|||
|
||||
import * as types from './types'
|
||||
|
||||
/**
|
||||
* Extracts the first JSON object string from a given string.
|
||||
*
|
||||
* @param text - string from which to extract the JSON object
|
||||
* @returns extracted JSON object string, or `undefined` if no JSON object is found
|
||||
*/
|
||||
export function extractJSONObjectFromString(text: string): string | undefined {
|
||||
return text.match(/\{(.|\n)*\}/gm)?.[0]
|
||||
return text.match(/\{([^}]|\n)*\}/gm)?.[0]
|
||||
}
|
||||
|
||||
/**
|
||||
* Extracts the first JSON array string from a given string.
|
||||
*
|
||||
* @param text - string from which to extract the JSON array
|
||||
* @returns extracted JSON array string, or `undefined` if no JSON array is found
|
||||
*/
|
||||
export function extractJSONArrayFromString(text: string): string | undefined {
|
||||
return text.match(/\[(.|\n)*\]/gm)?.[0]
|
||||
return text.match(/\[([^\]]|\n)*\]/gm)?.[0]
|
||||
}
|
||||
|
||||
/**
|
||||
* Pauses the execution of a function for a specified time.
|
||||
*
|
||||
* @param ms - number of milliseconds to pause
|
||||
* @returns promise that resolves after the specified number of milliseconds
|
||||
*/
|
||||
export function sleep(ms: number) {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms))
|
||||
}
|
||||
|
||||
/**
|
||||
* A default ID generator function that uses a custom alphabet based on URL safe symbols.
|
||||
*/
|
||||
export const defaultIDGeneratorFn: types.IDGeneratorFunction =
|
||||
customAlphabet(urlAlphabet)
|
||||
|
||||
|
|
@ -21,3 +42,35 @@ const taskNameRegex = /^[a-zA-Z_][a-zA-Z0-9_-]{0,63}$/
|
|||
export function isValidTaskIdentifier(id: string): boolean {
|
||||
return !!id && taskNameRegex.test(id)
|
||||
}
|
||||
|
||||
/**
|
||||
* Chunks a string into an array of chunks.
|
||||
*
|
||||
* @param text - string to chunk
|
||||
* @param maxLength - maximum length of each chunk
|
||||
* @returns array of chunks
|
||||
*/
|
||||
export const chunkString = (text: string, maxLength: number) => {
|
||||
const words = text.split(' ')
|
||||
const chunks: string[] = []
|
||||
let chunk = ''
|
||||
|
||||
for (const word of words) {
|
||||
if (word.length > maxLength) {
|
||||
// Truncate the word if it's too long and indicate that it was truncated:
|
||||
chunks.push(word.substring(0, maxLength - 3) + '...')
|
||||
} else if ((chunk + word + 1).length > maxLength) {
|
||||
// +1 accounts for the space between words
|
||||
chunks.push(chunk.trim())
|
||||
chunk = word
|
||||
} else {
|
||||
chunk += (chunk ? ' ' : '') + word
|
||||
}
|
||||
}
|
||||
|
||||
if (chunk) {
|
||||
chunks.push(chunk.trim())
|
||||
}
|
||||
|
||||
return chunks
|
||||
}
|
||||
|
|
|
|||
|
|
@ -58,6 +58,29 @@ test.serial('TwilioConversationClient.sendMessage', async (t) => {
|
|||
await client.deleteConversation(conversationSid)
|
||||
})
|
||||
|
||||
test.serial('TwilioConversationClient.sendTextWithChunking', async (t) => {
|
||||
if (!process.env.TWILIO_ACCOUNT_SID || !process.env.TWILIO_AUTH_TOKEN) {
|
||||
return t.pass()
|
||||
}
|
||||
|
||||
const client = new TwilioConversationClient()
|
||||
|
||||
const { sid: conversationSid } = await client.createConversation(
|
||||
'send-message-test'
|
||||
)
|
||||
|
||||
const text =
|
||||
'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.'
|
||||
|
||||
const messages = await client.sendTextWithChunking({ conversationSid, text })
|
||||
|
||||
// Text should be sent in two messages:
|
||||
t.true(text.startsWith(messages[0].body))
|
||||
t.true(text.endsWith(messages[1].body))
|
||||
|
||||
await client.deleteConversation(conversationSid)
|
||||
})
|
||||
|
||||
test.serial('TwilioConversationClient.fetchMessages', async (t) => {
|
||||
if (!process.env.TWILIO_ACCOUNT_SID || !process.env.TWILIO_AUTH_TOKEN) {
|
||||
return t.pass()
|
||||
|
|
|
|||
|
|
@ -1,6 +1,13 @@
|
|||
import test from 'ava'
|
||||
|
||||
import { isValidTaskIdentifier } from '@/utils'
|
||||
import {
|
||||
chunkString,
|
||||
defaultIDGeneratorFn,
|
||||
extractJSONArrayFromString,
|
||||
extractJSONObjectFromString,
|
||||
isValidTaskIdentifier,
|
||||
sleep
|
||||
} from '@/utils'
|
||||
|
||||
test('isValidTaskIdentifier - valid', async (t) => {
|
||||
t.true(isValidTaskIdentifier('foo'))
|
||||
|
|
@ -18,3 +25,53 @@ test('isValidTaskIdentifier - invalid', async (t) => {
|
|||
t.false(isValidTaskIdentifier('x'.repeat(65)))
|
||||
t.false(isValidTaskIdentifier('-foo'))
|
||||
})
|
||||
|
||||
test('extractJSONObjectFromString should extract first JSON object from string', (t) => {
|
||||
const jsonString = '{"name":"John"} Some other text {"name":"Doe"}'
|
||||
const result = extractJSONObjectFromString(jsonString)
|
||||
t.is(result, '{"name":"John"}')
|
||||
})
|
||||
|
||||
test('extractJSONArrayFromString should extract first JSON array from string', (t) => {
|
||||
const jsonString = '[1,2,3] Some other text [4,5,6]'
|
||||
const result = extractJSONArrayFromString(jsonString)
|
||||
t.is(result, '[1,2,3]')
|
||||
})
|
||||
|
||||
test('extractJSONObjectFromString should return undefined if no JSON object is found', (t) => {
|
||||
const jsonString = 'Some text'
|
||||
const result = extractJSONObjectFromString(jsonString)
|
||||
t.is(result, undefined)
|
||||
})
|
||||
|
||||
test('extractJSONArrayFromString should return undefined if no JSON array is found', (t) => {
|
||||
const jsonString = 'Some text'
|
||||
const result = extractJSONArrayFromString(jsonString)
|
||||
t.is(result, undefined)
|
||||
})
|
||||
|
||||
test('sleep should delay execution', async (t) => {
|
||||
const start = Date.now()
|
||||
await sleep(1000) // for example, 1000ms / 1sec
|
||||
const end = Date.now()
|
||||
t.true(end - start >= 1000)
|
||||
})
|
||||
|
||||
test('defaultIDGeneratorFn should generate URL-safe string', (t) => {
|
||||
const result = defaultIDGeneratorFn()
|
||||
|
||||
// Check if generated string matches URL-safe characters:
|
||||
t.regex(result, /^[A-Za-z0-9\-_]+$/)
|
||||
})
|
||||
|
||||
test('chunkString should split string into chunks', (t) => {
|
||||
const text = 'Hello, this is a test string for chunkString function.'
|
||||
const chunks = chunkString(text, 12)
|
||||
t.deepEqual(chunks, [
|
||||
'Hello, this',
|
||||
'is a test',
|
||||
'string for',
|
||||
'chunkString',
|
||||
'function.'
|
||||
])
|
||||
})
|
||||
|
|
|
|||
Ładowanie…
Reference in New Issue