feat: move withHumanFeedback to a Task method

old-agentic-v1^2
Travis Fischer 2023-06-17 23:25:09 -07:00
rodzic 7318eb0988
commit 831a91b2c9
8 zmienionych plików z 144 dodań i 162 usunięć

Wyświetl plik

@ -2,12 +2,7 @@ import 'dotenv/config'
import { OpenAIClient } from 'openai-fetch'
import { z } from 'zod'
import {
Agentic,
NovuNotificationTool,
SerpAPITool,
withHumanFeedback
} from '@/index'
import { Agentic, NovuNotificationTool, SerpAPITool } from '@/index'
async function main() {
const openai = new OpenAIClient({ apiKey: process.env.OPENAI_API_KEY! })
@ -15,33 +10,31 @@ async function main() {
const question = 'How do I build a product that people will love?'
const task = withHumanFeedback(
agentic
.gpt4(
`Generate a list of {n} prominent experts that can answer the following question: {{question}}.`
)
.tools([new SerpAPITool()])
.output(
z.array(
z.object({
name: z.string(),
bio: z.string()
})
)
)
.input(
const { metadata } = await agentic
.gpt4(
`Generate a list of {n} prominent experts that can answer the following question: {{question}}.`
)
.tools([new SerpAPITool()])
.output(
z.array(
z.object({
question: z.string(),
n: z.number().int().default(5)
name: z.string(),
bio: z.string()
})
),
{
)
)
.input(
z.object({
question: z.string(),
n: z.number().int().default(5)
})
)
.withHumanFeedback({
type: 'selectN'
}
)
const { metadata } = await task.callWithMetadata({
question
})
})
.callWithMetadata({
question
})
if (
metadata.feedback &&
@ -78,6 +71,7 @@ async function main() {
${expert}: ${answer}`
}, '')
const notifier = new NovuNotificationTool()
await notifier.call({
name: 'send-email',

58
examples/hf0-demo.ts vendored
Wyświetl plik

@ -6,43 +6,43 @@ import {
Agentic,
HumanFeedbackMechanismTwilio,
SearchAndCrawlTool,
WeatherTool,
withHumanFeedback
WeatherTool
} from '@/index'
async function main() {
const openai = new OpenAIClient({ apiKey: process.env.OPENAI_API_KEY! })
const agentic = new Agentic({ openai })
const agentic = new Agentic({
openai,
humanFeedbackDefaults: {
mechanism: HumanFeedbackMechanismTwilio
}
})
const topic = process.argv[2] || 'OpenAI'
const res0 = await withHumanFeedback(
agentic
.gpt4({
messages: [
{
role: 'system',
content: `You are a McKinsey analyst who is an expert at writing executive summaries.`
},
{
role: 'user',
content: `What are the 3 most important questions we would need to answer in order to have a thorough understanding of this topic: {{topic}}? Be concise but creative in your questions, and make sure to capture the true essence of the topic.`
}
],
model: 'gpt-4',
temperature: 1.0
const res0 = await agentic
.gpt4({
messages: [
{
role: 'system',
content: `You are a McKinsey analyst who is an expert at writing executive summaries.`
},
{
role: 'user',
content: `What are the 3 most important questions we would need to answer in order to have a thorough understanding of this topic: {{topic}}? Be concise but creative in your questions, and make sure to capture the true essence of the topic.`
}
],
model: 'gpt-4',
temperature: 1.0
})
.input(
z.object({
topic: z.string()
})
.input(
z.object({
topic: z.string()
})
)
.output(z.array(z.string()).describe('question')),
{
type: 'selectN',
mechanism: HumanFeedbackMechanismTwilio
}
).callWithMetadata({ topic })
)
.output(z.array(z.string()).describe('question'))
.withHumanFeedback({ type: 'selectN' })
.callWithMetadata({ topic })
console.log()
console.log()

Wyświetl plik

@ -2,14 +2,13 @@ import 'dotenv/config'
import { OpenAIClient } from 'openai-fetch'
import { z } from 'zod'
import { HumanFeedbackMechanismCLI } from '@/human-feedback'
import { Agentic, withHumanFeedback } from '@/index'
import { Agentic } from '@/index'
async function main() {
const openai = new OpenAIClient({ apiKey: process.env.OPENAI_API_KEY! })
const ai = new Agentic({ openai })
const topicJokes = ai
const out = await ai
.gpt3(`Tell me {{num}} jokes about {{topic}}`)
.input(
z.object({
@ -19,19 +18,17 @@ async function main() {
)
.output(z.array(z.string()))
.modelParams({ temperature: 0.9 })
.withHumanFeedback({
type: 'selectN',
annotations: false,
abort: false,
editing: true
})
.callWithMetadata({
topic: 'politicians',
num: 5
})
const topicJokesFeedback = withHumanFeedback(topicJokes, {
type: 'selectN',
annotations: false,
abort: false,
editing: true,
mechanism: HumanFeedbackMechanismCLI
})
const out = await topicJokesFeedback.callWithMetadata({
topic: 'politicians',
num: 5
})
const feedback = out.metadata.feedback
console.log(JSON.stringify(feedback, null, 2))
}

Wyświetl plik

@ -2,14 +2,13 @@ import 'dotenv/config'
import { OpenAIClient } from 'openai-fetch'
import { z } from 'zod'
import { HumanFeedbackMechanismSlack } from '@/human-feedback'
import { Agentic, withHumanFeedback } from '@/index'
import { Agentic, HumanFeedbackMechanismSlack } from '@/index'
async function main() {
const openai = new OpenAIClient({ apiKey: process.env.OPENAI_API_KEY! })
const ai = new Agentic({ openai })
const topicJokes = ai
const out = await ai
.gpt3(`Tell me {{num}} jokes about {{topic}}`)
.input(
z.object({
@ -19,19 +18,18 @@ async function main() {
)
.output(z.array(z.string()))
.modelParams({ temperature: 0.9 })
.withHumanFeedback({
type: 'selectN',
annotations: false,
abort: false,
editing: true,
mechanism: HumanFeedbackMechanismSlack
})
.callWithMetadata({
topic: 'politicians',
num: 5
})
const topicJokesFeedback = withHumanFeedback(topicJokes, {
type: 'selectN',
annotations: false,
abort: false,
editing: true,
mechanism: HumanFeedbackMechanismSlack
})
const out = await topicJokesFeedback.callWithMetadata({
topic: 'politicians',
num: 5
})
const feedback = out.metadata.feedback
console.log(JSON.stringify(feedback, null, 2))
}

Wyświetl plik

@ -2,14 +2,13 @@ import 'dotenv/config'
import { OpenAIClient } from 'openai-fetch'
import { z } from 'zod'
import { HumanFeedbackMechanismTwilio } from '@/human-feedback'
import { Agentic, withHumanFeedback } from '@/index'
import { Agentic, HumanFeedbackMechanismTwilio } from '@/index'
async function main() {
const openai = new OpenAIClient({ apiKey: process.env.OPENAI_API_KEY! })
const ai = new Agentic({ openai })
const topicJokes = ai
const out = await ai
.gpt3(`Tell me {{num}} jokes about {{topic}}`)
.input(
z.object({
@ -19,19 +18,18 @@ async function main() {
)
.output(z.array(z.string()))
.modelParams({ temperature: 0.9 })
.withHumanFeedback({
type: 'selectN',
annotations: false,
abort: false,
editing: true,
mechanism: HumanFeedbackMechanismTwilio
})
.callWithMetadata({
topic: 'politicians',
num: 5
})
const topicJokesFeedback = withHumanFeedback(topicJokes, {
type: 'selectN',
annotations: false,
abort: false,
editing: true,
mechanism: HumanFeedbackMechanismTwilio
})
const out = await topicJokesFeedback.callWithMetadata({
topic: 'politicians',
num: 5
})
const feedback = out.metadata.feedback
console.log(JSON.stringify(feedback, null, 2))
}

Wyświetl plik

@ -1,10 +1,7 @@
import * as types from '@/types'
import { Agentic } from '@/agentic'
import { HumanFeedbackDeclineError } from '@/errors'
import { BaseTask } from '@/task'
import { HumanFeedbackMechanismCLI } from './cli'
/**
* Actions the user can take in the feedback selection prompt.
*/
@ -303,48 +300,3 @@ export abstract class HumanFeedbackMechanism<
return feedback as FeedbackTypeToMetadata<T>
}
}
export function withHumanFeedback<
TInput extends types.TaskInput,
TOutput extends types.TaskOutput,
V extends HumanFeedbackType
>(
task: BaseTask<TInput, TOutput>,
options: HumanFeedbackOptions<V, TOutput> = {}
) {
task = task.clone()
// Use Object.assign to merge the options, instance defaults, and hard-coded defaults:
const finalOptions: HumanFeedbackOptions<V, TOutput> = Object.assign(
{
type: 'confirm',
abort: false,
editing: false,
annotations: false,
timeoutMs: Number.POSITIVE_INFINITY,
mechanism: HumanFeedbackMechanismCLI
},
// Default options from the instance:
task.agentic.humanFeedbackDefaults,
// User-provided options (override instance defaults):
options
)
if (!finalOptions.mechanism) {
throw new Error(
'No feedback mechanism provided. Please provide a feedback mechanism to use.'
)
}
const feedbackMechanism = new finalOptions.mechanism({
task: task,
options: finalOptions
})
task.addAfterCallHook(async function onCall(output, ctx) {
const feedback = await feedbackMechanism.interact(output)
ctx.metadata = { ...ctx.metadata, feedback }
})
return task
}

Wyświetl plik

@ -4,6 +4,11 @@ import { ZodType } from 'zod'
import * as errors from './errors'
import * as types from './types'
import type { Agentic } from './agentic'
import {
HumanFeedbackMechanismCLI,
HumanFeedbackOptions,
HumanFeedbackType
} from './human-feedback'
import { defaultIDGeneratorFn, isValidTaskIdentifier } from './utils'
/**
@ -29,8 +34,8 @@ export abstract class BaseTask<
protected _timeoutMs?: number
protected _retryConfig: types.RetryConfig
private _preHooks: Array<types.BeforeCallHook<TInput>> = []
private _postHooks: Array<types.AfterCallHook<TInput, TOutput>> = []
private _preHooks: Array<types.TaskBeforeCallHook<TInput>> = []
private _postHooks: Array<types.TaskAfterCallHook<TInput, TOutput>> = []
constructor(options: types.BaseTaskOptions = {}) {
this._agentic = options.agentic ?? globalThis.__agentic?.deref()
@ -77,12 +82,14 @@ export abstract class BaseTask<
return ''
}
public addBeforeCallHook(hook: types.BeforeCallHook<TInput>): this {
public addBeforeCallHook(hook: types.TaskBeforeCallHook<TInput>): this {
this._preHooks.push(hook)
return this
}
public addAfterCallHook(hook: types.AfterCallHook<TInput, TOutput>): this {
public addAfterCallHook(
hook: types.TaskAfterCallHook<TInput, TOutput>
): this {
this._postHooks.push(hook)
return this
}
@ -106,6 +113,41 @@ export abstract class BaseTask<
throw new Error(`clone not implemented for task "${this.nameForModel}"`)
}
public withHumanFeedback<V extends HumanFeedbackType>(
options: HumanFeedbackOptions<V, TOutput> = {}
): this {
options = Object.assign(
{
type: 'confirm',
abort: false,
editing: false,
annotations: false,
timeoutMs: Number.POSITIVE_INFINITY,
mechanism: HumanFeedbackMechanismCLI
},
this.agentic.humanFeedbackDefaults,
options
)
if (!options.mechanism) {
throw new Error(
'No feedback mechanism provided. Please provide a feedback mechanism to use.'
)
}
const feedbackMechanism = new options.mechanism({
task: this,
options
})
this.addAfterCallHook(async (output, ctx) => {
const feedback = await feedbackMechanism.interact(output)
ctx.metadata = { ...ctx.metadata, feedback }
})
return this
}
public retryConfig(retryConfig: types.RetryConfig): this {
this._retryConfig = retryConfig
return this
@ -152,15 +194,16 @@ export abstract class BaseTask<
}
}
for (const hook of this._preHooks) {
await hook(ctx)
for (const preHook of this._preHooks) {
await preHook(ctx)
}
const result = await pRetry(
async () => {
const result = await this._call(ctx)
for (const hook of this._postHooks) {
await hook(result, ctx)
for (const postHook of this._postHooks) {
await postHook(result, ctx)
}
return result

Wyświetl plik

@ -155,11 +155,11 @@ export declare class CancelablePromise<T> extends Promise<T> {
// export type ProgressFunction = (partialResponse: ChatMessage) => void
export type BeforeCallHook<TInput extends TaskInput = void> = (
export type TaskBeforeCallHook<TInput extends TaskInput = void> = (
ctx: TaskCallContext<TInput>
) => void | Promise<void>
export type AfterCallHook<
export type TaskAfterCallHook<
TInput extends TaskInput = void,
TOutput extends TaskOutput = string
> = (output: TOutput, ctx: TaskCallContext<TInput>) => void | Promise<void>