Merge pull request #30 from transitive-bullshit/human-feedback-output

feat: refactor human feedback to change results

examples/expert-qa.ts

@@ -10,7 +10,7 @@ async function main() {
 
   const question = 'How do I build a product that people will love?'
 
-  const { metadata } = await agentic
+  const result = await agentic
     .gpt4(
       `Generate a list of {n} prominent experts that can answer the following question: {{question}}.`
     )
@@ -32,56 +32,50 @@ async function main() {
     .withHumanFeedback({
       type: 'multiselect'
     })
-    .callWithMetadata({
+    .call({
       question
     })
 
-  if (
-    metadata.feedback &&
-    metadata.feedback.type === 'multiselect' &&
-    metadata.feedback.selected
-  ) {
-    const answer = await agentic
-      .gpt4(
-        `Generate an answer to the following question: "{{question}}" from each of the following experts: {{#each experts}}
+  const answer = await agentic
+    .gpt4(
+      `Generate an answer to the following question: "{{question}}" from each of the following experts: {{#each experts}}
       - {{this.name}}: {{this.bio}}
       {{/each}}`
-      )
-      .output(
-        z.array(
-          z.object({
-            expert: z.string(),
-            answer: z.string()
-          })
-        )
-      )
-      .input(
+    )
+    .output(
+      z.array(
         z.object({
-          question: z.string(),
-          experts: z.array(z.object({ name: z.string(), bio: z.string() }))
+          expert: z.string(),
+          answer: z.string()
         })
       )
-      .call({
-        question,
-        experts: metadata.feedback.selected
+    )
+    .input(
+      z.object({
+        question: z.string(),
+        experts: z.array(z.object({ name: z.string(), bio: z.string() }))
       })
+    )
+    .call({
+      question,
+      experts: result
+    })
 
-    const message = answer.reduce((acc, { expert, answer }) => {
-      return `${acc}
+  const message = answer.reduce((acc, { expert, answer }) => {
+    return `${acc}
 
       ${expert}: ${answer}`
-    }, '')
+  }, '')
 
-    const notifier = new NovuNotificationTool()
-    await notifier.call({
-      name: 'send-email',
-      payload: {
-        subject: 'Experts have answered your question: ' + question,
-        message
-      },
-      to: [{ subscriberId: '123' }]
-    })
-  }
+  const notifier = new NovuNotificationTool()
+  await notifier.call({
+    name: 'send-email',
+    payload: {
+      subject: 'Experts have answered your question: ' + question,
+      message
+    },
+    to: [{ subscriberId: '123' }]
+  })
 }
 
 main()

examples/facts.ts

@@ -16,8 +16,12 @@ async function main() {
         numFacts: z.number().int().default(5)
       })
     )
-    .output(z.object({ facts: z.array(z.string()) }))
+    .output(z.array(z.string()))
     .modelParams({ temperature: 0.9 })
+    .withHumanFeedback({
+      type: 'confirm',
+      editing: true
+    })
     .call({ topic: 'cats' })
 
   console.log(out)

src/human-feedback/feedback.ts

@@ -253,7 +253,7 @@ export abstract class HumanFeedbackMechanism<
             throw new Error('Expected output to be an array')
           }
 
-          feedback.chosen = await this._select(output)
+          feedback.chosen = [await this._select(output)]
         }
 
         break

src/task.ts

@@ -172,6 +172,18 @@ export abstract class BaseTask<
     this.addAfterCallHook(async (output, ctx) => {
       const feedback = await feedbackMechanism.interact(output)
       ctx.metadata = { ...ctx.metadata, feedback }
+      if (feedback.editedOutput) {
+        return feedback.editedOutput
+      }
+
+      switch (feedback.type) {
+        case 'confirm':
+          return output
+        case 'select':
+          return feedback.chosen
+        case 'multiselect':
+          return feedback.selected
+      }
     })
 
     return this
@@ -250,12 +262,14 @@ export abstract class BaseTask<
 
     const result = await pRetry(
       async () => {
-        const result = await this._call(ctx)
+        let result = await this._call(ctx)
 
         for (const { hook: postHook } of this._postHooks) {
           const postHookResult = await postHook(result, ctx)
           if (postHookResult === SKIP_HOOKS) {
             break
+          } else if (postHookResult !== undefined) {
+            result = postHookResult
           }
         }
 

src/types.ts

@@ -191,4 +191,8 @@ export type TaskAfterCallHook<
 > = (
   output: TOutput,
   ctx: TaskCallContext<TInput>
-) => void | typeof SKIP_HOOKS | Promise<void | typeof SKIP_HOOKS>
+) =>
+  | void
+  | TOutput
+  | typeof SKIP_HOOKS
+  | Promise<void | typeof SKIP_HOOKS | TOutput>