From c20ce31e00eeb892ad5433a602487dcfb0c68559 Mon Sep 17 00:00:00 2001
From: Philipp Burckhardt <pburckhardt@outlook.com>
Date: Wed, 28 Jun 2023 18:42:00 -0400
Subject: [PATCH] feat: refactor human feedback to change results

---
 examples/expert-qa.ts          | 70 ++++++++++++++++------------------
 examples/facts.ts              |  6 ++-
 src/human-feedback/feedback.ts |  2 +-
 src/task.ts                    | 16 +++++++-
 src/types.ts                   |  6 ++-
 5 files changed, 58 insertions(+), 42 deletions(-)

diff --git a/examples/expert-qa.ts b/examples/expert-qa.ts
index 0d31f34..25452f2 100644
--- a/examples/expert-qa.ts
+++ b/examples/expert-qa.ts
@@ -10,7 +10,7 @@ async function main() {
 
   const question = 'How do I build a product that people will love?'
 
-  const { metadata } = await agentic
+  const result = await agentic
     .gpt4(
       `Generate a list of {n} prominent experts that can answer the following question: {{question}}.`
     )
@@ -32,56 +32,50 @@ async function main() {
     .withHumanFeedback({
       type: 'multiselect'
     })
-    .callWithMetadata({
+    .call({
       question
     })
 
-  if (
-    metadata.feedback &&
-    metadata.feedback.type === 'multiselect' &&
-    metadata.feedback.selected
-  ) {
-    const answer = await agentic
-      .gpt4(
-        `Generate an answer to the following question: "{{question}}" from each of the following experts: {{#each experts}}
+  const answer = await agentic
+    .gpt4(
+      `Generate an answer to the following question: "{{question}}" from each of the following experts: {{#each experts}}
       - {{this.name}}: {{this.bio}}
       {{/each}}`
-      )
-      .output(
-        z.array(
-          z.object({
-            expert: z.string(),
-            answer: z.string()
-          })
-        )
-      )
-      .input(
+    )
+    .output(
+      z.array(
         z.object({
-          question: z.string(),
-          experts: z.array(z.object({ name: z.string(), bio: z.string() }))
+          expert: z.string(),
+          answer: z.string()
         })
       )
-      .call({
-        question,
-        experts: metadata.feedback.selected
+    )
+    .input(
+      z.object({
+        question: z.string(),
+        experts: z.array(z.object({ name: z.string(), bio: z.string() }))
       })
+    )
+    .call({
+      question,
+      experts: result
+    })
 
-    const message = answer.reduce((acc, { expert, answer }) => {
-      return `${acc}
+  const message = answer.reduce((acc, { expert, answer }) => {
+    return `${acc}
 
       ${expert}: ${answer}`
-    }, '')
+  }, '')
 
-    const notifier = new NovuNotificationTool()
-    await notifier.call({
-      name: 'send-email',
-      payload: {
-        subject: 'Experts have answered your question: ' + question,
-        message
-      },
-      to: [{ subscriberId: '123' }]
-    })
-  }
+  const notifier = new NovuNotificationTool()
+  await notifier.call({
+    name: 'send-email',
+    payload: {
+      subject: 'Experts have answered your question: ' + question,
+      message
+    },
+    to: [{ subscriberId: '123' }]
+  })
 }
 
 main()
diff --git a/examples/facts.ts b/examples/facts.ts
index 0c308c0..a8ef574 100644
--- a/examples/facts.ts
+++ b/examples/facts.ts
@@ -16,8 +16,12 @@ async function main() {
         numFacts: z.number().int().default(5)
       })
     )
-    .output(z.object({ facts: z.array(z.string()) }))
+    .output(z.array(z.string()))
     .modelParams({ temperature: 0.9 })
+    .withHumanFeedback({
+      type: 'confirm',
+      editing: true
+    })
     .call({ topic: 'cats' })
 
   console.log(out)
diff --git a/src/human-feedback/feedback.ts b/src/human-feedback/feedback.ts
index 849dffb..54253d3 100644
--- a/src/human-feedback/feedback.ts
+++ b/src/human-feedback/feedback.ts
@@ -253,7 +253,7 @@ export abstract class HumanFeedbackMechanism<
             throw new Error('Expected output to be an array')
           }
 
-          feedback.chosen = await this._select(output)
+          feedback.chosen = [await this._select(output)]
         }
 
         break
diff --git a/src/task.ts b/src/task.ts
index a4dc82a..ef3f3b2 100644
--- a/src/task.ts
+++ b/src/task.ts
@@ -172,6 +172,18 @@ export abstract class BaseTask<
     this.addAfterCallHook(async (output, ctx) => {
       const feedback = await feedbackMechanism.interact(output)
       ctx.metadata = { ...ctx.metadata, feedback }
+      if (feedback.editedOutput) {
+        return feedback.editedOutput
+      }
+
+      switch (feedback.type) {
+        case 'confirm':
+          return output
+        case 'select':
+          return feedback.chosen
+        case 'multiselect':
+          return feedback.selected
+      }
     })
 
     return this
@@ -250,12 +262,14 @@ export abstract class BaseTask<
 
     const result = await pRetry(
       async () => {
-        const result = await this._call(ctx)
+        let result = await this._call(ctx)
 
         for (const { hook: postHook } of this._postHooks) {
           const postHookResult = await postHook(result, ctx)
           if (postHookResult === SKIP_HOOKS) {
             break
+          } else if (postHookResult !== undefined) {
+            result = postHookResult
           }
         }
 
diff --git a/src/types.ts b/src/types.ts
index 26f9ba9..e6bc995 100644
--- a/src/types.ts
+++ b/src/types.ts
@@ -191,4 +191,8 @@ export type TaskAfterCallHook<
 > = (
   output: TOutput,
   ctx: TaskCallContext<TInput>
-) => void | typeof SKIP_HOOKS | Promise<void | typeof SKIP_HOOKS>
+) =>
+  | void
+  | TOutput
+  | typeof SKIP_HOOKS
+  | Promise<void | typeof SKIP_HOOKS | TOutput>