diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..53e8005 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,14 @@ +# See https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners + +# This file auto-assigns reviewers for pull requests based on file/directory changes. +# Pattern: path/to/code @owner1 @owner2 + +# Example usages: +# * @username # username is the owner of the whole repo +# /folder/ @username # username is the owner of 'folder/' directory +# *.ext @username # username is the owner of all '.ext' files + +# Remember: +# 1. Users/teams must have write access to the repo. +# 2. Multiple owners can be specified. +# 3. The last matching pattern takes precedence. diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 0000000..f151819 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,73 @@ +name: 🐛 Bug Report +description: Report an issue or possible bug with Agentic +labels: [bug] +body: + - type: markdown + attributes: + value: | + Thank you for taking the time to file a bug report! Please fill out this form as completely as possible. + - type: input + id: version + attributes: + label: Agentic Version + description: What version of agentic are you using? + placeholder: 0.0.0 + validations: + required: true + - type: input + id: node-version + attributes: + label: Node.js / npm Version + description: What Node.js / npm version(s) are you using? + validations: + required: true + - type: input + id: platform + attributes: + label: Platform + description: What operating system are you on? + placeholder: Windows 10, macOS 11.5.2, Ubuntu 20.04, etc. + validations: + required: true + - type: textarea + id: bug-description + attributes: + label: Describe the Bug + description: A clear and concise description of what the bug is. + validations: + required: true + - type: textarea + id: expected-results + attributes: + label: Expected Results + description: What are the expected results? + placeholder: Insert expected results here + validations: + required: true + - type: textarea + id: actual-results + attributes: + label: Actual Results + description: What are the actual results? + placeholder: Insert actual results here + validations: + required: true + - type: input + id: bug-reproduction + attributes: + label: Link to Minimal Reproducible Example + description: 'If relevant, provide a link to a minimal reproduction of the problem. You may use a code playground like [Replit](https://replit.com) or [CodeSandbox](https://codesandbox.io). If you are unable to provide a link, please provide a code snippet or a detailed description of the steps necessary to reproduce the problem.' + validations: + required: true + - type: checkboxes + id: checklist + attributes: + label: Checklist + description: Please ensure the following tasks are completed before filing a bug report. + options: + - label: Read and understood the Code of Conduct. + required: true + - label: Searched for existing issues and pull requests. + required: true + - label: I am willing to submit a pull request for this issue. + required: false diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..3786b54 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,8 @@ +blank_issues_enabled: true +contact_links: + - name: ❓ Question + url: https://github.com/transitive-bullshit/agentic/discussions/new?category=Q%26A + about: Got a (non-bug related) question? Ask the community for help! + - name: 💡 Feature Request + url: https://github.com/transitive-bullshit/agentic/discussions/new?category=Ideas + about: Share ideas for new features with the community. diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..de3d32a --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,16 @@ +## Changes + +- This PR... (please explain what changes you made and why). +- Fixes #issue_number (if the PR fixes an open issue) + +## Testing + + + +## Additional Information + + diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..33369e0 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,10 @@ +version: 2 +updates: +- package-ecosystem: npm + directory: "/" + schedule: + interval: monthly + open-pull-requests-limit: 10 + allow: + - dependency-name: "*" + dependency-type: "all" diff --git a/citation.cff b/citation.cff new file mode 100644 index 0000000..a571b04 --- /dev/null +++ b/citation.cff @@ -0,0 +1,30 @@ +cff-version: 1.2.0 +title: Agentic +message: >- + If you use this software, please cite it using the + metadata from this file. + +type: software + +authors: + - given-names: Travis + family-names: Fischer + email: travis@transitivebullsh.it + - given-names: Philipp + family-names: Burckhardt + email: pburckhardt@outlook.com + +repository-code: https://github.com/transitive-bullshit/agentic +url: https://github.com/transitive-bullshit/agentic + +abstract: | + Build reliable AI Agents with TypeScript. + +keywords: +- AI +- Agents +- TypeScript + +license: MIT + +date-released: 2023 diff --git a/package.json b/package.json index c26ffed..1f3b415 100644 --- a/package.json +++ b/package.json @@ -65,7 +65,6 @@ "uuid": "^9.0.0", "zod": "^3.21.4", "zod-to-json-schema": "^3.21.2", - "zod-to-ts": "^1.1.4", "zod-validation-error": "^1.3.0" }, "devDependencies": { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f3c94b7..755d437 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -86,9 +86,6 @@ dependencies: zod-to-json-schema: specifier: ^3.21.2 version: 3.21.2(zod@3.21.4) - zod-to-ts: - specifier: ^1.1.4 - version: 1.1.4(typescript@5.1.3)(zod@3.21.4) zod-validation-error: specifier: ^1.3.0 version: 1.3.0(zod@3.21.4) @@ -4448,6 +4445,7 @@ packages: resolution: {integrity: sha512-XH627E9vkeqhlZFQuL+UsyAXEnibT0kWR2FWONlr4sTjvxyJYnyefgrkyECLzM5NenmKzRAy2rR/OlYLA1HkZw==} engines: {node: '>=14.17'} hasBin: true + dev: true /uglify-js@3.17.4: resolution: {integrity: sha512-T9q82TJI9e/C1TAxYvfb16xO120tMVFZrGA3f9/P4424DNu6ypK103y0GPFVa17yotwSyZW5iYXgjYHkGrJW/g==} @@ -4645,16 +4643,6 @@ packages: zod: 3.21.4 dev: false - /zod-to-ts@1.1.4(typescript@5.1.3)(zod@3.21.4): - resolution: {integrity: sha512-jsCg+pTNxLAdJOfW4ul+SpechdGYEJPPnssSbqWdR2LSIkotT22k+UvqPb1nEHwe/YbEcbUOlZUfGM0npgR+Jg==} - peerDependencies: - typescript: ^4.9.4 || ^5.0.2 - zod: ^3 - dependencies: - typescript: 5.1.3 - zod: 3.21.4 - dev: false - /zod-validation-error@1.3.0(zod@3.21.4): resolution: {integrity: sha512-4WoQnuWnj06kwKR4A+cykRxFmy+CTvwMQO5ogTXLiVx1AuvYYmMjixh7sbkSsQTr1Fvtss6d5kVz8PGeMPUQjQ==} engines: {node: '>=16.0.0'} diff --git a/src/llms/chat.ts b/src/llms/chat.ts index 5f36a95..7e6cb6e 100644 --- a/src/llms/chat.ts +++ b/src/llms/chat.ts @@ -2,18 +2,14 @@ import { JSONRepairError, jsonrepair } from 'jsonrepair' import { dedent } from 'ts-dedent' import { type SetRequired } from 'type-fest' import { ZodType, z } from 'zod' -import { printNode, zodToTs } from 'zod-to-ts' +import { zodToJsonSchema } from 'zod-to-json-schema' import * as errors from '@/errors' import * as types from '@/types' +import { parseOutput } from '@/llms/parse-output' import { BaseTask } from '@/task' import { getCompiledTemplate } from '@/template' -import { - extractFunctionIdentifierFromString, - extractJSONArrayFromString, - extractJSONObjectFromString, - stringifyForModel -} from '@/utils' +import { extractFunctionIdentifierFromString, stringifyForModel } from '@/utils' import { BaseLLM } from './llm' import { @@ -21,113 +17,6 @@ import { getNumTokensForChatMessages } from './llm-utils' -const BOOLEAN_OUTPUTS = { - true: true, - false: false, - t: true, - f: false, - yes: true, - no: false, - y: true, - n: false, - '1': true, - '0': false -} - -function parseArrayOutput(output: string): Array { - try { - const trimmedOutput = extractJSONArrayFromString(output) - const parsedOutput = JSON.parse(jsonrepair(trimmedOutput ?? output)) - return parsedOutput - } catch (err: any) { - if (err instanceof JSONRepairError) { - throw new errors.OutputValidationError(err.message, { cause: err }) - } else if (err instanceof SyntaxError) { - throw new errors.OutputValidationError( - `Invalid JSON array: ${err.message}`, - { cause: err } - ) - } else { - throw err - } - } -} - -function parseObjectOutput(output) { - try { - const trimmedOutput = extractJSONObjectFromString(output) - output = JSON.parse(jsonrepair(trimmedOutput ?? output)) - - if (Array.isArray(output)) { - // TODO - output = output[0] - } - - return output - } catch (err: any) { - if (err instanceof JSONRepairError) { - throw new errors.OutputValidationError(err.message, { cause: err }) - } else if (err instanceof SyntaxError) { - throw new errors.OutputValidationError( - `Invalid JSON object: ${err.message}`, - { cause: err } - ) - } else { - throw err - } - } -} - -function parseBooleanOutput(output): boolean { - output = output - .toLowerCase() - .trim() - .replace(/[.!?]+$/, '') - - const booleanOutput = BOOLEAN_OUTPUTS[output] - - if (booleanOutput !== undefined) { - return booleanOutput - } else { - throw new errors.OutputValidationError(`Invalid boolean output: ${output}`) - } -} - -function parseNumberOutput(output, outputSchema: z.ZodNumber): number { - output = output.trim() - - const numberOutput = outputSchema.isInt - ? parseInt(output) - : parseFloat(output) - - if (isNaN(numberOutput)) { - throw new errors.OutputValidationError(`Invalid number output: ${output}`) - } - - return numberOutput -} - -function parseOutput(output: any, outputSchema: ZodType) { - if (outputSchema instanceof z.ZodArray) { - output = parseArrayOutput(output) - } else if (outputSchema instanceof z.ZodObject) { - output = parseObjectOutput(output) - } else if (outputSchema instanceof z.ZodBoolean) { - output = parseBooleanOutput(output) - } else if (outputSchema instanceof z.ZodNumber) { - output = parseNumberOutput(output, outputSchema) - } - - // TODO: fix typescript issue here with recursive types - const safeResult = (outputSchema.safeParse as any)(output) - - if (!safeResult.success) { - throw new errors.ZodOutputValidationError(safeResult.error) - } - - return safeResult.data -} - export abstract class BaseChatCompletion< TInput extends types.TaskInput = void, TOutput extends types.TaskOutput = string, @@ -275,22 +164,12 @@ export abstract class BaseChatCompletion< return null } - // TODO: replace zod-to-ts with zod-to-json-schema? - const { node } = zodToTs(outputSchema) - - if (node.kind === 152) { - // Handle raw strings differently: - return dedent`Output a raw string only, without any additional text.` - } - - const tsTypeString = printNode(node, { - removeComments: false, - // TODO: this doesn't seem to actually work, so we're doing it manually below - omitTrailingSemicolon: true, - noEmitHelpers: true - }) - .replace(/^ {4}/gm, ' ') - .replace(/;$/gm, '') + const schema = zodToJsonSchema(outputSchema) as types.Jsonifiable + const schemaStr = stringifyForModel(schema, [ + 'default', + 'additionalProperties', + '$schema' + ]) let label: string if (outputSchema instanceof z.ZodArray) { label = 'JSON array (minified)' @@ -306,9 +185,9 @@ export abstract class BaseChatCompletion< label = 'JSON value' } - return dedent`Do not output code. Output a single ${label} in the following TypeScript format: - \`\`\`ts - ${tsTypeString} + return dedent`Do not output code. Output a single ${label} according to the following JSON Schema: + \`\`\`json + ${schemaStr} \`\`\`` } @@ -472,7 +351,7 @@ export abstract class BaseChatCompletion< // console.log('<<<') if (this._outputSchema) { - return parseOutput(output, this._outputSchema) + return parseOutput(output as string, this._outputSchema) } else { return output } diff --git a/src/llms/parse-output.ts b/src/llms/parse-output.ts new file mode 100644 index 0000000..3dea6f6 --- /dev/null +++ b/src/llms/parse-output.ts @@ -0,0 +1,247 @@ +import { JSONRepairError, jsonrepair } from 'jsonrepair' +import { JsonValue } from 'type-fest' +import { ZodType, z } from 'zod' + +import * as errors from '@/errors' + +/** + * Checks if character at the specified index in a string is escaped. + * + * @param str - string to check + * @param i - index of the character to check + * @returns whether the character is escaped + */ +function isEscaped(str: string, i: number): boolean { + return i > 0 && str[i - 1] === '\\' && !(i > 1 && str[i - 2] === '\\') +} + +/** + * Extracts JSON objects or arrays from a string. + * + * @param input - string to extract JSON from + * @param jsonStructureType - type of JSON structure to extract + * @returns array of extracted JSON objects or arrays + */ +export function extractJSONFromString( + input: string, + jsonStructureType: 'object' | 'array' +) { + const startChar = jsonStructureType === 'object' ? '{' : '[' + const endChar = jsonStructureType === 'object' ? '}' : ']' + const extractedJSONValues: JsonValue[] = [] + let nestingLevel = 0 + let startIndex = 0 + const isInsideQuoted = { '"': false, "'": false } + + for (let i = 0; i < input.length; i++) { + const ch = input.charAt(i) + switch (ch) { + case '"': + case "'": + if (!isInsideQuoted[ch === '"' ? "'" : '"'] && !isEscaped(input, i)) { + isInsideQuoted[ch] = !isInsideQuoted[ch] + } + + break + + default: + if (!isInsideQuoted['"'] && !isInsideQuoted["'"]) { + switch (ch) { + case startChar: + if (nestingLevel === 0) { + startIndex = i + } + + nestingLevel += 1 + + break + + case endChar: + nestingLevel -= 1 + if (nestingLevel === 0) { + const candidate = input.slice(startIndex, i + 1) + const parsed = JSON.parse(jsonrepair(candidate)) + if (parsed && typeof parsed === 'object') { + extractedJSONValues.push(parsed) + } + } else if (nestingLevel < 0) { + throw new Error( + `Invalid JSON string: unexpected ${endChar} at position ${i}` + ) + } + } + } + } + } + + if (nestingLevel !== 0) { + throw new Error( + 'Invalid JSON string: unmatched ' + startChar + ' or ' + endChar + ) + } + + return extractedJSONValues +} + +const BOOLEAN_OUTPUTS = { + true: true, + false: false, + t: true, + f: false, + yes: true, + no: false, + y: true, + n: false, + '1': true, + '0': false +} + +/** + * Parses an array output from a string. + * + * @param output - string to parse + * @returns parsed array + */ +export function parseArrayOutput(output: string): Array { + try { + const arr = extractJSONFromString(output, 'array') + if (arr.length === 0) { + throw new errors.OutputValidationError(`Invalid JSON array: ${output}`) + } + + const parsedOutput = arr[0] + if (!Array.isArray(parsedOutput)) { + throw new errors.OutputValidationError( + `Invalid JSON array: ${JSON.stringify(parsedOutput)}` + ) + } + + return parsedOutput + } catch (err: any) { + if (err instanceof JSONRepairError) { + throw new errors.OutputValidationError(err.message, { cause: err }) + } else if (err instanceof SyntaxError) { + throw new errors.OutputValidationError( + `Invalid JSON array: ${err.message}`, + { cause: err } + ) + } else { + throw err + } + } +} + +/** + * Parses an object output from a string. + * + * @param output - string to parse + * @returns parsed object + */ +export function parseObjectOutput(output: string) { + try { + const arr = extractJSONFromString(output, 'object') + if (arr.length === 0) { + throw new errors.OutputValidationError(`Invalid JSON object: ${output}`) + } + + let parsedOutput = arr[0] + if (Array.isArray(parsedOutput)) { + // TODO + parsedOutput = parsedOutput[0] + } else if (typeof parsedOutput !== 'object') { + throw new errors.OutputValidationError( + `Invalid JSON object: ${JSON.stringify(parsedOutput)}` + ) + } + + return parsedOutput + } catch (err: any) { + if (err instanceof JSONRepairError) { + throw new errors.OutputValidationError(err.message, { cause: err }) + } else if (err instanceof SyntaxError) { + throw new errors.OutputValidationError( + `Invalid JSON object: ${err.message}`, + { cause: err } + ) + } else { + throw err + } + } +} + +/** + * Parses a boolean output from a string. + * + * @param output - string to parse + * @returns parsed boolean + */ +export function parseBooleanOutput(output: string): boolean { + output = output + .toLowerCase() + .trim() + .replace(/[.!?]+$/, '') + + const booleanOutput = BOOLEAN_OUTPUTS[output] + + if (booleanOutput !== undefined) { + return booleanOutput + } else { + throw new errors.OutputValidationError(`Invalid boolean output: ${output}`) + } +} + +/** + * Parses a number output from a string. + * + * @param output - string to parse + * @param outputSchema - zod number schema + * @returns parsed number + */ +export function parseNumberOutput( + output: string, + outputSchema: z.ZodNumber +): number { + output = output.trim() + + const numberOutput = outputSchema.isInt + ? parseInt(output) + : parseFloat(output) + + if (isNaN(numberOutput)) { + throw new errors.OutputValidationError(`Invalid number output: ${output}`) + } + + return numberOutput +} + +/** + * Parses an output value from a string. + * + * @param output - string to parse + * @param outputSchema - zod schema + * @returns parsed output + */ +export function parseOutput(output: string, outputSchema: ZodType) { + let result + if (outputSchema instanceof z.ZodArray) { + result = parseArrayOutput(output) + } else if (outputSchema instanceof z.ZodObject) { + result = parseObjectOutput(output) + } else if (outputSchema instanceof z.ZodBoolean) { + result = parseBooleanOutput(output) + } else if (outputSchema instanceof z.ZodNumber) { + result = parseNumberOutput(output, outputSchema) + } else { + // Default to string output... + result = output + } + + // TODO: fix typescript issue here with recursive types + const safeResult = (outputSchema.safeParse as any)(result) + + if (!safeResult.success) { + throw new errors.ZodOutputValidationError(safeResult.error) + } + + return safeResult.data +} diff --git a/src/utils.ts b/src/utils.ts index 982a7f6..3e8a2c2 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1,28 +1,9 @@ import { customAlphabet, urlAlphabet } from 'nanoid' import type { ThrottledFunction } from 'p-throttle' +import { JsonValue } from 'type-fest' import * as types from './types' -/** - * Extracts a JSON object string from a given string. - * - * @param text - string from which to extract the JSON object - * @returns extracted JSON object string, or `undefined` if no JSON object is found - */ -export function extractJSONObjectFromString(text: string): string | undefined { - return text.match(/\{(.|\n)*\}/gm)?.[0] // FIXME: This breaks if there are multiple JSON objects in the string -} - -/** - * Extracts a JSON array string from a given string. - * - * @param text - string from which to extract the JSON array - * @returns extracted JSON array string, or `undefined` if no JSON array is found - */ -export function extractJSONArrayFromString(text: string): string | undefined { - return text.match(/\[(.|\n)*\]/gm)?.[0] // FIXME: This breaks if there are multiple JSON arrays in the string -} - /** * Pauses the execution of a function for a specified time. * @@ -132,7 +113,10 @@ export function chunkMultipleStrings( * @param json - JSON value to stringify * @returns stringified value with all double quotes around object keys removed */ -export function stringifyForModel(json: types.Jsonifiable): string { +export function stringifyForModel( + json: types.Jsonifiable, + omit: string[] = [] +): string { const UNIQUE_PREFIX = defaultIDGeneratorFn() return ( JSON.stringify(json, replacer) @@ -143,7 +127,11 @@ export function stringifyForModel(json: types.Jsonifiable): string { /** * Replacer function prefixing all keys with a unique identifier. */ - function replacer(_: string, value: any) { + function replacer(key: string, value: JsonValue) { + if (omit.includes(key)) { + return undefined + } + if (value && typeof value === 'object') { if (Array.isArray(value)) { return value @@ -152,7 +140,7 @@ export function stringifyForModel(json: types.Jsonifiable): string { const replacement = {} for (const k in value) { - if (Object.hasOwnProperty.call(value, k)) { + if (Object.hasOwnProperty.call(value, k) && !omit.includes(k)) { replacement[UNIQUE_PREFIX + k] = value[k] } } diff --git a/test/.snapshots/test/llms/parse-output.test.ts.md b/test/.snapshots/test/llms/parse-output.test.ts.md new file mode 100644 index 0000000..f59dbc4 --- /dev/null +++ b/test/.snapshots/test/llms/parse-output.test.ts.md @@ -0,0 +1,217 @@ +# Snapshot report for `test/llms/parse-output.test.ts` + +The actual snapshot is saved in `parse-output.test.ts.snap`. + +Generated by [AVA](https://avajs.dev). + +## parseArrayOutput - handles valid arrays correctly + +> should return [1, 2, 3] for "[1,2,3]" + + [ + 1, + 2, + 3, + ] + +> should return ["a", "b", "c"] for "["a", "b", "c"] + + [ + 'a', + 'b', + 'c', + ] + +> should return [{"a": 1}, {"b": 2}] for [{"a": 1}, {"b": 2}] + + [ + { + a: 1, + }, + { + b: 2, + }, + ] + +## parseArrayOutput - handles arrays surrounded by text correctly + +> should return [1, 2, 3] for "The array is [1,2,3]" + + [ + 1, + 2, + 3, + ] + +> should return ["a", "b", "c"] for "Array: ["a", "b", "c"]. That's all!" + + [ + 'a', + 'b', + 'c', + ] + +> should return [{"a": 1}, {"b": 2}] for "This is the array [{"a": 1}, {"b": 2}] in the text" + + [ + { + a: 1, + }, + { + b: 2, + }, + ] + +## parseArrayOutput - throws error for invalid arrays + +> Snapshot 1 + + 'Invalid JSON array: not a valid array' + +## parseObjectOutput - handles valid objects correctly + +> should return {"a":1,"b":2,"c":3} for {"a":1,"b":2,"c":3} + + { + a: 1, + b: 2, + c: 3, + } + +> should return {"name":"John","age":30,"city":"New York"} for {"name":"John","age":30,"city":"New York"} + + { + age: 30, + city: 'New York', + name: 'John', + } + +## parseObjectOutput - handles objects surrounded by text correctly + +> should return {"a":1,"b":2,"c":3} for "The object is {"a":1,"b":2,"c":3}" + + { + a: 1, + b: 2, + c: 3, + } + +> should return {"name":"John","age":30,"city":"New York"} for "Object: {"name":"John","age":30,"city":"New York"}. That's all!" + + { + age: 30, + city: 'New York', + name: 'John', + } + +## parseObjectOutput - handles JSON array of objects + +> should return first object {"a":1,"b":2} for [{"a":1,"b":2},{"c":3,"d":4}] + + { + a: 1, + b: 2, + } + +## parseObjectOutput - throws error for invalid objects + +> Snapshot 1 + + 'Invalid JSON object: not a valid object' + +## parseBooleanOutput - handles `true` outputs correctly + +> should return true for "True" + + true + +> should return true for "TRUE" + + true + +> should return true for "true." + + true + +## parseBooleanOutput - handles `false` outputs correctly + +> should return false for "False" + + false + +> should return false for "FALSE" + + false + +> should return false for "false!" + + false + +## parseBooleanOutput - throws error for invalid outputs + +> Snapshot 1 + + 'Invalid boolean output: notbooleanvalue' + +## parseNumberOutput - handles integer outputs correctly + +> should return 42 for "42" + + 42 + +> should return -5 for " -5 " + + -5 + +## parseNumberOutput - handles float outputs correctly + +> should return 42.42 for "42.42" + + 42.42 + +> should return -5.5 for " -5.5 " + + -5.5 + +## parseNumberOutput - throws error for invalid outputs + +> Snapshot 1 + + 'Invalid number output: NotANumber' + +## parseOutput - handles arrays correctly + +> should parse and return [1, 2, 3] for "[1, 2, 3]" + + [ + 1, + 2, + 3, + ] + +## parseOutput - handles objects correctly + +> should parse and return {"a": 1, "b": "two"} for "{"a": 1, "b": "two"}" + + { + a: 1, + b: 'two', + } + +## parseOutput - handles booleans correctly + +> should parse and return true for "True" + + true + +## parseOutput - handles numbers correctly + +> should parse and return 123.45 for "123.45" + + 123.45 + +## parseOutput - throws error for invalid data + +> Snapshot 1 + + 'Invalid number output: not a number' diff --git a/test/.snapshots/test/llms/parse-output.test.ts.snap b/test/.snapshots/test/llms/parse-output.test.ts.snap new file mode 100644 index 0000000..5832b61 Binary files /dev/null and b/test/.snapshots/test/llms/parse-output.test.ts.snap differ diff --git a/test/llms/parse-output.test.ts b/test/llms/parse-output.test.ts new file mode 100644 index 0000000..85ec085 --- /dev/null +++ b/test/llms/parse-output.test.ts @@ -0,0 +1,301 @@ +import test from 'ava' +import { z } from 'zod' + +import { + extractJSONFromString, + parseArrayOutput, + parseBooleanOutput, + parseNumberOutput, + parseObjectOutput, + parseOutput +} from '@/llms/parse-output' + +test('extractJSONFromString should extract JSON object from string', (t) => { + let jsonStr = 'Some text {"name":"John Doe"} more text' + let result = extractJSONFromString(jsonStr, 'object') + t.deepEqual(result[0], { name: 'John Doe' }) + + jsonStr = + 'Some text {"name":"John Doe","age":42,"address":{"street":"Main Street","number":42}} more text' + result = extractJSONFromString(jsonStr, 'object') + t.deepEqual(result[0], { + name: 'John Doe', + age: 42, + address: { street: 'Main Street', number: 42 } + }) + + jsonStr = 'foo {"name":"John Doe","school":"St. John\'s"} bar' + result = extractJSONFromString(jsonStr, 'object') + t.deepEqual(result[0], { name: 'John Doe', school: "St. John's" }) +}) + +test('extractJSONFromString should extract an invalid JSON object from string', (t) => { + let jsonStr = 'Some text {"name":\'John Doe\'} more text' + let result = extractJSONFromString(jsonStr, 'object') + t.deepEqual(result[0], { name: 'John Doe' }) + + jsonStr = 'Some text {"name":"John Doe","age":42,} more text' + result = extractJSONFromString(jsonStr, 'object') + t.deepEqual(result[0], { name: 'John Doe', age: 42 }) +}) + +test('extractJSONFromString should extract multiple JSON objects from string', (t) => { + let jsonStr = 'Some text {"name":"John Doe"} more text {"name":"Jane Doe"}' + let result = extractJSONFromString(jsonStr, 'object') + t.deepEqual(result[0], { name: 'John Doe' }) + t.deepEqual(result[1], { name: 'Jane Doe' }) + + jsonStr = + 'Some text {"name":"John Doe","age":42,"address":{"street":"Main Street","number":42}} more text {"name":"Jane Doe","age":42,"address":{"street":"Main Street","number":42}}' + result = extractJSONFromString(jsonStr, 'object') + t.deepEqual(result[0], { + name: 'John Doe', + age: 42, + address: { street: 'Main Street', number: 42 } + }) + t.deepEqual(result[1], { + name: 'Jane Doe', + age: 42, + address: { street: 'Main Street', number: 42 } + }) +}) + +test('extractJSONFromString should extract JSON array from string', (t) => { + let jsonString = 'Some text [1,2,3] more text' + let result = extractJSONFromString(jsonString, 'array') + t.deepEqual(result[0], [1, 2, 3]) + + jsonString = 'Some text ["foo","bar","\'quoted\'"] more text' + result = extractJSONFromString(jsonString, 'array') + t.deepEqual(result[0], ['foo', 'bar', "'quoted'"]) +}) + +test('extractJSONFromString should extract an invalid JSON array from string', (t) => { + let jsonString = 'Some text [1,2,3,] more text' + let result = extractJSONFromString(jsonString, 'array') + t.deepEqual(result[0], [1, 2, 3]) + + jsonString = "Some text ['foo','bar'] more text" + result = extractJSONFromString(jsonString, 'array') + t.deepEqual(result[0], ['foo', 'bar']) +}) + +test('extractJSONFromString should extract multiple JSON arrays from string', (t) => { + const jsonString = 'Some text [1,2,3] more text [4,5,6]' + const result = extractJSONFromString(jsonString, 'array') + t.deepEqual(result[0], [1, 2, 3]) + t.deepEqual(result[1], [4, 5, 6]) +}) + +test('extractJSONFromString should return an empty array if no JSON object is found', (t) => { + const jsonString = 'Some text' + const result = extractJSONFromString(jsonString, 'object') + t.deepEqual(result, []) +}) + +test('extractJSONFromString should return an empty array if no JSON array is found', (t) => { + const jsonString = 'Some text' + const result = extractJSONFromString(jsonString, 'array') + t.deepEqual(result, []) +}) + +test('parseArrayOutput - handles valid arrays correctly', (t) => { + const output1 = parseArrayOutput('[1,2,3]') + const output2 = parseArrayOutput('["a", "b", "c"]') + const output3 = parseArrayOutput('[{"a": 1}, {"b": 2}]') + + t.snapshot(output1, 'should return [1, 2, 3] for "[1,2,3]"') + t.snapshot(output2, 'should return ["a", "b", "c"] for "["a", "b", "c"]') + t.snapshot( + output3, + 'should return [{"a": 1}, {"b": 2}] for [{"a": 1}, {"b": 2}]' + ) +}) + +test('parseArrayOutput - handles arrays surrounded by text correctly', (t) => { + const output1 = parseArrayOutput('The array is [1,2,3]') + const output2 = parseArrayOutput('Array: ["a", "b", "c"]. That\'s all!') + const output3 = parseArrayOutput( + 'This is the array [{"a": 1}, {"b": 2}] in the text' + ) + + t.snapshot(output1, 'should return [1, 2, 3] for "The array is [1,2,3]"') + t.snapshot( + output2, + 'should return ["a", "b", "c"] for "Array: ["a", "b", "c"]. That\'s all!"' + ) + t.snapshot( + output3, + 'should return [{"a": 1}, {"b": 2}] for "This is the array [{"a": 1}, {"b": 2}] in the text"' + ) +}) + +test('parseArrayOutput - throws error for invalid arrays', (t) => { + const error = t.throws( + () => { + parseArrayOutput('not a valid array') + }, + { instanceOf: Error } + ) + + t.snapshot(error?.message) +}) + +test('parseObjectOutput - handles valid objects correctly', (t) => { + const output1 = parseObjectOutput('{"a":1,"b":2,"c":3}') + const output2 = parseObjectOutput( + '{"name":"John","age":30,"city":"New York"}' + ) + + t.snapshot( + output1, + 'should return {"a":1,"b":2,"c":3} for {"a":1,"b":2,"c":3}' + ) + t.snapshot( + output2, + 'should return {"name":"John","age":30,"city":"New York"} for {"name":"John","age":30,"city":"New York"}' + ) +}) + +test('parseObjectOutput - handles objects surrounded by text correctly', (t) => { + const output1 = parseObjectOutput('The object is {"a":1,"b":2,"c":3}') + const output2 = parseObjectOutput( + 'Object: {"name":"John","age":30,"city":"New York"}. That\'s all!' + ) + + t.snapshot( + output1, + 'should return {"a":1,"b":2,"c":3} for "The object is {"a":1,"b":2,"c":3}"' + ) + t.snapshot( + output2, + 'should return {"name":"John","age":30,"city":"New York"} for "Object: {"name":"John","age":30,"city":"New York"}. That\'s all!"' + ) +}) + +test('parseObjectOutput - handles JSON array of objects', (t) => { + const output = parseObjectOutput('[{"a":1,"b":2},{"c":3,"d":4}]') + + t.snapshot( + output, + 'should return first object {"a":1,"b":2} for [{"a":1,"b":2},{"c":3,"d":4}]' + ) +}) + +test('parseObjectOutput - throws error for invalid objects', (t) => { + const error = t.throws( + () => { + parseObjectOutput('not a valid object') + }, + { instanceOf: Error } + ) + + t.snapshot(error?.message) +}) + +test('parseBooleanOutput - handles `true` outputs correctly', (t) => { + const output1 = parseBooleanOutput('True') + const output2 = parseBooleanOutput('TRUE') + const output3 = parseBooleanOutput('true.') + + t.snapshot(output1, 'should return true for "True"') + t.snapshot(output2, 'should return true for "TRUE"') + t.snapshot(output3, 'should return true for "true."') +}) + +test('parseBooleanOutput - handles `false` outputs correctly', (t) => { + const output1 = parseBooleanOutput('False') + const output2 = parseBooleanOutput('FALSE') + const output3 = parseBooleanOutput('false!') + + t.snapshot(output1, 'should return false for "False"') + t.snapshot(output2, 'should return false for "FALSE"') + t.snapshot(output3, 'should return false for "false!"') +}) + +test('parseBooleanOutput - throws error for invalid outputs', (t) => { + const error = t.throws( + () => { + parseBooleanOutput('NotBooleanValue') + }, + { instanceOf: Error } + ) + + t.snapshot(error?.message) +}) + +test('parseNumberOutput - handles integer outputs correctly', (t) => { + const output1 = parseNumberOutput('42', z.number().int()) + const output2 = parseNumberOutput(' -5 ', z.number().int()) + + t.snapshot(output1, 'should return 42 for "42"') + t.snapshot(output2, 'should return -5 for " -5 "') +}) + +test('parseNumberOutput - handles float outputs correctly', (t) => { + const output1 = parseNumberOutput('42.42', z.number()) + const output2 = parseNumberOutput(' -5.5 ', z.number()) + + t.snapshot(output1, 'should return 42.42 for "42.42"') + t.snapshot(output2, 'should return -5.5 for " -5.5 "') +}) + +test('parseNumberOutput - throws error for invalid outputs', (t) => { + const error = t.throws( + () => { + parseNumberOutput('NotANumber', z.number()) + }, + { instanceOf: Error } + ) + + t.snapshot(error?.message) +}) + +test('parseOutput - handles arrays correctly', (t) => { + const arraySchema = z.array(z.number()) + const output = '[1, 2, 3]' + const result = parseOutput(output, arraySchema) + + t.snapshot(result, 'should parse and return [1, 2, 3] for "[1, 2, 3]"') +}) + +test('parseOutput - handles objects correctly', (t) => { + const objectSchema = z.object({ a: z.number(), b: z.string() }) + const output = '{"a": 1, "b": "two"}' + const result = parseOutput(output, objectSchema) + + t.snapshot( + result, + 'should parse and return {"a": 1, "b": "two"} for "{"a": 1, "b": "two"}"' + ) +}) + +test('parseOutput - handles booleans correctly', (t) => { + const booleanSchema = z.boolean() + const output = 'True' + const result = parseOutput(output, booleanSchema) + + t.snapshot(result, 'should parse and return true for "True"') +}) + +test('parseOutput - handles numbers correctly', (t) => { + const numberSchema = z.number() + const output = '123.45' + const result = parseOutput(output, numberSchema) + + t.snapshot(result, 'should parse and return 123.45 for "123.45"') +}) + +test('parseOutput - throws error for invalid data', (t) => { + const numberSchema = z.number() + const output = 'not a number' + + const error = t.throws( + () => { + parseOutput(output, numberSchema) + }, + { instanceOf: Error } + ) + + t.snapshot(error?.message) +}) diff --git a/test/utils.test.ts b/test/utils.test.ts index 0e4804e..59079f3 100644 --- a/test/utils.test.ts +++ b/test/utils.test.ts @@ -6,8 +6,6 @@ import { chunkString, defaultIDGeneratorFn, extractFunctionIdentifierFromString, - extractJSONArrayFromString, - extractJSONObjectFromString, isValidTaskIdentifier, sleep, stringifyForModel, @@ -33,30 +31,6 @@ test('isValidTaskIdentifier - invalid', async (t) => { t.false(isValidTaskIdentifier('-foo')) }) -test('extractJSONObjectFromString should extract JSON object from string', (t) => { - const jsonString = 'Some text {"name":"John Doe"} more text' - const result = extractJSONObjectFromString(jsonString) - t.is(result, '{"name":"John Doe"}') -}) - -test('extractJSONArrayFromString should extract JSON array from string', (t) => { - const jsonString = 'Some text [1,2,3] more text' - const result = extractJSONArrayFromString(jsonString) - t.is(result, '[1,2,3]') -}) - -test('extractJSONObjectFromString should return undefined if no JSON object is found', (t) => { - const jsonString = 'Some text' - const result = extractJSONObjectFromString(jsonString) - t.is(result, undefined) -}) - -test('extractJSONArrayFromString should return undefined if no JSON array is found', (t) => { - const jsonString = 'Some text' - const result = extractJSONArrayFromString(jsonString) - t.is(result, undefined) -}) - test('sleep should delay execution', async (t) => { const start = Date.now() await sleep(1000) // for example, 1000ms / 1sec