From 28276d19951a7eee55ed35e69a958dc8b093b487 Mon Sep 17 00:00:00 2001 From: Travis Fischer Date: Thu, 12 Jun 2025 03:48:11 +0700 Subject: [PATCH] =?UTF-8?q?=F0=9F=98=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/__snapshots__/http-e2e.test.ts.snap | 18 +---- .../src/__snapshots__/mcp-e2e.test.ts.snap | 26 ++++++ apps/e2e/src/http-fixtures.ts | 45 +++++------ apps/e2e/src/mcp-e2e.test.ts | 21 +++-- apps/e2e/src/mcp-fixtures.ts | 81 ++++++++++++++++++- 5 files changed, 142 insertions(+), 49 deletions(-) diff --git a/apps/e2e/src/__snapshots__/http-e2e.test.ts.snap b/apps/e2e/src/__snapshots__/http-e2e.test.ts.snap index 60fa2ae8..3bc30d5c 100644 --- a/apps/e2e/src/__snapshots__/http-e2e.test.ts.snap +++ b/apps/e2e/src/__snapshots__/http-e2e.test.ts.snap @@ -54,7 +54,7 @@ exports[`Basic MCP origin "add" tool call success > 5.1: GET @dev/test-basic-mcp ] `; -exports[`Basic OpenAPI getPost errors > 1.9: POST @dev/test-basic-openapi/getPost 1`] = ` +exports[`Basic OpenAPI getPost errors > 1.8: POST @dev/test-basic-openapi/getPost 1`] = ` { "body": "quia et suscipit suscipit recusandae consequuntur expedita et cum @@ -66,7 +66,7 @@ nostrum rerum est autem sunt rem eveniet architecto", } `; -exports[`Basic OpenAPI getPost errors > 1.10: GET @dev/test-basic-openapi/getPost 1`] = ` +exports[`Basic OpenAPI getPost errors > 1.9: GET @dev/test-basic-openapi/getPost 1`] = ` { "body": "quia et suscipit suscipit recusandae consequuntur expedita et cum @@ -263,17 +263,3 @@ exports[`OpenAPI kitchen sink pure tool > 7.1: POST @dev/test-everything-openapi "nala": "kitten", } `; - -exports[`OpenAPI kitchen sink unpure_marked_pure tool > 10.0: POST @dev/test-everything-openapi/unpure_marked_pure 1`] = ` -{ - "nala": "cat", - "now": 1749631781793, -} -`; - -exports[`OpenAPI kitchen sink unpure_marked_pure tool > 10.1: POST @dev/test-everything-openapi/unpure_marked_pure 1`] = ` -{ - "nala": "cat", - "now": 1749631781793, -} -`; diff --git a/apps/e2e/src/__snapshots__/mcp-e2e.test.ts.snap b/apps/e2e/src/__snapshots__/mcp-e2e.test.ts.snap index cde27015..c2296126 100644 --- a/apps/e2e/src/__snapshots__/mcp-e2e.test.ts.snap +++ b/apps/e2e/src/__snapshots__/mcp-e2e.test.ts.snap @@ -32,6 +32,32 @@ molestiae porro eius odio et labore et velit aut", } `; +exports[`Basic MCP => OpenAPI everything errors > 5.0: @dev/test-everything-openapi/mcp strict_additional_properties 1`] = ` +{ + "content": [], + "isError": false, + "structuredContent": { + "foo": "bar", + }, +} +`; + +exports[`Basic MCP => OpenAPI get_post errors > 4.3: @dev/test-basic-openapi/mcp get_post 1`] = ` +{ + "content": [], + "isError": false, + "structuredContent": { + "body": "dolore placeat quibusdam ea quo vitae +magni quis enim qui quis quo nemo aut saepe +quidem repellat excepturi ut quia +sunt ut sequi eos ea sed quas", + "id": 7, + "title": "magnam facilis autem", + "userId": 1, + }, +} +`; + exports[`Basic MCP => OpenAPI get_post success > 0.0: @dev/test-basic-openapi/mcp get_post 1`] = ` { "content": [], diff --git a/apps/e2e/src/http-fixtures.ts b/apps/e2e/src/http-fixtures.ts index 57289587..b778055a 100644 --- a/apps/e2e/src/http-fixtures.ts +++ b/apps/e2e/src/http-fixtures.ts @@ -111,6 +111,9 @@ export const fixtureSuites: E2ETestFixtureSuite[] = [ fixtures: [ { path: '@dev/test-basic-openapi/getPost', + request: { + method: 'GET' + }, response: { // Missing `postId` parameter. status: 400 @@ -125,6 +128,7 @@ export const fixtureSuites: E2ETestFixtureSuite[] = [ { path: '@dev/test-basic-openapi@00000000/getPost', response: { + // deployment hash 00000000 doesn't exist status: 404 } }, @@ -137,6 +141,7 @@ export const fixtureSuites: E2ETestFixtureSuite[] = [ } }, response: { + // PUT is not a valid method (must be POST) status: 405 } }, @@ -158,6 +163,19 @@ export const fixtureSuites: E2ETestFixtureSuite[] = [ status: 400 } }, + { + path: '@dev/test-basic-openapi/getPost', + request: { + method: 'POST', + json: { + // invalid `postId` field type + postId: 'not-a-number' + } + }, + response: { + status: 400 + } + }, { path: '@dev/test-basic-openapi/getPost', request: { @@ -170,32 +188,6 @@ export const fixtureSuites: E2ETestFixtureSuite[] = [ status: 400 } }, - { - path: '@dev/test-basic-openapi/getPost', - request: { - method: 'POST', - json: { - // invalid `postId` field type - postId: 'foo' - } - }, - response: { - status: 400 - } - }, - { - path: '@dev/test-basic-openapi/getPost', - request: { - method: 'POST', - json: { - // invalid `postId` field type - postId: 'foo' - } - }, - response: { - status: 400 - } - }, { path: '@dev/test-basic-openapi/getPost', request: { @@ -598,6 +590,7 @@ export const fixtureSuites: E2ETestFixtureSuite[] = [ { title: 'OpenAPI kitchen sink unpure_marked_pure tool', compareResponseBodies: true, + snapshot: false, fixtures: [ { path: '@dev/test-everything-openapi/unpure_marked_pure', diff --git a/apps/e2e/src/mcp-e2e.test.ts b/apps/e2e/src/mcp-e2e.test.ts index 447b9bd4..e69a16e4 100644 --- a/apps/e2e/src/mcp-e2e.test.ts +++ b/apps/e2e/src/mcp-e2e.test.ts @@ -24,6 +24,18 @@ for (const [i, fixtureSuite] of fixtureSuites.entries()) { new URL(fixtureSuite.path, env.AGENTIC_GATEWAY_BASE_URL) ) await client.connect(transport) + + const { tools } = await client.listTools() + + // Ensure all tools used by the test fixtures in this suite are available. + // Ignore test fixtures which are expected to error. + for (const [_, fixture] of fixtures.entries()) { + const { isError } = fixture.response ?? {} + if (!isError) { + const toolName = fixture.request.name + expect(tools.map((t) => t.name)).toContain(toolName) + } + } }, 120_000) afterAll(async () => { @@ -41,6 +53,7 @@ for (const [i, fixtureSuite] of fixtureSuites.entries()) { _agenticMetaHeaders: expectedAgenticMetaHeaders, validate } = fixture.response ?? {} + const toolName = fixture.request.name const expectedSnapshot = fixture.response?.snapshot ?? fixtureSuite.snapshot ?? false const expectedStableSnapshot = @@ -53,7 +66,7 @@ for (const [i, fixtureSuite] of fixtureSuites.entries()) { fixture.only ?? fixtureSuite.only ) - const fixtureName = `${i}.${j}: ${fixtureSuite.path} ${fixture.request.name}` + const fixtureName = `${i}.${j}: ${fixtureSuite.path} ${toolName}` let testFn = (fixture.only ?? fixture.debug) ? test.only : test if (fixtureSuite.sequential) { @@ -67,12 +80,8 @@ for (const [i, fixtureSuite] of fixtureSuites.entries()) { }, // eslint-disable-next-line no-loop-func async () => { - const { tools } = await client.listTools() - // console.log('tools', tools) - expect(tools.map((t) => t.name)).toContain(fixture.request.name) - const result = await client.callTool({ - name: fixture.request.name, + name: toolName, arguments: fixture.request.args }) diff --git a/apps/e2e/src/mcp-fixtures.ts b/apps/e2e/src/mcp-fixtures.ts index 31f07a0e..10f04d63 100644 --- a/apps/e2e/src/mcp-fixtures.ts +++ b/apps/e2e/src/mcp-fixtures.ts @@ -152,13 +152,14 @@ export const fixtureSuites: MCPE2ETestFixtureSuite[] = [ ] }, { - title: 'Basic MCP => MCP "echo" tool call errors', + title: 'Basic MCP => OpenAPI get_post errors', path: '@dev/test-basic-openapi/mcp', fixtures: [ { request: { name: 'get_post', args: { + // Missing required `postId` parameter nala: 'kitten', num: 123, now @@ -170,6 +171,84 @@ export const fixtureSuites: MCPE2ETestFixtureSuite[] = [ status: 400 } } + }, + { + request: { + name: 'get_post', + args: { + // invalid `postId` parameter + postId: 'not-a-number' + } + }, + response: { + isError: true, + _agenticMeta: { + status: 400 + } + } + }, + { + request: { + name: 'get_kittens', + args: { + postId: 7 + } + }, + response: { + isError: true, + _agenticMeta: { + // 'get_kittens' tool doesn't exist + status: 404, + toolName: 'get_kittens' + } + } + }, + { + request: { + name: 'get_post', + args: { + postId: 7, + // additional json body params are allowed by default + foo: 'bar' + } + }, + response: { + isError: false + } + } + ] + }, + { + title: 'Basic MCP => OpenAPI everything errors', + path: '@dev/test-everything-openapi/mcp', + fixtures: [ + { + request: { + name: 'strict_additional_properties', + args: { + foo: 'bar' + } + }, + response: { + isError: false + } + }, + { + request: { + name: 'strict_additional_properties', + args: { + foo: 'bar', + // additional params should throw an error if the tool + // config has `additionalProperties: false` + extra: 'nala' + } + }, + response: { + isError: true, + _agenticMeta: { + status: 400 + } + } } ] }