kopia lustrzana https://github.com/transitive-bullshit/chatgpt-api
feat: add caching for mcp origin tool calls
rodzic
fb1f67a685
commit
f18ba2ba8c
|
@ -197,3 +197,15 @@ et est aut quod aut provident voluptas autem voluptas",
|
||||||
"userId": 1,
|
"userId": 1,
|
||||||
}
|
}
|
||||||
`;
|
`;
|
||||||
|
|
||||||
|
exports[`Bypass caching > 2.4: GET @dev/test-basic-openapi@fc856666/get_post?postId=9 1`] = `
|
||||||
|
{
|
||||||
|
"body": "consectetur animi nesciunt iure dolore
|
||||||
|
enim quia ad
|
||||||
|
veniam autem ut quam aut nobis
|
||||||
|
et est aut quod aut provident voluptas autem voluptas",
|
||||||
|
"id": 9,
|
||||||
|
"title": "nesciunt iure omnis dolorem tempora et accusantium",
|
||||||
|
"userId": 1,
|
||||||
|
}
|
||||||
|
`;
|
||||||
|
|
|
@ -246,7 +246,7 @@ export const fixtureSuites: E2ETestFixtureSuite[] = [
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
// ensure we bypass the cache for requests with `pragma: no-cache`
|
// ensure we bypass the cache for requests with `cache-control: no-cache`
|
||||||
path: '@dev/test-basic-openapi@fc856666/getPost?postId=9',
|
path: '@dev/test-basic-openapi@fc856666/getPost?postId=9',
|
||||||
request: {
|
request: {
|
||||||
headers: {
|
headers: {
|
||||||
|
@ -285,6 +285,19 @@ export const fixtureSuites: E2ETestFixtureSuite[] = [
|
||||||
'cf-cache-status': 'BYPASS'
|
'cf-cache-status': 'BYPASS'
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
path: '@dev/test-basic-openapi@fc856666/get_post?postId=9',
|
||||||
|
request: {
|
||||||
|
headers: {
|
||||||
|
'cache-control': 'private, max-age=3600, must-revalidate'
|
||||||
|
}
|
||||||
|
},
|
||||||
|
response: {
|
||||||
|
headers: {
|
||||||
|
'cf-cache-status': 'BYPASS'
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
|
@ -297,6 +310,10 @@ export const fixtureSuites: E2ETestFixtureSuite[] = [
|
||||||
// first request to ensure the cache is populated
|
// first request to ensure the cache is populated
|
||||||
path: '@dev/test-basic-openapi@fc856666/getPost',
|
path: '@dev/test-basic-openapi@fc856666/getPost',
|
||||||
request: {
|
request: {
|
||||||
|
headers: {
|
||||||
|
'cache-control':
|
||||||
|
'public, max-age=3600, s-maxage=3600, stale-while-revalidate=3600'
|
||||||
|
},
|
||||||
searchParams: {
|
searchParams: {
|
||||||
postId: 9
|
postId: 9
|
||||||
}
|
}
|
||||||
|
@ -305,6 +322,12 @@ export const fixtureSuites: E2ETestFixtureSuite[] = [
|
||||||
{
|
{
|
||||||
// second request should hit the cache
|
// second request should hit the cache
|
||||||
path: '@dev/test-basic-openapi@fc856666/getPost?postId=9',
|
path: '@dev/test-basic-openapi@fc856666/getPost?postId=9',
|
||||||
|
request: {
|
||||||
|
headers: {
|
||||||
|
'cache-control':
|
||||||
|
'public, max-age=3600, s-maxage=3600, stale-while-revalidate=3600'
|
||||||
|
}
|
||||||
|
},
|
||||||
response: {
|
response: {
|
||||||
headers: {
|
headers: {
|
||||||
'cf-cache-status': 'HIT'
|
'cf-cache-status': 'HIT'
|
||||||
|
@ -312,8 +335,14 @@ export const fixtureSuites: E2ETestFixtureSuite[] = [
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
// normalized request should also hit the cache
|
// normalized request with different path should also hit the cache
|
||||||
path: '@dev/test-basic-openapi@fc856666/get_post?postId=9',
|
path: '@dev/test-basic-openapi@fc856666/get_post?postId=9',
|
||||||
|
request: {
|
||||||
|
headers: {
|
||||||
|
'cache-control':
|
||||||
|
'public, max-age=3600, s-maxage=3600, stale-while-revalidate=3600'
|
||||||
|
}
|
||||||
|
},
|
||||||
response: {
|
response: {
|
||||||
headers: {
|
headers: {
|
||||||
'cf-cache-status': 'HIT'
|
'cf-cache-status': 'HIT'
|
||||||
|
@ -331,6 +360,9 @@ export const fixtureSuites: E2ETestFixtureSuite[] = [
|
||||||
// first request to ensure the cache is populated
|
// first request to ensure the cache is populated
|
||||||
path: '@dev/test-basic-openapi@fc856666/get_post',
|
path: '@dev/test-basic-openapi@fc856666/get_post',
|
||||||
request: {
|
request: {
|
||||||
|
headers: {
|
||||||
|
'cache-control': 'public, s-max-age=3600'
|
||||||
|
},
|
||||||
method: 'POST',
|
method: 'POST',
|
||||||
json: {
|
json: {
|
||||||
postId: 13
|
postId: 13
|
||||||
|
@ -342,6 +374,9 @@ export const fixtureSuites: E2ETestFixtureSuite[] = [
|
||||||
path: '@dev/test-basic-openapi@fc856666/get_post',
|
path: '@dev/test-basic-openapi@fc856666/get_post',
|
||||||
request: {
|
request: {
|
||||||
method: 'POST',
|
method: 'POST',
|
||||||
|
headers: {
|
||||||
|
'cache-control': 'public, s-max-age=3600'
|
||||||
|
},
|
||||||
json: {
|
json: {
|
||||||
postId: 13
|
postId: 13
|
||||||
}
|
}
|
||||||
|
|
|
@ -53,7 +53,7 @@ for (const [i, fixtureSuite] of fixtureSuites.entries()) {
|
||||||
// eslint-disable-next-line no-loop-func
|
// eslint-disable-next-line no-loop-func
|
||||||
async () => {
|
async () => {
|
||||||
const { tools } = await client.listTools()
|
const { tools } = await client.listTools()
|
||||||
console.log('tools', tools)
|
// console.log('tools', tools)
|
||||||
expect(tools.map((t) => t.name)).toContain(fixture.request.name)
|
expect(tools.map((t) => t.name)).toContain(fixture.request.name)
|
||||||
|
|
||||||
const result = await client.callTool({
|
const result = await client.callTool({
|
||||||
|
|
|
@ -76,6 +76,7 @@ app.all(async (ctx) => {
|
||||||
deployment: resolvedEdgeRequest.deployment,
|
deployment: resolvedEdgeRequest.deployment,
|
||||||
consumer: resolvedEdgeRequest.consumer,
|
consumer: resolvedEdgeRequest.consumer,
|
||||||
pricingPlan: resolvedEdgeRequest.pricingPlan,
|
pricingPlan: resolvedEdgeRequest.pricingPlan,
|
||||||
|
cacheControl: resolvedEdgeRequest.cacheControl,
|
||||||
sessionId: ctx.get('sessionId')!,
|
sessionId: ctx.get('sessionId')!,
|
||||||
ip: ctx.get('ip'),
|
ip: ctx.get('ip'),
|
||||||
env: ctx.env,
|
env: ctx.env,
|
||||||
|
|
|
@ -1,6 +1,7 @@
|
||||||
import { hashObject, sha256 } from '@agentic/platform-core'
|
import { hashObject, sha256 } from '@agentic/platform-core'
|
||||||
import contentType from 'fast-content-type-parse'
|
import contentType from 'fast-content-type-parse'
|
||||||
|
|
||||||
|
import { isCacheControlPubliclyCacheable } from './is-cache-control-publicly-cacheable'
|
||||||
import { normalizeUrl } from './normalize-url'
|
import { normalizeUrl } from './normalize-url'
|
||||||
|
|
||||||
// TODO: what is a reasonable upper bound for hashing the POST body size?
|
// TODO: what is a reasonable upper bound for hashing the POST body size?
|
||||||
|
@ -16,11 +17,8 @@ export async function getRequestCacheKey(
|
||||||
}
|
}
|
||||||
|
|
||||||
const cacheControl = request.headers.get('cache-control')
|
const cacheControl = request.headers.get('cache-control')
|
||||||
if (cacheControl) {
|
if (!isCacheControlPubliclyCacheable(cacheControl)) {
|
||||||
const directives = new Set(cacheControl.split(',').map((s) => s.trim()))
|
return
|
||||||
if (directives.has('no-store') || directives.has('no-cache')) {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (request.method === 'POST' || request.method === 'PUT') {
|
if (request.method === 'POST' || request.method === 'PUT') {
|
||||||
|
|
|
@ -0,0 +1,19 @@
|
||||||
|
export function isCacheControlPubliclyCacheable(
|
||||||
|
cacheControl?: string | null
|
||||||
|
): boolean {
|
||||||
|
if (!cacheControl) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
const directives = new Set(cacheControl.split(',').map((s) => s.trim()))
|
||||||
|
if (
|
||||||
|
directives.has('no-store') ||
|
||||||
|
directives.has('no-cache') ||
|
||||||
|
directives.has('private') ||
|
||||||
|
!directives.has('public')
|
||||||
|
) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
return true
|
||||||
|
}
|
|
@ -19,6 +19,7 @@ export type ResolvedHttpEdgeRequest = {
|
||||||
|
|
||||||
tool: Tool
|
tool: Tool
|
||||||
toolCallArgs: ToolCallArgs
|
toolCallArgs: ToolCallArgs
|
||||||
|
cacheControl?: string
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -34,6 +35,7 @@ export async function resolveHttpEdgeRequest(
|
||||||
const logger = ctx.get('logger')
|
const logger = ctx.get('logger')
|
||||||
const ip = ctx.get('ip')
|
const ip = ctx.get('ip')
|
||||||
|
|
||||||
|
const cacheControl = ctx.req.header('cache-control')
|
||||||
const { method } = ctx.req
|
const { method } = ctx.req
|
||||||
const requestUrl = new URL(ctx.req.url)
|
const requestUrl = new URL(ctx.req.url)
|
||||||
const { pathname } = requestUrl
|
const { pathname } = requestUrl
|
||||||
|
@ -115,6 +117,7 @@ export async function resolveHttpEdgeRequest(
|
||||||
consumer,
|
consumer,
|
||||||
pricingPlan,
|
pricingPlan,
|
||||||
tool,
|
tool,
|
||||||
toolCallArgs
|
toolCallArgs,
|
||||||
|
cacheControl
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -19,6 +19,7 @@ import { createHttpRequestForOpenAPIOperation } from './create-http-request-for-
|
||||||
import { enforceRateLimit } from './enforce-rate-limit'
|
import { enforceRateLimit } from './enforce-rate-limit'
|
||||||
import { fetchCache } from './fetch-cache'
|
import { fetchCache } from './fetch-cache'
|
||||||
import { getRequestCacheKey } from './get-request-cache-key'
|
import { getRequestCacheKey } from './get-request-cache-key'
|
||||||
|
import { isCacheControlPubliclyCacheable } from './is-cache-control-publicly-cacheable'
|
||||||
import { updateOriginRequest } from './update-origin-request'
|
import { updateOriginRequest } from './update-origin-request'
|
||||||
|
|
||||||
export type ResolvedOriginToolCallResult = {
|
export type ResolvedOriginToolCallResult = {
|
||||||
|
@ -48,6 +49,7 @@ export async function resolveOriginToolCall({
|
||||||
sessionId,
|
sessionId,
|
||||||
env,
|
env,
|
||||||
ip,
|
ip,
|
||||||
|
cacheControl,
|
||||||
waitUntil
|
waitUntil
|
||||||
}: {
|
}: {
|
||||||
tool: Tool
|
tool: Tool
|
||||||
|
@ -58,6 +60,7 @@ export async function resolveOriginToolCall({
|
||||||
sessionId: string
|
sessionId: string
|
||||||
env: RawEnv
|
env: RawEnv
|
||||||
ip?: string
|
ip?: string
|
||||||
|
cacheControl?: string
|
||||||
waitUntil: (promise: Promise<any>) => void
|
waitUntil: (promise: Promise<any>) => void
|
||||||
}): Promise<ResolvedOriginToolCallResult> {
|
}): Promise<ResolvedOriginToolCallResult> {
|
||||||
// TODO: rate-limiting
|
// TODO: rate-limiting
|
||||||
|
@ -106,6 +109,17 @@ export async function resolveOriginToolCall({
|
||||||
rateLimit = toolConfig.rateLimit as RateLimit
|
rateLimit = toolConfig.rateLimit as RateLimit
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (!cacheControl) {
|
||||||
|
if (toolConfig.cacheControl !== undefined) {
|
||||||
|
cacheControl = toolConfig.cacheControl
|
||||||
|
} else if (toolConfig.pure) {
|
||||||
|
cacheControl =
|
||||||
|
'public, max-age=31560000, s-maxage=31560000, stale-while-revalidate=3600'
|
||||||
|
} else {
|
||||||
|
cacheControl = 'no-store'
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
const pricingPlanToolConfig = pricingPlan
|
const pricingPlanToolConfig = pricingPlan
|
||||||
? toolConfig.pricingPlanConfig?.[pricingPlan.slug]
|
? toolConfig.pricingPlanConfig?.[pricingPlan.slug]
|
||||||
: undefined
|
: undefined
|
||||||
|
@ -162,7 +176,7 @@ export async function resolveOriginToolCall({
|
||||||
deployment
|
deployment
|
||||||
})
|
})
|
||||||
|
|
||||||
updateOriginRequest(originRequest, { consumer, deployment })
|
updateOriginRequest(originRequest, { consumer, deployment, cacheControl })
|
||||||
|
|
||||||
const cacheKey = await getRequestCacheKey(originRequest)
|
const cacheKey = await getRequestCacheKey(originRequest)
|
||||||
|
|
||||||
|
@ -173,15 +187,17 @@ export async function resolveOriginToolCall({
|
||||||
waitUntil
|
waitUntil
|
||||||
})
|
})
|
||||||
|
|
||||||
// non-cached version
|
|
||||||
// const originResponse = await fetch(originRequest)
|
|
||||||
|
|
||||||
return {
|
return {
|
||||||
toolCallArgs,
|
toolCallArgs,
|
||||||
originRequest,
|
originRequest,
|
||||||
originResponse
|
originResponse
|
||||||
}
|
}
|
||||||
} else if (originAdapter.type === 'mcp') {
|
} else if (originAdapter.type === 'mcp') {
|
||||||
|
const { projectIdentifier } = parseDeploymentIdentifier(
|
||||||
|
deployment.identifier,
|
||||||
|
{ errorStatusCode: 500 }
|
||||||
|
)
|
||||||
|
|
||||||
const id = env.DO_MCP_CLIENT.idFromName(sessionId)
|
const id = env.DO_MCP_CLIENT.idFromName(sessionId)
|
||||||
const originMcpClient = env.DO_MCP_CLIENT.get(id)
|
const originMcpClient = env.DO_MCP_CLIENT.get(id)
|
||||||
|
|
||||||
|
@ -191,11 +207,6 @@ export async function resolveOriginToolCall({
|
||||||
version: originAdapter.serverInfo.version
|
version: originAdapter.serverInfo.version
|
||||||
})
|
})
|
||||||
|
|
||||||
const { projectIdentifier } = parseDeploymentIdentifier(
|
|
||||||
deployment.identifier,
|
|
||||||
{ errorStatusCode: 500 }
|
|
||||||
)
|
|
||||||
|
|
||||||
const originMcpRequestMetadata = {
|
const originMcpRequestMetadata = {
|
||||||
agenticProxySecret: deployment._secret,
|
agenticProxySecret: deployment._secret,
|
||||||
sessionId,
|
sessionId,
|
||||||
|
@ -216,8 +227,35 @@ export async function resolveOriginToolCall({
|
||||||
projectIdentifier
|
projectIdentifier
|
||||||
} as AgenticMcpRequestMetadata
|
} as AgenticMcpRequestMetadata
|
||||||
|
|
||||||
|
let cacheKey: Request | undefined
|
||||||
|
|
||||||
|
if (cacheControl && isCacheControlPubliclyCacheable(cacheControl)) {
|
||||||
|
const fakeOriginRequest = new Request(deployment.originUrl, {
|
||||||
|
method: 'POST',
|
||||||
|
headers: {
|
||||||
|
'content-type': 'application/json',
|
||||||
|
'cache-control': cacheControl
|
||||||
|
},
|
||||||
|
body: JSON.stringify({
|
||||||
|
name: tool.name,
|
||||||
|
args: toolCallArgs,
|
||||||
|
metadata: originMcpRequestMetadata!
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
cacheKey = await getRequestCacheKey(fakeOriginRequest)
|
||||||
|
if (cacheKey) {
|
||||||
|
const response = await caches.default.match(cacheKey)
|
||||||
|
if (response) {
|
||||||
|
return {
|
||||||
|
toolCallArgs,
|
||||||
|
toolCallResponse: (await response.json()) as McpToolCallResponse
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// TODO: add timeout support to the origin tool call?
|
// TODO: add timeout support to the origin tool call?
|
||||||
// TODO: add response caching for origin MCP tool calls
|
|
||||||
const toolCallResponseString = await originMcpClient.callTool({
|
const toolCallResponseString = await originMcpClient.callTool({
|
||||||
name: tool.name,
|
name: tool.name,
|
||||||
args: toolCallArgs,
|
args: toolCallArgs,
|
||||||
|
@ -227,6 +265,16 @@ export async function resolveOriginToolCall({
|
||||||
toolCallResponseString
|
toolCallResponseString
|
||||||
) as McpToolCallResponse
|
) as McpToolCallResponse
|
||||||
|
|
||||||
|
if (cacheKey && cacheControl) {
|
||||||
|
const fakeHttpResponse = new Response(toolCallResponseString, {
|
||||||
|
headers: {
|
||||||
|
'content-type': 'application/json',
|
||||||
|
'cache-control': cacheControl
|
||||||
|
}
|
||||||
|
})
|
||||||
|
waitUntil(caches.default.put(cacheKey, fakeHttpResponse))
|
||||||
|
}
|
||||||
|
|
||||||
return {
|
return {
|
||||||
toolCallArgs,
|
toolCallArgs,
|
||||||
toolCallResponse
|
toolCallResponse
|
||||||
|
|
|
@ -9,10 +9,12 @@ export function updateOriginRequest(
|
||||||
originRequest: Request,
|
originRequest: Request,
|
||||||
{
|
{
|
||||||
deployment,
|
deployment,
|
||||||
consumer
|
consumer,
|
||||||
|
cacheControl
|
||||||
}: {
|
}: {
|
||||||
deployment: AdminDeployment
|
deployment: AdminDeployment
|
||||||
consumer?: AdminConsumer
|
consumer?: AdminConsumer
|
||||||
|
cacheControl?: string
|
||||||
}
|
}
|
||||||
) {
|
) {
|
||||||
// originRequest.headers.delete('authorization')
|
// originRequest.headers.delete('authorization')
|
||||||
|
@ -88,5 +90,9 @@ export function updateOriginRequest(
|
||||||
// https://support.cloudflare.com/hc/en-us/articles/360029779472-Troubleshooting-Cloudflare-1XXX-errors#error1000
|
// https://support.cloudflare.com/hc/en-us/articles/360029779472-Troubleshooting-Cloudflare-1XXX-errors#error1000
|
||||||
// originRequest.headers.set('x-forwarded-for', ip)
|
// originRequest.headers.set('x-forwarded-for', ip)
|
||||||
|
|
||||||
|
if (cacheControl) {
|
||||||
|
originRequest.headers.set('cache-control', cacheControl)
|
||||||
|
}
|
||||||
|
|
||||||
originRequest.headers.set('x-agentic-proxy-secret', deployment._secret)
|
originRequest.headers.set('x-agentic-proxy-secret', deployment._secret)
|
||||||
}
|
}
|
||||||
|
|
|
@ -495,7 +495,8 @@ export interface components {
|
||||||
/** @default true */
|
/** @default true */
|
||||||
enabled: boolean;
|
enabled: boolean;
|
||||||
/** @default false */
|
/** @default false */
|
||||||
immutable: boolean;
|
pure: boolean;
|
||||||
|
cacheControl?: string;
|
||||||
/** @default true */
|
/** @default true */
|
||||||
reportUsage: boolean;
|
reportUsage: boolean;
|
||||||
rateLimit?: components["schemas"]["RateLimit"] | null;
|
rateLimit?: components["schemas"]["RateLimit"] | null;
|
||||||
|
|
|
@ -114,7 +114,20 @@ export const toolConfigSchema = z
|
||||||
*
|
*
|
||||||
* @default false
|
* @default false
|
||||||
*/
|
*/
|
||||||
immutable: z.boolean().optional().default(false),
|
pure: z.boolean().optional().default(false),
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A `Cache-Control` header value to use for caching this tool's responses.
|
||||||
|
*
|
||||||
|
* If `pure` is `true`, this defaults to: `public, max-age=31560000, s-maxage=31560000, stale-while-revalidate=3600` (cache publicly for up to 1 year).
|
||||||
|
*
|
||||||
|
* If `pure` is `false`, this defaults to the origin server's
|
||||||
|
* `cache-control` header value. If the origin server does not set a
|
||||||
|
* `cache-control` header, it defaults to `no-store`.
|
||||||
|
*
|
||||||
|
* @default undefined
|
||||||
|
*/
|
||||||
|
cacheControl: z.string().optional(),
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Whether calls to this tool should be reported as usage for the default
|
* Whether calls to this tool should be reported as usage for the default
|
||||||
|
|
|
@ -15,8 +15,7 @@
|
||||||
- => OpenAPI: `GET/POST/ETC originUrl/toolName` operation with transformed tool params
|
- => OpenAPI: `GET/POST/ETC originUrl/toolName` operation with transformed tool params
|
||||||
- RAW: `METHOD gateway.agentic.so/deploymentIdentifier/<pathname>`
|
- RAW: `METHOD gateway.agentic.so/deploymentIdentifier/<pathname>`
|
||||||
- => Raw HTTP: `METHOD originUrl/<pathname>` simple HTTP proxy request
|
- => Raw HTTP: `METHOD originUrl/<pathname>` simple HTTP proxy request
|
||||||
|
- TODO: remove / disable `raw` support for now
|
||||||
**do I just ditch public REST interface and focus on MCP?**
|
|
||||||
|
|
||||||
## TODO
|
## TODO
|
||||||
|
|
||||||
|
@ -36,8 +35,6 @@
|
||||||
- how to handle binary bodies and responses?
|
- how to handle binary bodies and responses?
|
||||||
- add support for `immutable` in `toolConfigs`
|
- add support for `immutable` in `toolConfigs`
|
||||||
- **Public MCP server interface**
|
- **Public MCP server interface**
|
||||||
- _McpAgent.serve_
|
|
||||||
- how do I use consumer auth tokens with this flow?
|
|
||||||
- how does oauth work with this flow?
|
- how does oauth work with this flow?
|
||||||
- **Origin MCP servers**
|
- **Origin MCP servers**
|
||||||
- how to guarantee that the request is coming from agentic?
|
- how to guarantee that the request is coming from agentic?
|
||||||
|
|
Ładowanie…
Reference in New Issue