better amazon bedrock caching with anthropic models

This commit is contained in:
Dax Raad 2025-06-29 19:26:50 -04:00
parent 98ee5a3d87
commit ca48a4f0fb
3 changed files with 18 additions and 0 deletions

View file

@ -185,6 +185,7 @@ export namespace Provider {
source,
info,
options,
getModel,
}
return
}

View file

@ -20,6 +20,19 @@ export namespace ProviderTransform {
}
}
}
if (providerID === "amazon-bedrock" || modelID.includes("anthropic")) {
const system = msgs.filter((msg) => msg.role === "system").slice(0, 2)
const final = msgs.filter((msg) => msg.role !== "system").slice(-2)
for (const msg of unique([...system, ...final])) {
msg.providerMetadata = {
...msg.providerMetadata,
bedrock: {
cachePoint: { type: "ephemeral" },
},
}
}
}
return msgs
}
}

View file

@ -882,8 +882,12 @@ export namespace Session {
reasoning: 0,
cache: {
write: (metadata?.["anthropic"]?.["cacheCreationInputTokens"] ??
// @ts-expect-error
metadata?.["bedrock"]?.["usage"]?.["cacheWriteInputTokens"] ??
0) as number,
read: (metadata?.["anthropic"]?.["cacheReadInputTokens"] ??
// @ts-expect-error
metadata?.["bedrock"]?.["usage"]?.["cacheReadInputTokens"] ??
0) as number,
},
}