alihan/crush-code-agent-ide
1
0
Fork 0
mirror of https://github.com/charmbracelet/crush.git synced 2025-08-02 05:20:46 +03:00
Code Issues Packages Projects Releases Wiki Activity

chore: proper caching for anthropic models with openrouter

Browse Source
This commit is contained in:
Kujtim Hoxha
2025-07-28 20:27:23 +02:00
parent 59b9254e5b
commit c35e700017
1 changed files with 42 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
internal/llm/provider/openai.go
View File

@@ -7,6 +7,7 @@ import (
"fmt"
"io"
"log/slog"
"strings"
"time"
"github.com/charmbracelet/catwalk/pkg/catwalk"
@@ -56,14 +57,33 @@ func createOpenAIClient(opts providerClientOptions) openai.Client {
}
func (o *openaiClient) convertMessages(messages []message.Message) (openaiMessages []openai.ChatCompletionMessageParamUnion) {
isAnthropicModel := o.providerOptions.config.ID == "openrouter" && strings.HasPrefix(o.Model().ID, "anthropic/")
// Add system message first
systemMessage := o.providerOptions.systemMessage
if o.providerOptions.systemPromptPrefix != "" {
systemMessage = o.providerOptions.systemPromptPrefix + "\n" + systemMessage
}
openaiMessages = append(openaiMessages, openai.SystemMessage(systemMessage))
for _, msg := range messages {
systemTextBlock := openai.ChatCompletionContentPartTextParam{Text: systemMessage}
if isAnthropicModel && !o.providerOptions.disableCache {
systemTextBlock.SetExtraFields(
map[string]any{
"cache_control": map[string]string{
"type": "ephemeral",
},
},
)
}
var content []openai.ChatCompletionContentPartTextParam
content = append(content, systemTextBlock)
system := openai.SystemMessage(content)
openaiMessages = append(openaiMessages, system)
for i, msg := range messages {
cache := false
if i > len(messages)-3 {
cache = true
}
switch msg.Role {
case message.User:
var content []openai.ChatCompletionContentPartUnionParam
@@ -75,6 +95,13 @@ func (o *openaiClient) convertMessages(messages []message.Message) (openaiMessag
content = append(content, openai.ChatCompletionContentPartUnionParam{OfImageURL: &imageBlock})
}
if cache && !o.providerOptions.disableCache && isAnthropicModel {
textBlock.SetExtraFields(map[string]any{
"cache_control": map[string]string{
"type": "ephemeral",
},
})
}
openaiMessages = append(openaiMessages, openai.UserMessage(content))
@@ -86,8 +113,20 @@ func (o *openaiClient) convertMessages(messages []message.Message) (openaiMessag
hasContent := false
if msg.Content().String() != "" {
hasContent = true
textBlock := openai.ChatCompletionContentPartTextParam{Text: msg.Content().String()}
if cache && !o.providerOptions.disableCache && isAnthropicModel {
textBlock.SetExtraFields(map[string]any{
"cache_control": map[string]string{
"type": "ephemeral",
},
})
}
assistantMsg.Content = openai.ChatCompletionAssistantMessageParamContentUnion{
OfString: openai.String(msg.Content().String()),
OfArrayOfContentParts: []openai.ChatCompletionAssistantMessageParamContentArrayOfContentPartUnion{
{
OfText: &textBlock,
},
},
}
}