diff --git a/internal/lua/adapters/anthropic.lua b/internal/lua/adapters/anthropic.lua index 56ad3ab..acf018e 100644 --- a/internal/lua/adapters/anthropic.lua +++ b/internal/lua/adapters/anthropic.lua @@ -11,13 +11,42 @@ function adapter.transform_request(raw_body) local ok, req = pcall(json.decode, raw_body) if not ok then return raw_body end + -- 将 OpenAI 风格 content(字符串或 [{type:*}] 数组)拆成文本/图片块 + local function collect_blocks(content) + if type(content) == "string" then + return { { type = "text", text = content } } + end + local blocks = {} + for _, p in ipairs(content or {}) do + if p.type == "text" then + table.insert(blocks, { type = "text", text = p.text }) + elseif p.type == "image_url" and type(p.image_url) == "table" and p.image_url.url then + local mt, b64 = string.match(p.image_url.url, "^data:([^,]+);base64,(.+)$") + if b64 then + table.insert(blocks, { type = "image", source = { type = "base64", media_type = mt or "image/png", data = b64 } }) + else + table.insert(blocks, { type = "image", source = { type = "url", url = p.image_url.url } }) + end + end + end + return blocks + end + local function text_of(content) + if type(content) == "string" then return content end + local t = "" + for _, p in ipairs(content or {}) do + if p.type == "text" and p.text then t = t .. p.text end + end + return t + end + local msgs = {} local system = "" for _, m in ipairs(req.messages or {}) do if m.role == "system" then - system = system .. m.content .. "\n" + system = system .. text_of(m.content) .. "\n" else - table.insert(msgs, { role = m.role, content = m.content }) + table.insert(msgs, { role = m.role, content = collect_blocks(m.content) }) end end @@ -74,12 +103,41 @@ function adapter.transform_stream_chunk(raw_chunk) if chunk.type == "message_delta" then return json.encode({ content = "", done = (chunk.delta and chunk.delta.stop_reason ~= nil) }) end + if chunk.type == "content_block_start" and chunk.content_block + and chunk.content_block.type == "tool_use" then + -- first fragment of a tool call: emit index + id + name, empty args + return json.encode({ + content = "", done = false, + tool_calls = { { + index = chunk.index or 0, + id = chunk.content_block.id or "", + type = "function", + ["function"] = { name = chunk.content_block.name or "", arguments = "" } + } } + }) + end if chunk.type == "content_block_delta" and chunk.delta then + if chunk.delta.type == "input_json_delta" then + -- incremental JSON fragment; clients accumulate across chunks + local unified = { content = "", done = false, tool_calls = { { + index = chunk.index or 0, + id = "", + type = "function", + ["function"] = { name = "", arguments = chunk.delta.partial_json or "" } + } } } + return json.encode(unified) + end + if chunk.delta.type == "thinking_delta" and chunk.delta.thinking then + return json.encode({ content = "", done = false, reasoning_content = chunk.delta.thinking }) + end return json.encode({ content = chunk.delta.text or "", done = false }) end if chunk.type == "message_stop" then return json.encode({ content = "", done = true }) end + if chunk.type == "content_block_stop" then + return json.encode({ content = "", done = false }) + end return "" end diff --git a/internal/lua/adapters/ollama.lua b/internal/lua/adapters/ollama.lua index a931045..4e13d71 100644 --- a/internal/lua/adapters/ollama.lua +++ b/internal/lua/adapters/ollama.lua @@ -19,11 +19,29 @@ function adapter.transform_request(raw_body) } } - -- 转换 messages 格式(Ollama 兼容 OpenAI 的 messages 格式) + -- 转换 messages 格式(Ollama messages 支持 images base64 数组) if req.messages then local msgs = {} for _, m in ipairs(req.messages) do - table.insert(msgs, { role = m.role, content = m.content }) + local text, images + if type(m.content) == "string" then + text, images = m.content, nil + else + text = "" + images = {} + for _, p in ipairs(m.content or {}) do + if p.type == "text" then + text = text .. (p.text or "") + elseif p.type == "image_url" and type(p.image_url) == "table" and p.image_url.url then + local b64 = string.match(p.image_url.url, "^data:[^,]+;base64,(.+)$") + if b64 then table.insert(images, b64) end + end + end + if #images == 0 then images = nil end + end + local msg = { role = m.role, content = text } + if images then msg.images = images end + table.insert(msgs, msg) end ollama_req.messages = msgs end @@ -54,10 +72,29 @@ function adapter.transform_stream_chunk(raw_chunk) if not ok then return "" end if not chunk.message then return "" end - return json.encode({ + local unified = { content = chunk.message.content or "", done = chunk.done or false - }) + } + if chunk.message.reasoning_content then + unified.reasoning_content = chunk.message.reasoning_content + end + if chunk.message.tool_calls then + local tools = {} + for _, tc in ipairs(chunk.message.tool_calls) do + table.insert(tools, { + index = #tools, + id = tc.id or ("call_" .. #tools), + type = "function", + ["function"] = { + name = tc["function"] and tc["function"].name or "", + arguments = tc["function"] and (tc["function"].arguments or "{}") or "{}" + } + }) + end + unified.tool_calls = tools + end + return json.encode(unified) end return adapter diff --git a/internal/lua/vm_test.go b/internal/lua/vm_test.go index 763d437..a9cd4b4 100644 --- a/internal/lua/vm_test.go +++ b/internal/lua/vm_test.go @@ -134,4 +134,59 @@ func TestVMConcurrentCalls(t *testing.T) { for err := range errs { t.Fatalf("concurrent call: %v", err) } -} \ No newline at end of file +} +func TestOllamaMultimodal(t *testing.T) { + vm := NewVM(t.TempDir()) + if err := vm.Start(); err != nil { + t.Fatalf("start vm: %v", err) + } + defer vm.Stop() + + raw := `{"model":"llama3","messages":[{"role":"user","content":"hi"},{"role":"user","content":[{"type":"text","text":"look"},{"type":"image_url","image_url":{"url":"data:image/png;base64,AAAA"}}]}]}` + out, err := vm.CallTransformRequest("ollama", raw) + if err != nil { + t.Fatalf("transform_request: %v", err) + } + if !strings.Contains(out, `"images":["AAAA"]`) { + t.Fatalf("expected base64 images array, got: %s", out) + } + if !strings.Contains(out, `"content":"look"`) { + t.Fatalf("text not preserved: %s", out) + } +} + +func TestAnthropicMultimodal(t *testing.T) { + vm := NewVM(t.TempDir()) + if err := vm.Start(); err != nil { + t.Fatalf("start vm: %v", err) + } + defer vm.Stop() + + raw := `{"model":"claude-sonnet-4-20250514","messages":[{"role":"user","content":[{"type":"text","text":"what is this"},{"type":"image_url","image_url":{"url":"data:image/png;base64,AAEC"}}]}]}` + out, err := vm.CallTransformRequest("anthropic", raw) + if err != nil { + t.Fatalf("transform_request: %v", err) + } + if !strings.Contains(out, `"type":"image"`) || !strings.Contains(out, `"media_type":"image/png"`) || !strings.Contains(out, `"data":"AAEC"`) { + t.Fatalf("expected anthropic image source block, got: %s", out) + } + if !strings.Contains(out, `"type":"text"`) { + t.Fatalf("expected text block preserved: %s", out) + } +} + +func TestOpenAIPassthroughKeepsMultimodal(t *testing.T) { + vm := NewVM(t.TempDir()) + if err := vm.Start(); err != nil { + t.Fatalf("start vm: %v", err) + } + defer vm.Stop() + raw := `{"model":"auto","messages":[{"role":"user","content":[{"type":"text","text":"hi"},{"type":"image_url","image_url":{"url":"data:image/png;base64,QUJD"}}]}]}` + out, err := vm.CallTransformRequest("openai", raw) + if err != nil { + t.Fatalf("transform_request: %v", err) + } + if !strings.Contains(out, `"image_url"`) || !strings.Contains(out, "QUJD") { + t.Fatalf("openai passthrough dropped multimodal: %s", out) + } +}