diff --git a/cmd/ohos/HomeAgent/entry/src/main/ets/common/BridgeCaps.ets b/cmd/ohos/HomeAgent/entry/src/main/ets/common/BridgeCaps.ets index 8c13fb9..6c895f2 100644 --- a/cmd/ohos/HomeAgent/entry/src/main/ets/common/BridgeCaps.ets +++ b/cmd/ohos/HomeAgent/entry/src/main/ets/common/BridgeCaps.ets @@ -27,6 +27,10 @@ export interface CapResult { status: string; // 'ok' | 'error' output: string; error: string; + // chunked 为 true 时表示结果**已由能力内部经二进制分块回传**（如录像）， + // DeviceBridge 不要再发 cmd_result；否则网关会把后续分块挂在一条已完成的 + // 请求上，或先用 cmd_result 结束、再来的 cmd_data_start 找不到归属。 + chunked?: boolean; } interface DeviceStatusPayload { @@ -75,6 +79,19 @@ function errResult(errMsg: string): CapResult { return r; } +/** + * 二进制分块发送回调。由 BridgeRouter 注入（它持有 deviceBridge + reqId）， + * BridgeCaps 因此不必 import DeviceBridge —— 否则 DeviceBridge 为取 CapResult + * 而 import BridgeCaps，两边成环。分层也更干净：能力实现不碰 socket。 + */ +export type DataChunkSender = (kind: string, mime: string, bytes: Uint8Array) => void; + +// chunkedResult：结果已由能力自己分块发出，不再回 cmd_result。 +function chunkedResult(output: string): CapResult { + const r: CapResult = { status: 'ok', output: output, error: '', chunked: true }; + return r; +} + // ===== screensee：截取本应用当前画面（前台时为整屏可见内容）===== const SNAPSHOT_COMPONENT_ID: string = 'homeagent-root'; @@ -144,25 +161,40 @@ export async function capScreensee(): Promise { // READ_IMAGEVIDEO 这类受限权限。 const CAMERASUE_MAX_B64: number = 950000; +/** 录像回传上限：与网关 mediaDir 落盘模式配合，避免把设备内存/WS 打爆。 */ +const CAMERASUE_MAX_VIDEO: number = 64 * 1024 * 1024; + /** * camerasue 实现。 * * 参数语义与 homeagent-cmdrun 的说明一致：无参数 = 抓拍单张； - * `` = 录 N 秒视频。录像回传暂未接二进制分块（DeviceBridge 已具备 - * sendDataChunked，接线待做）——这里明确拒掉，而不是回一个超长 base64 - * 把模型上下文和 bridge 都撑爆。 + * `` = 录 N 秒视频。 + * + * 视频为什么要走二进制分块：一段 10s 录像动辄数 MB，base64 后还要再膨胀 + * 1/3，既撑爆模型上下文也撑爆 WS 单帧。cameraPicker 本身支持 VIDEO + * 模式（系统相机会直接进录像界面），取回文件后用 sendDataChunked 按 + * cmd_data_start/分块/cmd_data_end 回传——网关侧聚合后落盘成文件，agent 拿路径。 + * 这与 GUI/CLI 客户端的 camerasue 录像路径一致。 */ export async function capCamerasue(context: common.UIAbilityContext, - rawArgs: string): Promise { + rawArgs: string, + sendChunked: DataChunkSender | null): Promise { const raw: string = rawArgs.trim(); + let videoSeconds: number = 0; if (raw.length > 0) { const digits: RegExp = new RegExp('^\\d+$'); - if (digits.test(raw)) { - return errResult('鸿蒙端暂不支持录像回传，请用 camerasue 抓拍单张'); + if (!digits.test(raw)) { + return errResult('camerasue 参数只接受纯数字秒数，如 camerasue 5'); + } + videoSeconds = parseInt(raw, 10); + if (videoSeconds <= 0 || videoSeconds > 300) { + return errResult('录像时长需在 1~300 秒之间'); } } + const isVideo: boolean = videoSeconds > 0; - const filePath: string = context.filesDir + '/camerasue_' + Date.now().toString() + '.jpg'; + const ext: string = isVideo ? '.mp4' : '.jpg'; + const filePath: string = context.filesDir + '/camerasue_' + Date.now().toString() + ext; try { // cameraPicker 要求 saveUri 指向的文件存在且可写，先建空文件占位 const f: fileIo.File = fileIo.openSync(filePath, @@ -177,15 +209,60 @@ export async function capCamerasue(context: common.UIAbilityContext, cameraPosition: camera.CameraPosition.CAMERA_POSITION_BACK, saveUri: fileUri.getUriFromPath(filePath), }; + if (isVideo) { + profile.videoDuration = videoSeconds; + } + const mediaType: cameraPicker.PickerMediaType = isVideo + ? cameraPicker.PickerMediaType.VIDEO + : cameraPicker.PickerMediaType.PHOTO; const res: cameraPicker.PickerResult = - await cameraPicker.pick(context, [cameraPicker.PickerMediaType.PHOTO], profile); + await cameraPicker.pick(context, [mediaType], profile); if (res.resultCode !== 0 || res.resultUri.length === 0) { - return errResult('未获取到照片（可能被取消）'); + return errResult(isVideo ? '未获取到录像（可能被取消）' : '未获取到照片（可能被取消）'); } } catch (e) { return errResult('相机不可用或未授权，请确认应用在前台并允许使用相机'); } + if (isVideo) { + return readAndSendVideo(filePath, sendChunked); + } + return readPhotoAsBase64(filePath); +} + +/** 读回录像并以二进制分块回传；网关聚合后落盘，agent 拿文件路径。 */ +function readAndSendVideo(filePath: string, sendChunked: DataChunkSender | null): CapResult { + let fd: number = -1; + try { + const stat: fileIo.Stat = fileIo.statSync(filePath); + if (stat.size <= 0) { + return errResult('录像文件为空，请重试'); + } + if (stat.size > CAMERASUE_MAX_VIDEO) { + return errResult('录像文件过大（超过 64MB），请缩短时长'); + } + const buf: ArrayBuffer = new ArrayBuffer(stat.size); + const rf: fileIo.File = fileIo.openSync(filePath, fileIo.OpenMode.READ_ONLY); + fd = rf.fd; + fileIo.readSync(fd, buf); + fileIo.closeSync(rf); + fd = -1; + if (sendChunked === null) { + return errResult('录像回传通道未就绪，请重试'); + } + sendChunked('camera_video', 'video/mp4', new Uint8Array(buf)); + // 分块已代表本次请求的完整结果，DeviceBridge 不再回 cmd_result。 + return chunkedResult('录像已回传（' + stat.size.toString() + ' 字节）'); + } catch (e) { + if (fd >= 0) { + try { fileIo.closeSync(fd); } catch (ignore) {} + } + return errResult('录像读取失败，请重试'); + } +} + +/** 照片仍走小体积 base64 内联（图片不大，不必分块）。 */ +function readPhotoAsBase64(filePath: string): CapResult { let fd: number = -1; try { const stat: fileIo.Stat = fileIo.statSync(filePath); diff --git a/cmd/ohos/HomeAgent/entry/src/main/ets/common/BridgeRouter.ets b/cmd/ohos/HomeAgent/entry/src/main/ets/common/BridgeRouter.ets index 8adfe92..a6eabe1 100644 --- a/cmd/ohos/HomeAgent/entry/src/main/ets/common/BridgeRouter.ets +++ b/cmd/ohos/HomeAgent/entry/src/main/ets/common/BridgeRouter.ets @@ -1,6 +1,7 @@ import { deviceBridge } from './DeviceBridge'; import { CapResult, + DataChunkSender, capScreensee, capCamerasue, capClipboardSee, @@ -89,7 +90,12 @@ async function executeCommand(reqId: string, command: string): Promise { + deviceBridge.sendDataChunked(reqId, kind, mime, bytes); + }; + return capCamerasue(appContext, args, sender); } if (name === 'clipboardsee') { if (hasArgs(args)) { diff --git a/cmd/ohos/HomeAgent/entry/src/main/ets/common/DeviceBridge.ets b/cmd/ohos/HomeAgent/entry/src/main/ets/common/DeviceBridge.ets index 16d0b4c..366ea3d 100644 --- a/cmd/ohos/HomeAgent/entry/src/main/ets/common/DeviceBridge.ets +++ b/cmd/ohos/HomeAgent/entry/src/main/ets/common/DeviceBridge.ets @@ -247,6 +247,11 @@ export class DeviceBridgeClient { } const handler: BridgeCmdHandler = this.cmdHandler; handler(reqId, command).then((res: CapResult) => { + // res.chunked 时结果已由能力自己用二进制分块发完（如录像）： + // 此时再发 cmd_result 会让网关把一条已完成请求与后续分块错配。 + if (res.chunked === true) { + return; + } this.sendResult(reqId, res.status, res.output, res.error); }).catch((e: Object) => { this.sendResult(reqId, 'error', '', '本机能力执行失败，请稍后重试'); diff --git a/cmd/waiter/builtin.go b/cmd/waiter/builtin.go index d39ddf7..01dce11 100644 --- a/cmd/waiter/builtin.go +++ b/cmd/waiter/builtin.go @@ -38,6 +38,8 @@ Server commands (local 与 remote 行为一致): /memory query query graph memory /memory graph dump full graph memory snapshot /memory text [n] recent text-memory events + stats + /memory context [q] assembled memory context (what gets injected) + /memory tools memory tool definitions + tool prompt /knowledge list knowledge base (+stats) /knowledge delete delete knowledge item /config dump kernel config (JSON) @@ -265,8 +267,15 @@ Any other text is sent to the agent directly.`) case sub == "text" || strings.HasPrefix(sub, "text "): d, _ := rc.DoAPI("GET", "/api/v1/memory/text", "") printJSON(out, d) + case sub == "context" || strings.HasPrefix(sub, "context "): + q := strings.TrimSpace(strings.TrimPrefix(sub, "context")) + d, _ := rc.DoAPI("GET", "/api/v1/memory/context?q="+q, "") + printJSON(out, d) + case sub == "tools": + d, _ := rc.DoAPI("GET", "/api/v1/memory/tools", "") + printJSON(out, d) default: - fmt.Fprintln(out, "usage: /memory query | /memory graph | /memory text [n]") + fmt.Fprintln(out, "usage: /memory query | /memory graph | /memory text [n] | /memory context [q] | /memory tools") } } else { state.Send(cmd) diff --git a/internal/plugins/cli/plugin.go b/internal/plugins/cli/plugin.go index 741800d..ea867da 100644 --- a/internal/plugins/cli/plugin.go +++ b/internal/plugins/cli/plugin.go @@ -356,6 +356,8 @@ func (p *Plugin) cmdHelp(conn net.Conn) { /memory query <关键词> 查询图记忆 /memory graph 导出整张图记忆快照 /memory text [n] 最近 n 条文本记忆事件 + 统计 + /memory context [q] 组装后的记忆上下文（会注入什么） + /memory tools 记忆工具定义 + 工具提示词 /knowledge 列出知识库 /knowledge delete 删除一条知识 /knowledge stats 知识库统计 @@ -747,8 +749,42 @@ func (p *Plugin) cmdMemory(conn net.Conn, parts []string, s *sdk.PluginSDK) { "events": events, "stats": tm.Stats(), }) + case "context": + // 与 WebUI GET /api/v1/memory/context 同源：都走 IndexerAPI。 + // 打印“输进去会拼成什么上下文”，是排查召回质量最直接的一眼。 + idx := s.Indexer() + if idx == nil { + writeLine(conn, map[string]interface{}{"type": "error", "error": "indexer not available"}) + return + } + q := "" + if len(parts) >= 3 { + q = strings.Join(parts[2:], " ") + } + injected, err := idx.BuildContext(q) + if err != nil { + writeLine(conn, map[string]interface{}{"type": "error", "error": err.Error()}) + return + } + writeJSONContent(conn, map[string]interface{}{ + "context": idx.FormatContext(injected), + "summary": injected.Summary, + "entities": injected.Entities, + "token_estimate": injected.TokenEstimate, + }) + case "tools": + // 与 WebUI GET /api/v1/memory/tools 同源。 + idx := s.Indexer() + if idx == nil { + writeLine(conn, map[string]interface{}{"type": "error", "error": "indexer not available"}) + return + } + writeJSONContent(conn, map[string]interface{}{ + "tools": idx.GetToolDefinitions(), + "tool_prompt": idx.BuildToolPrompt(), + }) default: - writeLine(conn, map[string]interface{}{"type": "response", "content": "用法: /memory query <关键词> | /memory graph | /memory text [n]"}) + writeLine(conn, map[string]interface{}{"type": "response", "content": "用法: /memory query <关键词> | /memory graph | /memory text [n] | /memory context [q] | /memory tools"}) } }