mirror of
https://gitcode.com/JianFeeeee/HomeAgent.git
synced 2026-09-22 09:58:06 +00:00
feat(cli,ohos): 补齐两处能力缺口——CLI /memory context|tools、鸿蒙 camerasue 录像回传
核对「CLI 与 WebUI 插件能力对齐」时发现两个此前遗漏的缺口,一并补齐。 1) CLI /memory 缺 context 与 tools(internal/plugins/cli/plugin.go) WebUI 有 GET /api/v1/memory/context 与 /api/v1/memory/tools,CLI 只有 query/graph/text。IndexerAPI 本就对内部插件开放(BuildContext/ FormatContext/GetToolDefinitions/BuildToolPrompt),不是 SDK 缺口。 补上后与 WebUI 同源:context 打印「实际会注入什么上下文」, tools 打印工具定义 + 工具提示词。 waiter 同步接上两条远端路由与 help 文案。 2) 鸿蒙 camerasue 录像(BridgeCaps/BridgeRouter/DeviceBridge.ets) 此前 `camerasue <N秒>` 直接返回「暂不支持录像回传」。查 SDK 后发现 cameraPicker 本身就有 PickerMediaType.VIDEO 与 PickerProfile.videoDuration —— 录像完全可行,只是回传通道没接。 现改为:VIDEO 模式取回 mp4,经 DeviceBridge.sendDataChunked 按 cmd_data_start/分块/cmd_data_end 回传(与 GUI/CLI 录像路径一致), 网关聚合后落盘成文件、agent 拿路径;照片仍走小体积 base64 内联。 上限 64MB、时长 1~300s,超限明确报错而不是把 WS/上下文撑爆。 依赖方向处理:BridgeCaps 需要「往本请求回传字节」,但 DeviceBridge 为取 CapResult 已 import BridgeCaps,反向 import 会成环。改为 BridgeRouter 注入 DataChunkSender 回调(它同时持有 deviceBridge 与 reqId), BridgeCaps 不碰 socket。新增 CapResult.chunked 标记「结果已由能力分块 发完」,DeviceBridge 据此不再回 cmd_result,避免网关把已完成请求与后续 分块错配。 验证:hvigor assembleHap BUILD SUCCESSFUL(ArkTS 编译通过,改动文件零告警); make check-client-versions 一致;go vet 干净;全量 go test ./internal/... ./cmd/... 零失败。
This commit is contained in:
@ -27,6 +27,10 @@ export interface CapResult {
|
||||
status: string; // 'ok' | 'error'
|
||||
output: string;
|
||||
error: string;
|
||||
// chunked 为 true 时表示结果**已由能力内部经二进制分块回传**(如录像),
|
||||
// DeviceBridge 不要再发 cmd_result;否则网关会把后续分块挂在一条已完成的
|
||||
// 请求上,或先用 cmd_result 结束、再来的 cmd_data_start 找不到归属。
|
||||
chunked?: boolean;
|
||||
}
|
||||
|
||||
interface DeviceStatusPayload {
|
||||
@ -75,6 +79,19 @@ function errResult(errMsg: string): CapResult {
|
||||
return r;
|
||||
}
|
||||
|
||||
/**
|
||||
* 二进制分块发送回调。由 BridgeRouter 注入(它持有 deviceBridge + reqId),
|
||||
* BridgeCaps 因此不必 import DeviceBridge —— 否则 DeviceBridge 为取 CapResult
|
||||
* 而 import BridgeCaps,两边成环。分层也更干净:能力实现不碰 socket。
|
||||
*/
|
||||
export type DataChunkSender = (kind: string, mime: string, bytes: Uint8Array) => void;
|
||||
|
||||
// chunkedResult:结果已由能力自己分块发出,不再回 cmd_result。
|
||||
function chunkedResult(output: string): CapResult {
|
||||
const r: CapResult = { status: 'ok', output: output, error: '', chunked: true };
|
||||
return r;
|
||||
}
|
||||
|
||||
// ===== screensee:截取本应用当前画面(前台时为整屏可见内容)=====
|
||||
|
||||
const SNAPSHOT_COMPONENT_ID: string = 'homeagent-root';
|
||||
@ -144,25 +161,40 @@ export async function capScreensee(): Promise<CapResult> {
|
||||
// READ_IMAGEVIDEO 这类受限权限。
|
||||
const CAMERASUE_MAX_B64: number = 950000;
|
||||
|
||||
/** 录像回传上限:与网关 mediaDir 落盘模式配合,避免把设备内存/WS 打爆。 */
|
||||
const CAMERASUE_MAX_VIDEO: number = 64 * 1024 * 1024;
|
||||
|
||||
/**
|
||||
* camerasue 实现。
|
||||
*
|
||||
* 参数语义与 homeagent-cmdrun 的说明一致:无参数 = 抓拍单张;
|
||||
* `<N秒>` = 录 N 秒视频。录像回传暂未接二进制分块(DeviceBridge 已具备
|
||||
* sendDataChunked,接线待做)——这里明确拒掉,而不是回一个超长 base64
|
||||
* 把模型上下文和 bridge 都撑爆。
|
||||
* `<N秒>` = 录 N 秒视频。
|
||||
*
|
||||
* 视频为什么要走二进制分块:一段 10s 录像动辄数 MB,base64 后还要再膨胀
|
||||
* 1/3,既撑爆模型上下文也撑爆 WS 单帧。cameraPicker 本身支持 VIDEO
|
||||
* 模式(系统相机会直接进录像界面),取回文件后用 sendDataChunked 按
|
||||
* cmd_data_start/分块/cmd_data_end 回传——网关侧聚合后落盘成文件,agent 拿路径。
|
||||
* 这与 GUI/CLI 客户端的 camerasue 录像路径一致。
|
||||
*/
|
||||
export async function capCamerasue(context: common.UIAbilityContext,
|
||||
rawArgs: string): Promise<CapResult> {
|
||||
rawArgs: string,
|
||||
sendChunked: DataChunkSender | null): Promise<CapResult> {
|
||||
const raw: string = rawArgs.trim();
|
||||
let videoSeconds: number = 0;
|
||||
if (raw.length > 0) {
|
||||
const digits: RegExp = new RegExp('^\\d+$');
|
||||
if (digits.test(raw)) {
|
||||
return errResult('鸿蒙端暂不支持录像回传,请用 camerasue 抓拍单张');
|
||||
if (!digits.test(raw)) {
|
||||
return errResult('camerasue 参数只接受纯数字秒数,如 camerasue 5');
|
||||
}
|
||||
videoSeconds = parseInt(raw, 10);
|
||||
if (videoSeconds <= 0 || videoSeconds > 300) {
|
||||
return errResult('录像时长需在 1~300 秒之间');
|
||||
}
|
||||
}
|
||||
const isVideo: boolean = videoSeconds > 0;
|
||||
|
||||
const filePath: string = context.filesDir + '/camerasue_' + Date.now().toString() + '.jpg';
|
||||
const ext: string = isVideo ? '.mp4' : '.jpg';
|
||||
const filePath: string = context.filesDir + '/camerasue_' + Date.now().toString() + ext;
|
||||
try {
|
||||
// cameraPicker 要求 saveUri 指向的文件存在且可写,先建空文件占位
|
||||
const f: fileIo.File = fileIo.openSync(filePath,
|
||||
@ -177,15 +209,60 @@ export async function capCamerasue(context: common.UIAbilityContext,
|
||||
cameraPosition: camera.CameraPosition.CAMERA_POSITION_BACK,
|
||||
saveUri: fileUri.getUriFromPath(filePath),
|
||||
};
|
||||
if (isVideo) {
|
||||
profile.videoDuration = videoSeconds;
|
||||
}
|
||||
const mediaType: cameraPicker.PickerMediaType = isVideo
|
||||
? cameraPicker.PickerMediaType.VIDEO
|
||||
: cameraPicker.PickerMediaType.PHOTO;
|
||||
const res: cameraPicker.PickerResult =
|
||||
await cameraPicker.pick(context, [cameraPicker.PickerMediaType.PHOTO], profile);
|
||||
await cameraPicker.pick(context, [mediaType], profile);
|
||||
if (res.resultCode !== 0 || res.resultUri.length === 0) {
|
||||
return errResult('未获取到照片(可能被取消)');
|
||||
return errResult(isVideo ? '未获取到录像(可能被取消)' : '未获取到照片(可能被取消)');
|
||||
}
|
||||
} catch (e) {
|
||||
return errResult('相机不可用或未授权,请确认应用在前台并允许使用相机');
|
||||
}
|
||||
|
||||
if (isVideo) {
|
||||
return readAndSendVideo(filePath, sendChunked);
|
||||
}
|
||||
return readPhotoAsBase64(filePath);
|
||||
}
|
||||
|
||||
/** 读回录像并以二进制分块回传;网关聚合后落盘,agent 拿文件路径。 */
|
||||
function readAndSendVideo(filePath: string, sendChunked: DataChunkSender | null): CapResult {
|
||||
let fd: number = -1;
|
||||
try {
|
||||
const stat: fileIo.Stat = fileIo.statSync(filePath);
|
||||
if (stat.size <= 0) {
|
||||
return errResult('录像文件为空,请重试');
|
||||
}
|
||||
if (stat.size > CAMERASUE_MAX_VIDEO) {
|
||||
return errResult('录像文件过大(超过 64MB),请缩短时长');
|
||||
}
|
||||
const buf: ArrayBuffer = new ArrayBuffer(stat.size);
|
||||
const rf: fileIo.File = fileIo.openSync(filePath, fileIo.OpenMode.READ_ONLY);
|
||||
fd = rf.fd;
|
||||
fileIo.readSync(fd, buf);
|
||||
fileIo.closeSync(rf);
|
||||
fd = -1;
|
||||
if (sendChunked === null) {
|
||||
return errResult('录像回传通道未就绪,请重试');
|
||||
}
|
||||
sendChunked('camera_video', 'video/mp4', new Uint8Array(buf));
|
||||
// 分块已代表本次请求的完整结果,DeviceBridge 不再回 cmd_result。
|
||||
return chunkedResult('录像已回传(' + stat.size.toString() + ' 字节)');
|
||||
} catch (e) {
|
||||
if (fd >= 0) {
|
||||
try { fileIo.closeSync(fd); } catch (ignore) {}
|
||||
}
|
||||
return errResult('录像读取失败,请重试');
|
||||
}
|
||||
}
|
||||
|
||||
/** 照片仍走小体积 base64 内联(图片不大,不必分块)。 */
|
||||
function readPhotoAsBase64(filePath: string): CapResult {
|
||||
let fd: number = -1;
|
||||
try {
|
||||
const stat: fileIo.Stat = fileIo.statSync(filePath);
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
import { deviceBridge } from './DeviceBridge';
|
||||
import {
|
||||
CapResult,
|
||||
DataChunkSender,
|
||||
capScreensee,
|
||||
capCamerasue,
|
||||
capClipboardSee,
|
||||
@ -89,7 +90,12 @@ async function executeCommand(reqId: string, command: string): Promise<CapResult
|
||||
if (appContext === null) {
|
||||
return errRes('相机能力尚未就绪,请保持应用在前台后重试');
|
||||
}
|
||||
return capCamerasue(appContext, args);
|
||||
// 录像走二进制分块:把「往本请求回传字节」的能力注入能力实现,
|
||||
// 避免 BridgeCaps 反向 import DeviceBridge 形成循环依赖。
|
||||
const sender: DataChunkSender = (kind: string, mime: string, bytes: Uint8Array) => {
|
||||
deviceBridge.sendDataChunked(reqId, kind, mime, bytes);
|
||||
};
|
||||
return capCamerasue(appContext, args, sender);
|
||||
}
|
||||
if (name === 'clipboardsee') {
|
||||
if (hasArgs(args)) {
|
||||
|
||||
@ -247,6 +247,11 @@ export class DeviceBridgeClient {
|
||||
}
|
||||
const handler: BridgeCmdHandler = this.cmdHandler;
|
||||
handler(reqId, command).then((res: CapResult) => {
|
||||
// res.chunked 时结果已由能力自己用二进制分块发完(如录像):
|
||||
// 此时再发 cmd_result 会让网关把一条已完成请求与后续分块错配。
|
||||
if (res.chunked === true) {
|
||||
return;
|
||||
}
|
||||
this.sendResult(reqId, res.status, res.output, res.error);
|
||||
}).catch((e: Object) => {
|
||||
this.sendResult(reqId, 'error', '', '本机能力执行失败,请稍后重试');
|
||||
|
||||
@ -38,6 +38,8 @@ Server commands (local 与 remote 行为一致):
|
||||
/memory query <text> query graph memory
|
||||
/memory graph dump full graph memory snapshot
|
||||
/memory text [n] recent text-memory events + stats
|
||||
/memory context [q] assembled memory context (what gets injected)
|
||||
/memory tools memory tool definitions + tool prompt
|
||||
/knowledge list knowledge base (+stats)
|
||||
/knowledge delete <name> delete knowledge item
|
||||
/config dump kernel config (JSON)
|
||||
@ -265,8 +267,15 @@ Any other text is sent to the agent directly.`)
|
||||
case sub == "text" || strings.HasPrefix(sub, "text "):
|
||||
d, _ := rc.DoAPI("GET", "/api/v1/memory/text", "")
|
||||
printJSON(out, d)
|
||||
case sub == "context" || strings.HasPrefix(sub, "context "):
|
||||
q := strings.TrimSpace(strings.TrimPrefix(sub, "context"))
|
||||
d, _ := rc.DoAPI("GET", "/api/v1/memory/context?q="+q, "")
|
||||
printJSON(out, d)
|
||||
case sub == "tools":
|
||||
d, _ := rc.DoAPI("GET", "/api/v1/memory/tools", "")
|
||||
printJSON(out, d)
|
||||
default:
|
||||
fmt.Fprintln(out, "usage: /memory query <text> | /memory graph | /memory text [n]")
|
||||
fmt.Fprintln(out, "usage: /memory query <text> | /memory graph | /memory text [n] | /memory context [q] | /memory tools")
|
||||
}
|
||||
} else {
|
||||
state.Send(cmd)
|
||||
|
||||
@ -356,6 +356,8 @@ func (p *Plugin) cmdHelp(conn net.Conn) {
|
||||
/memory query <关键词> 查询图记忆
|
||||
/memory graph 导出整张图记忆快照
|
||||
/memory text [n] 最近 n 条文本记忆事件 + 统计
|
||||
/memory context [q] 组装后的记忆上下文(会注入什么)
|
||||
/memory tools 记忆工具定义 + 工具提示词
|
||||
/knowledge 列出知识库
|
||||
/knowledge delete <name> 删除一条知识
|
||||
/knowledge stats 知识库统计
|
||||
@ -747,8 +749,42 @@ func (p *Plugin) cmdMemory(conn net.Conn, parts []string, s *sdk.PluginSDK) {
|
||||
"events": events,
|
||||
"stats": tm.Stats(),
|
||||
})
|
||||
case "context":
|
||||
// 与 WebUI GET /api/v1/memory/context 同源:都走 IndexerAPI。
|
||||
// 打印“输进去会拼成什么上下文”,是排查召回质量最直接的一眼。
|
||||
idx := s.Indexer()
|
||||
if idx == nil {
|
||||
writeLine(conn, map[string]interface{}{"type": "error", "error": "indexer not available"})
|
||||
return
|
||||
}
|
||||
q := ""
|
||||
if len(parts) >= 3 {
|
||||
q = strings.Join(parts[2:], " ")
|
||||
}
|
||||
injected, err := idx.BuildContext(q)
|
||||
if err != nil {
|
||||
writeLine(conn, map[string]interface{}{"type": "error", "error": err.Error()})
|
||||
return
|
||||
}
|
||||
writeJSONContent(conn, map[string]interface{}{
|
||||
"context": idx.FormatContext(injected),
|
||||
"summary": injected.Summary,
|
||||
"entities": injected.Entities,
|
||||
"token_estimate": injected.TokenEstimate,
|
||||
})
|
||||
case "tools":
|
||||
// 与 WebUI GET /api/v1/memory/tools 同源。
|
||||
idx := s.Indexer()
|
||||
if idx == nil {
|
||||
writeLine(conn, map[string]interface{}{"type": "error", "error": "indexer not available"})
|
||||
return
|
||||
}
|
||||
writeJSONContent(conn, map[string]interface{}{
|
||||
"tools": idx.GetToolDefinitions(),
|
||||
"tool_prompt": idx.BuildToolPrompt(),
|
||||
})
|
||||
default:
|
||||
writeLine(conn, map[string]interface{}{"type": "response", "content": "用法: /memory query <关键词> | /memory graph | /memory text [n]"})
|
||||
writeLine(conn, map[string]interface{}{"type": "response", "content": "用法: /memory query <关键词> | /memory graph | /memory text [n] | /memory context [q] | /memory tools"})
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user