mirror of
https://gitcode.com/JianFeeeee/HomeAgent.git
synced 2026-09-28 21:33:05 +00:00
feat(ohos+waiter): 补 camerasue 能力;修 waiter 本地斜杠命令被当聊天文本
ohos camerasue: - LOCAL_DEVICE_CAPS 增 camerasue;BridgeRouter 增 camerasue 路由。 - 实现走系统相机选择器 cameraPicker(三方应用无法无界面直驱摄像头), 结果落应用沙箱(saveUri=filesDir),不写系统媒体库、不需 READ_IMAGEVIDEO。 - 录像(camerasue <N秒>)明确返回“暂不支持录像回传”,而不是回一个超长 base64 撑爆上下文;二进制分块回传待接线。 waiter 本地/远端命令语义统一: - 修 bug:/settings、/settings set、/plugin install/remove/info、 /memory query、/knowledge delete 本地分支发的是 cmd[1:](丢掉前导 /), 于是 CLI 插件不认、被当成聊天文本丢给 LLM。改为原样发(保留 /)。 - 补 /stop、/interrupt:/help 一直写着但 handleBuiltin 没实现,会落到 “当普通消息发给 Agent”。远端走 POST /chat/interrupt,本地交给 CLI 插件。 - 补 /plugin disable|enable:远端插件管理 REST 动作,本地 CLI 插件。 - /help 文案改为“local 与 remote 行为一致”并列出新命令。
This commit is contained in:
@ -5,6 +5,8 @@ import { deviceInfo } from '@kit.BasicServicesKit';
|
||||
import { textToSpeech } from '@kit.CoreSpeechKit';
|
||||
import { componentSnapshot } from '@kit.ArkUI';
|
||||
import { abilityAccessCtrl, common, PermissionRequestResult, Permissions } from '@kit.AbilityKit';
|
||||
import { camera, cameraPicker } from '@kit.CameraKit';
|
||||
import { fileIo, fileUri } from '@kit.CoreFileKit';
|
||||
import { appVersion } from './AppVersion';
|
||||
|
||||
// ===== 能力结果 =====
|
||||
@ -18,6 +20,7 @@ export const LOCAL_DEVICE_CAPS: string[] = [
|
||||
'clipboardsee',
|
||||
'clipboardsue',
|
||||
'speakeruse',
|
||||
'camerasue',
|
||||
];
|
||||
|
||||
export interface CapResult {
|
||||
@ -130,6 +133,82 @@ export async function capScreensee(): Promise<CapResult> {
|
||||
}
|
||||
}
|
||||
|
||||
// ===== camerasue:系统相机抓拍 =====
|
||||
//
|
||||
// 与桌面/CLI 端的实现路径不同:鸿蒙三方应用不能无界面地直接驱动摄像头
|
||||
// (CameraKit 需要预览 surface + CAMERA 权限,且后台采集受限),能拿到
|
||||
// “用户正在拍的这一张”的合规路径是系统相机选择器 cameraPicker —— 由系统
|
||||
// 相机完成采集,本应用只取回结果文件。语义与桌面端一致:现在给 agent 拍一张。
|
||||
//
|
||||
// 结果落在应用沙箱(saveUri 指向 filesDir),不写系统媒体库,也就不需要
|
||||
// READ_IMAGEVIDEO 这类受限权限。
|
||||
const CAMERASUE_MAX_B64: number = 950000;
|
||||
|
||||
/**
|
||||
* camerasue 实现。
|
||||
*
|
||||
* 参数语义与 homeagent-cmdrun 的说明一致:无参数 = 抓拍单张;
|
||||
* `<N秒>` = 录 N 秒视频。录像回传暂未接二进制分块(DeviceBridge 已具备
|
||||
* sendDataChunked,接线待做)——这里明确拒掉,而不是回一个超长 base64
|
||||
* 把模型上下文和 bridge 都撑爆。
|
||||
*/
|
||||
export async function capCamerasue(context: common.UIAbilityContext,
|
||||
rawArgs: string): Promise<CapResult> {
|
||||
const raw: string = rawArgs.trim();
|
||||
if (raw.length > 0) {
|
||||
const digits: RegExp = new RegExp('^\\d+$');
|
||||
if (digits.test(raw)) {
|
||||
return errResult('鸿蒙端暂不支持录像回传,请用 camerasue 抓拍单张');
|
||||
}
|
||||
}
|
||||
|
||||
const filePath: string = context.filesDir + '/camerasue_' + Date.now().toString() + '.jpg';
|
||||
try {
|
||||
// cameraPicker 要求 saveUri 指向的文件存在且可写,先建空文件占位
|
||||
const f: fileIo.File = fileIo.openSync(filePath,
|
||||
fileIo.OpenMode.READ_WRITE | fileIo.OpenMode.CREATE);
|
||||
fileIo.closeSync(f);
|
||||
} catch (e) {
|
||||
return errResult('无法创建相机输出文件');
|
||||
}
|
||||
|
||||
try {
|
||||
const profile: cameraPicker.PickerProfile = {
|
||||
cameraPosition: camera.CameraPosition.CAMERA_POSITION_BACK,
|
||||
saveUri: fileUri.getUriFromPath(filePath),
|
||||
};
|
||||
const res: cameraPicker.PickerResult =
|
||||
await cameraPicker.pick(context, [cameraPicker.PickerMediaType.PHOTO], profile);
|
||||
if (res.resultCode !== 0 || res.resultUri.length === 0) {
|
||||
return errResult('未获取到照片(可能被取消)');
|
||||
}
|
||||
} catch (e) {
|
||||
return errResult('相机不可用或未授权,请确认应用在前台并允许使用相机');
|
||||
}
|
||||
|
||||
let fd: number = -1;
|
||||
try {
|
||||
const stat: fileIo.Stat = fileIo.statSync(filePath);
|
||||
const buf: ArrayBuffer = new ArrayBuffer(stat.size);
|
||||
const rf: fileIo.File = fileIo.openSync(filePath, fileIo.OpenMode.READ_ONLY);
|
||||
fd = rf.fd;
|
||||
fileIo.readSync(fd, buf);
|
||||
fileIo.closeSync(rf);
|
||||
fd = -1;
|
||||
const helper: util.Base64Helper = new util.Base64Helper();
|
||||
const b64: string = helper.encodeToStringSync(new Uint8Array(buf));
|
||||
if (b64.length > CAMERASUE_MAX_B64) {
|
||||
return errResult('照片数据过大,请降低分辨率后重试');
|
||||
}
|
||||
return okResult('data:image/jpeg;base64,' + b64);
|
||||
} catch (e) {
|
||||
if (fd >= 0) {
|
||||
try { fileIo.closeSync(fd); } catch (ignore) {}
|
||||
}
|
||||
return errResult('照片读取失败,请重试');
|
||||
}
|
||||
}
|
||||
|
||||
// ===== clipboardsee / clipboardsue =====
|
||||
|
||||
const CLIPBOARD_PERMISSIONS: Array<Permissions> = ['ohos.permission.READ_PASTEBOARD'];
|
||||
|
||||
@ -2,6 +2,7 @@ import { deviceBridge } from './DeviceBridge';
|
||||
import {
|
||||
CapResult,
|
||||
capScreensee,
|
||||
capCamerasue,
|
||||
capClipboardSee,
|
||||
capClipboardsue,
|
||||
capSpeakerUse,
|
||||
@ -84,6 +85,12 @@ async function executeCommand(reqId: string, command: string): Promise<CapResult
|
||||
}
|
||||
return errRes('展示界面尚未就绪,请保持应用在前台后重试');
|
||||
}
|
||||
if (name === 'camerasue') {
|
||||
if (appContext === null) {
|
||||
return errRes('相机能力尚未就绪,请保持应用在前台后重试');
|
||||
}
|
||||
return capCamerasue(appContext, args);
|
||||
}
|
||||
if (name === 'clipboardsee') {
|
||||
if (hasArgs(args)) {
|
||||
return errRes('clipboardsee 不接受额外参数');
|
||||
|
||||
@ -24,7 +24,7 @@ func handleBuiltin(cmd string, cfg *Config, state *State, reconnect func(), out
|
||||
/conn use <name> switch to saved connection
|
||||
/conn del <name> delete saved connection
|
||||
|
||||
Server commands (sent to agent):
|
||||
Server commands (local 与 remote 行为一致):
|
||||
/status system status
|
||||
/kernel kernel status
|
||||
/settings [prefix] list settings
|
||||
@ -32,6 +32,8 @@ Server commands (sent to agent):
|
||||
/plugin list list installed plugins
|
||||
/plugin install <url> install plugin
|
||||
/plugin remove <name> remove plugin
|
||||
/plugin disable <name> disable plugin
|
||||
/plugin enable <name> enable plugin
|
||||
/plugin info <name> plugin details
|
||||
/memory query <text> query graph memory
|
||||
/knowledge list knowledge base
|
||||
@ -54,6 +56,26 @@ Any other text is sent to the agent directly.`)
|
||||
reconnect()
|
||||
return true
|
||||
|
||||
// /stop 与 /interrupt:取消当前生成(可附带一句新指令)。
|
||||
// 之前 /help 里写着这条命令,但 handleBuiltin 根本没有对应 case,
|
||||
// 于是它像普通文本一样被发给了 Agent。
|
||||
// 本地交给 CLI 插件(内核优先级 L3),远端走 WebUI 的 chat/interrupt
|
||||
// (内核优先级 L4)。两条路都是“真中断”,不是发一句话。
|
||||
case cmd == "/stop" || cmd == "/interrupt" ||
|
||||
strings.HasPrefix(cmd, "/stop ") || strings.HasPrefix(cmd, "/interrupt "):
|
||||
msg := stopMessage(cmd)
|
||||
if rc := state.RemoteConn(); rc != nil {
|
||||
body := fmt.Sprintf(`{"message":%q}`, msg)
|
||||
if _, err := rc.DoAPI("POST", "/api/v1/chat/interrupt", body); err != nil {
|
||||
fmt.Fprintf(out, "interrupt failed: %v\n", err)
|
||||
} else {
|
||||
fmt.Fprintln(out, "interrupt sent")
|
||||
}
|
||||
} else {
|
||||
state.Send(cmd)
|
||||
}
|
||||
return true
|
||||
|
||||
case strings.HasPrefix(cmd, "/connect "):
|
||||
cfg.Socket = strings.TrimSpace(cmd[9:])
|
||||
cfg.Remote = ""
|
||||
@ -143,7 +165,7 @@ Any other text is sent to the agent directly.`)
|
||||
rc.DoAPI("PUT", "/api/v1/settings", body)
|
||||
fmt.Fprintln(out, "ok")
|
||||
} else {
|
||||
state.Send(cmd[1:])
|
||||
state.Send(cmd)
|
||||
}
|
||||
return true
|
||||
|
||||
@ -152,7 +174,7 @@ Any other text is sent to the agent directly.`)
|
||||
d, _ := rc.DoAPI("GET", "/api/v1/settings", "")
|
||||
printJSON(out, d)
|
||||
} else {
|
||||
state.Send(cmd[1:])
|
||||
state.Send(cmd)
|
||||
}
|
||||
return true
|
||||
|
||||
@ -172,7 +194,7 @@ Any other text is sent to the agent directly.`)
|
||||
d, _ := rc.DoAPI("POST", "/api/v1/plugins", body)
|
||||
printJSON(out, d)
|
||||
} else {
|
||||
state.Send(cmd[1:])
|
||||
state.Send(cmd)
|
||||
}
|
||||
return true
|
||||
|
||||
@ -182,7 +204,7 @@ Any other text is sent to the agent directly.`)
|
||||
d, _ := rc.DoAPI("DELETE", "/api/v1/plugins/"+name, "")
|
||||
printJSON(out, d)
|
||||
} else {
|
||||
state.Send(cmd[1:])
|
||||
state.Send(cmd)
|
||||
}
|
||||
return true
|
||||
|
||||
@ -192,7 +214,27 @@ Any other text is sent to the agent directly.`)
|
||||
d, _ := rc.DoAPI("GET", "/api/v1/plugins/"+name, "")
|
||||
printJSON(out, d)
|
||||
} else {
|
||||
state.Send(cmd[1:])
|
||||
state.Send(cmd)
|
||||
}
|
||||
return true
|
||||
|
||||
// disable/enable:本地由 CLI 插件处理,远端走插件管理 REST 动作接口。
|
||||
case strings.HasPrefix(cmd, "/plugin disable ") || strings.HasPrefix(cmd, "/plugin enable "):
|
||||
verb := "disable"
|
||||
name := strings.TrimSpace(cmd[16:])
|
||||
if strings.HasPrefix(cmd, "/plugin enable ") {
|
||||
verb = "enable"
|
||||
name = strings.TrimSpace(cmd[15:])
|
||||
}
|
||||
if name == "" {
|
||||
fmt.Fprintln(out, "usage: /plugin disable|enable <name>")
|
||||
return true
|
||||
}
|
||||
if rc := state.RemoteConn(); rc != nil {
|
||||
d, _ := rc.DoAPI("POST", "/api/v1/plugins/"+name+"/"+verb, "")
|
||||
printJSON(out, d)
|
||||
} else {
|
||||
state.Send(cmd)
|
||||
}
|
||||
return true
|
||||
|
||||
@ -202,7 +244,7 @@ Any other text is sent to the agent directly.`)
|
||||
d, _ := rc.DoAPI("GET", "/api/v1/memory?query="+q, "")
|
||||
printJSON(out, d)
|
||||
} else {
|
||||
state.Send(cmd[1:])
|
||||
state.Send(cmd)
|
||||
}
|
||||
return true
|
||||
|
||||
@ -212,7 +254,7 @@ Any other text is sent to the agent directly.`)
|
||||
d, _ := rc.DoAPI("DELETE", "/api/v1/knowledge/"+name, "")
|
||||
printJSON(out, d)
|
||||
} else {
|
||||
state.Send(cmd[1:])
|
||||
state.Send(cmd)
|
||||
}
|
||||
return true
|
||||
|
||||
@ -239,6 +281,16 @@ Any other text is sent to the agent directly.`)
|
||||
}
|
||||
}
|
||||
|
||||
// stopMessage 从 /stop 或 /interrupt 行里取出可选的中断附带消息(空串=纯取消)。
|
||||
func stopMessage(cmd string) string {
|
||||
for _, prefix := range []string{"/interrupt", "/stop"} {
|
||||
if strings.HasPrefix(cmd, prefix) {
|
||||
return strings.TrimSpace(strings.TrimPrefix(cmd, prefix))
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func printJSON(out io.Writer, d map[string]interface{}) {
|
||||
if d == nil {
|
||||
fmt.Fprintln(out, "(no data)")
|
||||
|
||||
Reference in New Issue
Block a user