mirror of
https://gitcode.com/JianFeeeee/HomeAgent.git
synced 2026-10-14 19:43:37 +00:00
feat(ohos+waiter): 补 camerasue 能力;修 waiter 本地斜杠命令被当聊天文本
ohos camerasue: - LOCAL_DEVICE_CAPS 增 camerasue;BridgeRouter 增 camerasue 路由。 - 实现走系统相机选择器 cameraPicker(三方应用无法无界面直驱摄像头), 结果落应用沙箱(saveUri=filesDir),不写系统媒体库、不需 READ_IMAGEVIDEO。 - 录像(camerasue <N秒>)明确返回“暂不支持录像回传”,而不是回一个超长 base64 撑爆上下文;二进制分块回传待接线。 waiter 本地/远端命令语义统一: - 修 bug:/settings、/settings set、/plugin install/remove/info、 /memory query、/knowledge delete 本地分支发的是 cmd[1:](丢掉前导 /), 于是 CLI 插件不认、被当成聊天文本丢给 LLM。改为原样发(保留 /)。 - 补 /stop、/interrupt:/help 一直写着但 handleBuiltin 没实现,会落到 “当普通消息发给 Agent”。远端走 POST /chat/interrupt,本地交给 CLI 插件。 - 补 /plugin disable|enable:远端插件管理 REST 动作,本地 CLI 插件。 - /help 文案改为“local 与 remote 行为一致”并列出新命令。
This commit is contained in:
@ -5,6 +5,8 @@ import { deviceInfo } from '@kit.BasicServicesKit';
|
|||||||
import { textToSpeech } from '@kit.CoreSpeechKit';
|
import { textToSpeech } from '@kit.CoreSpeechKit';
|
||||||
import { componentSnapshot } from '@kit.ArkUI';
|
import { componentSnapshot } from '@kit.ArkUI';
|
||||||
import { abilityAccessCtrl, common, PermissionRequestResult, Permissions } from '@kit.AbilityKit';
|
import { abilityAccessCtrl, common, PermissionRequestResult, Permissions } from '@kit.AbilityKit';
|
||||||
|
import { camera, cameraPicker } from '@kit.CameraKit';
|
||||||
|
import { fileIo, fileUri } from '@kit.CoreFileKit';
|
||||||
import { appVersion } from './AppVersion';
|
import { appVersion } from './AppVersion';
|
||||||
|
|
||||||
// ===== 能力结果 =====
|
// ===== 能力结果 =====
|
||||||
@ -18,6 +20,7 @@ export const LOCAL_DEVICE_CAPS: string[] = [
|
|||||||
'clipboardsee',
|
'clipboardsee',
|
||||||
'clipboardsue',
|
'clipboardsue',
|
||||||
'speakeruse',
|
'speakeruse',
|
||||||
|
'camerasue',
|
||||||
];
|
];
|
||||||
|
|
||||||
export interface CapResult {
|
export interface CapResult {
|
||||||
@ -130,6 +133,82 @@ export async function capScreensee(): Promise<CapResult> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ===== camerasue:系统相机抓拍 =====
|
||||||
|
//
|
||||||
|
// 与桌面/CLI 端的实现路径不同:鸿蒙三方应用不能无界面地直接驱动摄像头
|
||||||
|
// (CameraKit 需要预览 surface + CAMERA 权限,且后台采集受限),能拿到
|
||||||
|
// “用户正在拍的这一张”的合规路径是系统相机选择器 cameraPicker —— 由系统
|
||||||
|
// 相机完成采集,本应用只取回结果文件。语义与桌面端一致:现在给 agent 拍一张。
|
||||||
|
//
|
||||||
|
// 结果落在应用沙箱(saveUri 指向 filesDir),不写系统媒体库,也就不需要
|
||||||
|
// READ_IMAGEVIDEO 这类受限权限。
|
||||||
|
const CAMERASUE_MAX_B64: number = 950000;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* camerasue 实现。
|
||||||
|
*
|
||||||
|
* 参数语义与 homeagent-cmdrun 的说明一致:无参数 = 抓拍单张;
|
||||||
|
* `<N秒>` = 录 N 秒视频。录像回传暂未接二进制分块(DeviceBridge 已具备
|
||||||
|
* sendDataChunked,接线待做)——这里明确拒掉,而不是回一个超长 base64
|
||||||
|
* 把模型上下文和 bridge 都撑爆。
|
||||||
|
*/
|
||||||
|
export async function capCamerasue(context: common.UIAbilityContext,
|
||||||
|
rawArgs: string): Promise<CapResult> {
|
||||||
|
const raw: string = rawArgs.trim();
|
||||||
|
if (raw.length > 0) {
|
||||||
|
const digits: RegExp = new RegExp('^\\d+$');
|
||||||
|
if (digits.test(raw)) {
|
||||||
|
return errResult('鸿蒙端暂不支持录像回传,请用 camerasue 抓拍单张');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const filePath: string = context.filesDir + '/camerasue_' + Date.now().toString() + '.jpg';
|
||||||
|
try {
|
||||||
|
// cameraPicker 要求 saveUri 指向的文件存在且可写,先建空文件占位
|
||||||
|
const f: fileIo.File = fileIo.openSync(filePath,
|
||||||
|
fileIo.OpenMode.READ_WRITE | fileIo.OpenMode.CREATE);
|
||||||
|
fileIo.closeSync(f);
|
||||||
|
} catch (e) {
|
||||||
|
return errResult('无法创建相机输出文件');
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const profile: cameraPicker.PickerProfile = {
|
||||||
|
cameraPosition: camera.CameraPosition.CAMERA_POSITION_BACK,
|
||||||
|
saveUri: fileUri.getUriFromPath(filePath),
|
||||||
|
};
|
||||||
|
const res: cameraPicker.PickerResult =
|
||||||
|
await cameraPicker.pick(context, [cameraPicker.PickerMediaType.PHOTO], profile);
|
||||||
|
if (res.resultCode !== 0 || res.resultUri.length === 0) {
|
||||||
|
return errResult('未获取到照片(可能被取消)');
|
||||||
|
}
|
||||||
|
} catch (e) {
|
||||||
|
return errResult('相机不可用或未授权,请确认应用在前台并允许使用相机');
|
||||||
|
}
|
||||||
|
|
||||||
|
let fd: number = -1;
|
||||||
|
try {
|
||||||
|
const stat: fileIo.Stat = fileIo.statSync(filePath);
|
||||||
|
const buf: ArrayBuffer = new ArrayBuffer(stat.size);
|
||||||
|
const rf: fileIo.File = fileIo.openSync(filePath, fileIo.OpenMode.READ_ONLY);
|
||||||
|
fd = rf.fd;
|
||||||
|
fileIo.readSync(fd, buf);
|
||||||
|
fileIo.closeSync(rf);
|
||||||
|
fd = -1;
|
||||||
|
const helper: util.Base64Helper = new util.Base64Helper();
|
||||||
|
const b64: string = helper.encodeToStringSync(new Uint8Array(buf));
|
||||||
|
if (b64.length > CAMERASUE_MAX_B64) {
|
||||||
|
return errResult('照片数据过大,请降低分辨率后重试');
|
||||||
|
}
|
||||||
|
return okResult('data:image/jpeg;base64,' + b64);
|
||||||
|
} catch (e) {
|
||||||
|
if (fd >= 0) {
|
||||||
|
try { fileIo.closeSync(fd); } catch (ignore) {}
|
||||||
|
}
|
||||||
|
return errResult('照片读取失败,请重试');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// ===== clipboardsee / clipboardsue =====
|
// ===== clipboardsee / clipboardsue =====
|
||||||
|
|
||||||
const CLIPBOARD_PERMISSIONS: Array<Permissions> = ['ohos.permission.READ_PASTEBOARD'];
|
const CLIPBOARD_PERMISSIONS: Array<Permissions> = ['ohos.permission.READ_PASTEBOARD'];
|
||||||
|
|||||||
@ -2,6 +2,7 @@ import { deviceBridge } from './DeviceBridge';
|
|||||||
import {
|
import {
|
||||||
CapResult,
|
CapResult,
|
||||||
capScreensee,
|
capScreensee,
|
||||||
|
capCamerasue,
|
||||||
capClipboardSee,
|
capClipboardSee,
|
||||||
capClipboardsue,
|
capClipboardsue,
|
||||||
capSpeakerUse,
|
capSpeakerUse,
|
||||||
@ -84,6 +85,12 @@ async function executeCommand(reqId: string, command: string): Promise<CapResult
|
|||||||
}
|
}
|
||||||
return errRes('展示界面尚未就绪,请保持应用在前台后重试');
|
return errRes('展示界面尚未就绪,请保持应用在前台后重试');
|
||||||
}
|
}
|
||||||
|
if (name === 'camerasue') {
|
||||||
|
if (appContext === null) {
|
||||||
|
return errRes('相机能力尚未就绪,请保持应用在前台后重试');
|
||||||
|
}
|
||||||
|
return capCamerasue(appContext, args);
|
||||||
|
}
|
||||||
if (name === 'clipboardsee') {
|
if (name === 'clipboardsee') {
|
||||||
if (hasArgs(args)) {
|
if (hasArgs(args)) {
|
||||||
return errRes('clipboardsee 不接受额外参数');
|
return errRes('clipboardsee 不接受额外参数');
|
||||||
|
|||||||
@ -24,7 +24,7 @@ func handleBuiltin(cmd string, cfg *Config, state *State, reconnect func(), out
|
|||||||
/conn use <name> switch to saved connection
|
/conn use <name> switch to saved connection
|
||||||
/conn del <name> delete saved connection
|
/conn del <name> delete saved connection
|
||||||
|
|
||||||
Server commands (sent to agent):
|
Server commands (local 与 remote 行为一致):
|
||||||
/status system status
|
/status system status
|
||||||
/kernel kernel status
|
/kernel kernel status
|
||||||
/settings [prefix] list settings
|
/settings [prefix] list settings
|
||||||
@ -32,6 +32,8 @@ Server commands (sent to agent):
|
|||||||
/plugin list list installed plugins
|
/plugin list list installed plugins
|
||||||
/plugin install <url> install plugin
|
/plugin install <url> install plugin
|
||||||
/plugin remove <name> remove plugin
|
/plugin remove <name> remove plugin
|
||||||
|
/plugin disable <name> disable plugin
|
||||||
|
/plugin enable <name> enable plugin
|
||||||
/plugin info <name> plugin details
|
/plugin info <name> plugin details
|
||||||
/memory query <text> query graph memory
|
/memory query <text> query graph memory
|
||||||
/knowledge list knowledge base
|
/knowledge list knowledge base
|
||||||
@ -54,6 +56,26 @@ Any other text is sent to the agent directly.`)
|
|||||||
reconnect()
|
reconnect()
|
||||||
return true
|
return true
|
||||||
|
|
||||||
|
// /stop 与 /interrupt:取消当前生成(可附带一句新指令)。
|
||||||
|
// 之前 /help 里写着这条命令,但 handleBuiltin 根本没有对应 case,
|
||||||
|
// 于是它像普通文本一样被发给了 Agent。
|
||||||
|
// 本地交给 CLI 插件(内核优先级 L3),远端走 WebUI 的 chat/interrupt
|
||||||
|
// (内核优先级 L4)。两条路都是“真中断”,不是发一句话。
|
||||||
|
case cmd == "/stop" || cmd == "/interrupt" ||
|
||||||
|
strings.HasPrefix(cmd, "/stop ") || strings.HasPrefix(cmd, "/interrupt "):
|
||||||
|
msg := stopMessage(cmd)
|
||||||
|
if rc := state.RemoteConn(); rc != nil {
|
||||||
|
body := fmt.Sprintf(`{"message":%q}`, msg)
|
||||||
|
if _, err := rc.DoAPI("POST", "/api/v1/chat/interrupt", body); err != nil {
|
||||||
|
fmt.Fprintf(out, "interrupt failed: %v\n", err)
|
||||||
|
} else {
|
||||||
|
fmt.Fprintln(out, "interrupt sent")
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
state.Send(cmd)
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
|
||||||
case strings.HasPrefix(cmd, "/connect "):
|
case strings.HasPrefix(cmd, "/connect "):
|
||||||
cfg.Socket = strings.TrimSpace(cmd[9:])
|
cfg.Socket = strings.TrimSpace(cmd[9:])
|
||||||
cfg.Remote = ""
|
cfg.Remote = ""
|
||||||
@ -143,7 +165,7 @@ Any other text is sent to the agent directly.`)
|
|||||||
rc.DoAPI("PUT", "/api/v1/settings", body)
|
rc.DoAPI("PUT", "/api/v1/settings", body)
|
||||||
fmt.Fprintln(out, "ok")
|
fmt.Fprintln(out, "ok")
|
||||||
} else {
|
} else {
|
||||||
state.Send(cmd[1:])
|
state.Send(cmd)
|
||||||
}
|
}
|
||||||
return true
|
return true
|
||||||
|
|
||||||
@ -152,7 +174,7 @@ Any other text is sent to the agent directly.`)
|
|||||||
d, _ := rc.DoAPI("GET", "/api/v1/settings", "")
|
d, _ := rc.DoAPI("GET", "/api/v1/settings", "")
|
||||||
printJSON(out, d)
|
printJSON(out, d)
|
||||||
} else {
|
} else {
|
||||||
state.Send(cmd[1:])
|
state.Send(cmd)
|
||||||
}
|
}
|
||||||
return true
|
return true
|
||||||
|
|
||||||
@ -172,7 +194,7 @@ Any other text is sent to the agent directly.`)
|
|||||||
d, _ := rc.DoAPI("POST", "/api/v1/plugins", body)
|
d, _ := rc.DoAPI("POST", "/api/v1/plugins", body)
|
||||||
printJSON(out, d)
|
printJSON(out, d)
|
||||||
} else {
|
} else {
|
||||||
state.Send(cmd[1:])
|
state.Send(cmd)
|
||||||
}
|
}
|
||||||
return true
|
return true
|
||||||
|
|
||||||
@ -182,7 +204,7 @@ Any other text is sent to the agent directly.`)
|
|||||||
d, _ := rc.DoAPI("DELETE", "/api/v1/plugins/"+name, "")
|
d, _ := rc.DoAPI("DELETE", "/api/v1/plugins/"+name, "")
|
||||||
printJSON(out, d)
|
printJSON(out, d)
|
||||||
} else {
|
} else {
|
||||||
state.Send(cmd[1:])
|
state.Send(cmd)
|
||||||
}
|
}
|
||||||
return true
|
return true
|
||||||
|
|
||||||
@ -192,7 +214,27 @@ Any other text is sent to the agent directly.`)
|
|||||||
d, _ := rc.DoAPI("GET", "/api/v1/plugins/"+name, "")
|
d, _ := rc.DoAPI("GET", "/api/v1/plugins/"+name, "")
|
||||||
printJSON(out, d)
|
printJSON(out, d)
|
||||||
} else {
|
} else {
|
||||||
state.Send(cmd[1:])
|
state.Send(cmd)
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
|
||||||
|
// disable/enable:本地由 CLI 插件处理,远端走插件管理 REST 动作接口。
|
||||||
|
case strings.HasPrefix(cmd, "/plugin disable ") || strings.HasPrefix(cmd, "/plugin enable "):
|
||||||
|
verb := "disable"
|
||||||
|
name := strings.TrimSpace(cmd[16:])
|
||||||
|
if strings.HasPrefix(cmd, "/plugin enable ") {
|
||||||
|
verb = "enable"
|
||||||
|
name = strings.TrimSpace(cmd[15:])
|
||||||
|
}
|
||||||
|
if name == "" {
|
||||||
|
fmt.Fprintln(out, "usage: /plugin disable|enable <name>")
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if rc := state.RemoteConn(); rc != nil {
|
||||||
|
d, _ := rc.DoAPI("POST", "/api/v1/plugins/"+name+"/"+verb, "")
|
||||||
|
printJSON(out, d)
|
||||||
|
} else {
|
||||||
|
state.Send(cmd)
|
||||||
}
|
}
|
||||||
return true
|
return true
|
||||||
|
|
||||||
@ -202,7 +244,7 @@ Any other text is sent to the agent directly.`)
|
|||||||
d, _ := rc.DoAPI("GET", "/api/v1/memory?query="+q, "")
|
d, _ := rc.DoAPI("GET", "/api/v1/memory?query="+q, "")
|
||||||
printJSON(out, d)
|
printJSON(out, d)
|
||||||
} else {
|
} else {
|
||||||
state.Send(cmd[1:])
|
state.Send(cmd)
|
||||||
}
|
}
|
||||||
return true
|
return true
|
||||||
|
|
||||||
@ -212,7 +254,7 @@ Any other text is sent to the agent directly.`)
|
|||||||
d, _ := rc.DoAPI("DELETE", "/api/v1/knowledge/"+name, "")
|
d, _ := rc.DoAPI("DELETE", "/api/v1/knowledge/"+name, "")
|
||||||
printJSON(out, d)
|
printJSON(out, d)
|
||||||
} else {
|
} else {
|
||||||
state.Send(cmd[1:])
|
state.Send(cmd)
|
||||||
}
|
}
|
||||||
return true
|
return true
|
||||||
|
|
||||||
@ -239,6 +281,16 @@ Any other text is sent to the agent directly.`)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// stopMessage 从 /stop 或 /interrupt 行里取出可选的中断附带消息(空串=纯取消)。
|
||||||
|
func stopMessage(cmd string) string {
|
||||||
|
for _, prefix := range []string{"/interrupt", "/stop"} {
|
||||||
|
if strings.HasPrefix(cmd, prefix) {
|
||||||
|
return strings.TrimSpace(strings.TrimPrefix(cmd, prefix))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
func printJSON(out io.Writer, d map[string]interface{}) {
|
func printJSON(out io.Writer, d map[string]interface{}) {
|
||||||
if d == nil {
|
if d == nil {
|
||||||
fmt.Fprintln(out, "(no data)")
|
fmt.Fprintln(out, "(no data)")
|
||||||
|
|||||||
Reference in New Issue
Block a user