mirror of
https://gitcode.com/JianFeeeee/HomeAgent.git
synced 2026-09-22 18:08:04 +00:00
Compare commits
86 Commits
v1.0.0
...
v1.2.0-bet
| Author | SHA1 | Date | |
|---|---|---|---|
| bb16f19ea5 | |||
| 2c7ab6a187 | |||
| 9311b41dd3 | |||
| 996e70a64d | |||
| 68b773ad61 | |||
| 18324f6e80 | |||
| 8c5c0a25a6 | |||
| fdfba4eaa7 | |||
| 215804cf2c | |||
| ca5b04532d | |||
| 01232a66da | |||
| 3a5f6e1634 | |||
| f1091676f8 | |||
| eb4762a2b8 | |||
| 4d2028fbef | |||
| d4c5e808c7 | |||
| 5299e18cd8 | |||
| 6a3439a49c | |||
| ba0b5a1fe0 | |||
| bfdb395731 | |||
| d1959cbe80 | |||
| a37bc7333e | |||
| 1a02971f88 | |||
| 5836c2ce5c | |||
| dae01f9c06 | |||
| e44164f5bd | |||
| 96c1d7baae | |||
| 8e88ae789f | |||
| e985151da9 | |||
| b393b7072c | |||
| 50ba4a64cb | |||
| 15d912ef34 | |||
| 2a4315cfe4 | |||
| b09f459aea | |||
| e2672c56d6 | |||
| b71a88a471 | |||
| e218d0f100 | |||
| b096e8ba2c | |||
| f6f92a8a6e | |||
| c03de9878d | |||
| ca584bda3f | |||
| cf3c99233b | |||
| 2bc813be13 | |||
| 9cfdfc9de8 | |||
| 772a494223 | |||
| 5608abdf5a | |||
| cf098a1d4f | |||
| a8e3633174 | |||
| ad016e4410 | |||
| 7a328679da | |||
| 580d5f5501 | |||
| 6c2039f5c9 | |||
| 6f8056d236 | |||
| 98365f3a55 | |||
| 8c9d96e065 | |||
| e9856c8f65 | |||
| 0af38a29c6 | |||
| b889934bba | |||
| 8cd884027f | |||
| e4be9667ec | |||
| 1b4643089c | |||
| 8c6b593a54 | |||
| 687e5655bc | |||
| e8d12db871 | |||
| aac88ce5ee | |||
| a7b54fef7f | |||
| 387b28ce09 | |||
| 775d9e8a2e | |||
| 968b01f26e | |||
| 6190a5a587 | |||
| 28fc833a6f | |||
| 74a24f93d7 | |||
| 2879e76883 | |||
| 98dcb2556c | |||
| f855893d1c | |||
| 5c214cac23 | |||
| 9b7b2675da | |||
| e2d04a31a7 | |||
| aa3ff5a12f | |||
| e272c686d3 | |||
| 4ef3371701 | |||
| a82267484a | |||
| 6bf34e051b | |||
| 4f31f942a5 | |||
| 3907347cac | |||
| 09071dc235 |
7
.gitignore
vendored
7
.gitignore
vendored
@ -48,7 +48,12 @@ codegraph.json
|
|||||||
/adapters/
|
/adapters/
|
||||||
/knowledge/
|
/knowledge/
|
||||||
/memory/
|
/memory/
|
||||||
/scripts/
|
# 注:/scripts/ **不**忽略。它是作者维护的工具目录(模型导出、侧车、部署校验),
|
||||||
|
# 不是运行期产物:deploy/systemd/embed-sidecar.service 直接引用
|
||||||
|
# scripts/embed_sidecar.py,忽略它会让那份 unit 在别人的机器上指向不存在的文件。
|
||||||
|
# 只忽略其中的缓存。
|
||||||
|
/scripts/__pycache__/
|
||||||
|
__pycache__/
|
||||||
terminal_locked_log.txt
|
terminal_locked_log.txt
|
||||||
|
|
||||||
dist/
|
dist/
|
||||||
|
|||||||
661
LICENSE
Normal file
661
LICENSE
Normal file
@ -0,0 +1,661 @@
|
|||||||
|
GNU AFFERO GENERAL PUBLIC LICENSE
|
||||||
|
Version 3, 19 November 2007
|
||||||
|
|
||||||
|
Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
|
||||||
|
Everyone is permitted to copy and distribute verbatim copies
|
||||||
|
of this license document, but changing it is not allowed.
|
||||||
|
|
||||||
|
Preamble
|
||||||
|
|
||||||
|
The GNU Affero General Public License is a free, copyleft license for
|
||||||
|
software and other kinds of works, specifically designed to ensure
|
||||||
|
cooperation with the community in the case of network server software.
|
||||||
|
|
||||||
|
The licenses for most software and other practical works are designed
|
||||||
|
to take away your freedom to share and change the works. By contrast,
|
||||||
|
our General Public Licenses are intended to guarantee your freedom to
|
||||||
|
share and change all versions of a program--to make sure it remains free
|
||||||
|
software for all its users.
|
||||||
|
|
||||||
|
When we speak of free software, we are referring to freedom, not
|
||||||
|
price. Our General Public Licenses are designed to make sure that you
|
||||||
|
have the freedom to distribute copies of free software (and charge for
|
||||||
|
them if you wish), that you receive source code or can get it if you
|
||||||
|
want it, that you can change the software or use pieces of it in new
|
||||||
|
free programs, and that you know you can do these things.
|
||||||
|
|
||||||
|
Developers that use our General Public Licenses protect your rights
|
||||||
|
with two steps: (1) assert copyright on the software, and (2) offer
|
||||||
|
you this License which gives you legal permission to copy, distribute
|
||||||
|
and/or modify the software.
|
||||||
|
|
||||||
|
A secondary benefit of defending all users' freedom is that
|
||||||
|
improvements made in alternate versions of the program, if they
|
||||||
|
receive widespread use, become available for other developers to
|
||||||
|
incorporate. Many developers of free software are heartened and
|
||||||
|
encouraged by the resulting cooperation. However, in the case of
|
||||||
|
software used on network servers, this result may fail to come about.
|
||||||
|
The GNU General Public License permits making a modified version and
|
||||||
|
letting the public access it on a server without ever releasing its
|
||||||
|
source code to the public.
|
||||||
|
|
||||||
|
The GNU Affero General Public License is designed specifically to
|
||||||
|
ensure that, in such cases, the modified source code becomes available
|
||||||
|
to the community. It requires the operator of a network server to
|
||||||
|
provide the source code of the modified version running there to the
|
||||||
|
users of that server. Therefore, public use of a modified version, on
|
||||||
|
a publicly accessible server, gives the public access to the source
|
||||||
|
code of the modified version.
|
||||||
|
|
||||||
|
An older license, called the Affero General Public License and
|
||||||
|
published by Affero, was designed to accomplish similar goals. This is
|
||||||
|
a different license, not a version of the Affero GPL, but Affero has
|
||||||
|
released a new version of the Affero GPL which permits relicensing under
|
||||||
|
this license.
|
||||||
|
|
||||||
|
The precise terms and conditions for copying, distribution and
|
||||||
|
modification follow.
|
||||||
|
|
||||||
|
TERMS AND CONDITIONS
|
||||||
|
|
||||||
|
0. Definitions.
|
||||||
|
|
||||||
|
"This License" refers to version 3 of the GNU Affero General Public License.
|
||||||
|
|
||||||
|
"Copyright" also means copyright-like laws that apply to other kinds of
|
||||||
|
works, such as semiconductor masks.
|
||||||
|
|
||||||
|
"The Program" refers to any copyrightable work licensed under this
|
||||||
|
License. Each licensee is addressed as "you". "Licensees" and
|
||||||
|
"recipients" may be individuals or organizations.
|
||||||
|
|
||||||
|
To "modify" a work means to copy from or adapt all or part of the work
|
||||||
|
in a fashion requiring copyright permission, other than the making of an
|
||||||
|
exact copy. The resulting work is called a "modified version" of the
|
||||||
|
earlier work or a work "based on" the earlier work.
|
||||||
|
|
||||||
|
A "covered work" means either the unmodified Program or a work based
|
||||||
|
on the Program.
|
||||||
|
|
||||||
|
To "propagate" a work means to do anything with it that, without
|
||||||
|
permission, would make you directly or secondarily liable for
|
||||||
|
infringement under applicable copyright law, except executing it on a
|
||||||
|
computer or modifying a private copy. Propagation includes copying,
|
||||||
|
distribution (with or without modification), making available to the
|
||||||
|
public, and in some countries other activities as well.
|
||||||
|
|
||||||
|
To "convey" a work means any kind of propagation that enables other
|
||||||
|
parties to make or receive copies. Mere interaction with a user through
|
||||||
|
a computer network, with no transfer of a copy, is not conveying.
|
||||||
|
|
||||||
|
An interactive user interface displays "Appropriate Legal Notices"
|
||||||
|
to the extent that it includes a convenient and prominently visible
|
||||||
|
feature that (1) displays an appropriate copyright notice, and (2)
|
||||||
|
tells the user that there is no warranty for the work (except to the
|
||||||
|
extent that warranties are provided), that licensees may convey the
|
||||||
|
work under this License, and how to view a copy of this License. If
|
||||||
|
the interface presents a list of user commands or options, such as a
|
||||||
|
menu, a prominent item in the list meets this criterion.
|
||||||
|
|
||||||
|
1. Source Code.
|
||||||
|
|
||||||
|
The "source code" for a work means the preferred form of the work
|
||||||
|
for making modifications to it. "Object code" means any non-source
|
||||||
|
form of a work.
|
||||||
|
|
||||||
|
A "Standard Interface" means an interface that either is an official
|
||||||
|
standard defined by a recognized standards body, or, in the case of
|
||||||
|
interfaces specified for a particular programming language, one that
|
||||||
|
is widely used among developers working in that language.
|
||||||
|
|
||||||
|
The "System Libraries" of an executable work include anything, other
|
||||||
|
than the work as a whole, that (a) is included in the normal form of
|
||||||
|
packaging a Major Component, but which is not part of that Major
|
||||||
|
Component, and (b) serves only to enable use of the work with that
|
||||||
|
Major Component, or to implement a Standard Interface for which an
|
||||||
|
implementation is available to the public in source code form. A
|
||||||
|
"Major Component", in this context, means a major essential component
|
||||||
|
(kernel, window system, and so on) of the specific operating system
|
||||||
|
(if any) on which the executable work runs, or a compiler used to
|
||||||
|
produce the work, or an object code interpreter used to run it.
|
||||||
|
|
||||||
|
The "Corresponding Source" for a work in object code form means all
|
||||||
|
the source code needed to generate, install, and (for an executable
|
||||||
|
work) run the object code and to modify the work, including scripts to
|
||||||
|
control those activities. However, it does not include the work's
|
||||||
|
System Libraries, or general-purpose tools or generally available free
|
||||||
|
programs which are used unmodified in performing those activities but
|
||||||
|
which are not part of the work. For example, Corresponding Source
|
||||||
|
includes interface definition files associated with source files for
|
||||||
|
the work, and the source code for shared libraries and dynamically
|
||||||
|
linked subprograms that the work is specifically designed to require,
|
||||||
|
such as by intimate data communication or control flow between those
|
||||||
|
subprograms and other parts of the work.
|
||||||
|
|
||||||
|
The Corresponding Source need not include anything that users
|
||||||
|
can regenerate automatically from other parts of the Corresponding
|
||||||
|
Source.
|
||||||
|
|
||||||
|
The Corresponding Source for a work in source code form is that
|
||||||
|
same work.
|
||||||
|
|
||||||
|
2. Basic Permissions.
|
||||||
|
|
||||||
|
All rights granted under this License are granted for the term of
|
||||||
|
copyright on the Program, and are irrevocable provided the stated
|
||||||
|
conditions are met. This License explicitly affirms your unlimited
|
||||||
|
permission to run the unmodified Program. The output from running a
|
||||||
|
covered work is covered by this License only if the output, given its
|
||||||
|
content, constitutes a covered work. This License acknowledges your
|
||||||
|
rights of fair use or other equivalent, as provided by copyright law.
|
||||||
|
|
||||||
|
You may make, run and propagate covered works that you do not
|
||||||
|
convey, without conditions so long as your license otherwise remains
|
||||||
|
in force. You may convey covered works to others for the sole purpose
|
||||||
|
of having them make modifications exclusively for you, or provide you
|
||||||
|
with facilities for running those works, provided that you comply with
|
||||||
|
the terms of this License in conveying all material for which you do
|
||||||
|
not control copyright. Those thus making or running the covered works
|
||||||
|
for you must do so exclusively on your behalf, under your direction
|
||||||
|
and control, on terms that prohibit them from making any copies of
|
||||||
|
your copyrighted material outside their relationship with you.
|
||||||
|
|
||||||
|
Conveying under any other circumstances is permitted solely under
|
||||||
|
the conditions stated below. Sublicensing is not allowed; section 10
|
||||||
|
makes it unnecessary.
|
||||||
|
|
||||||
|
3. Protecting Users' Legal Rights From Anti-Circumvention Law.
|
||||||
|
|
||||||
|
No covered work shall be deemed part of an effective technological
|
||||||
|
measure under any applicable law fulfilling obligations under article
|
||||||
|
11 of the WIPO copyright treaty adopted on 20 December 1996, or
|
||||||
|
similar laws prohibiting or restricting circumvention of such
|
||||||
|
measures.
|
||||||
|
|
||||||
|
When you convey a covered work, you waive any legal power to forbid
|
||||||
|
circumvention of technological measures to the extent such circumvention
|
||||||
|
is effected by exercising rights under this License with respect to
|
||||||
|
the covered work, and you disclaim any intention to limit operation or
|
||||||
|
modification of the work as a means of enforcing, against the work's
|
||||||
|
users, your or third parties' legal rights to forbid circumvention of
|
||||||
|
technological measures.
|
||||||
|
|
||||||
|
4. Conveying Verbatim Copies.
|
||||||
|
|
||||||
|
You may convey verbatim copies of the Program's source code as you
|
||||||
|
receive it, in any medium, provided that you conspicuously and
|
||||||
|
appropriately publish on each copy an appropriate copyright notice;
|
||||||
|
keep intact all notices stating that this License and any
|
||||||
|
non-permissive terms added in accord with section 7 apply to the code;
|
||||||
|
keep intact all notices of the absence of any warranty; and give all
|
||||||
|
recipients a copy of this License along with the Program.
|
||||||
|
|
||||||
|
You may charge any price or no price for each copy that you convey,
|
||||||
|
and you may offer support or warranty protection for a fee.
|
||||||
|
|
||||||
|
5. Conveying Modified Source Versions.
|
||||||
|
|
||||||
|
You may convey a work based on the Program, or the modifications to
|
||||||
|
produce it from the Program, in the form of source code under the
|
||||||
|
terms of section 4, provided that you also meet all of these conditions:
|
||||||
|
|
||||||
|
a) The work must carry prominent notices stating that you modified
|
||||||
|
it, and giving a relevant date.
|
||||||
|
|
||||||
|
b) The work must carry prominent notices stating that it is
|
||||||
|
released under this License and any conditions added under section
|
||||||
|
7. This requirement modifies the requirement in section 4 to
|
||||||
|
"keep intact all notices".
|
||||||
|
|
||||||
|
c) You must license the entire work, as a whole, under this
|
||||||
|
License to anyone who comes into possession of a copy. This
|
||||||
|
License will therefore apply, along with any applicable section 7
|
||||||
|
additional terms, to the whole of the work, and all its parts,
|
||||||
|
regardless of how they are packaged. This License gives no
|
||||||
|
permission to license the work in any other way, but it does not
|
||||||
|
invalidate such permission if you have separately received it.
|
||||||
|
|
||||||
|
d) If the work has interactive user interfaces, each must display
|
||||||
|
Appropriate Legal Notices; however, if the Program has interactive
|
||||||
|
interfaces that do not display Appropriate Legal Notices, your
|
||||||
|
work need not make them do so.
|
||||||
|
|
||||||
|
A compilation of a covered work with other separate and independent
|
||||||
|
works, which are not by their nature extensions of the covered work,
|
||||||
|
and which are not combined with it such as to form a larger program,
|
||||||
|
in or on a volume of a storage or distribution medium, is called an
|
||||||
|
"aggregate" if the compilation and its resulting copyright are not
|
||||||
|
used to limit the access or legal rights of the compilation's users
|
||||||
|
beyond what the individual works permit. Inclusion of a covered work
|
||||||
|
in an aggregate does not cause this License to apply to the other
|
||||||
|
parts of the aggregate.
|
||||||
|
|
||||||
|
6. Conveying Non-Source Forms.
|
||||||
|
|
||||||
|
You may convey a covered work in object code form under the terms
|
||||||
|
of sections 4 and 5, provided that you also convey the
|
||||||
|
machine-readable Corresponding Source under the terms of this License,
|
||||||
|
in one of these ways:
|
||||||
|
|
||||||
|
a) Convey the object code in, or embodied in, a physical product
|
||||||
|
(including a physical distribution medium), accompanied by the
|
||||||
|
Corresponding Source fixed on a durable physical medium
|
||||||
|
customarily used for software interchange.
|
||||||
|
|
||||||
|
b) Convey the object code in, or embodied in, a physical product
|
||||||
|
(including a physical distribution medium), accompanied by a
|
||||||
|
written offer, valid for at least three years and valid for as
|
||||||
|
long as you offer spare parts or customer support for that product
|
||||||
|
model, to give anyone who possesses the object code either (1) a
|
||||||
|
copy of the Corresponding Source for all the software in the
|
||||||
|
product that is covered by this License, on a durable physical
|
||||||
|
medium customarily used for software interchange, for a price no
|
||||||
|
more than your reasonable cost of physically performing this
|
||||||
|
conveying of source, or (2) access to copy the
|
||||||
|
Corresponding Source from a network server at no charge.
|
||||||
|
|
||||||
|
c) Convey individual copies of the object code with a copy of the
|
||||||
|
written offer to provide the Corresponding Source. This
|
||||||
|
alternative is allowed only occasionally and noncommercially, and
|
||||||
|
only if you received the object code with such an offer, in accord
|
||||||
|
with subsection 6b.
|
||||||
|
|
||||||
|
d) Convey the object code by offering access from a designated
|
||||||
|
place (gratis or for a charge), and offer equivalent access to the
|
||||||
|
Corresponding Source in the same way through the same place at no
|
||||||
|
further charge. You need not require recipients to copy the
|
||||||
|
Corresponding Source along with the object code. If the place to
|
||||||
|
copy the object code is a network server, the Corresponding Source
|
||||||
|
may be on a different server (operated by you or a third party)
|
||||||
|
that supports equivalent copying facilities, provided you maintain
|
||||||
|
clear directions next to the object code saying where to find the
|
||||||
|
Corresponding Source. Regardless of what server hosts the
|
||||||
|
Corresponding Source, you remain obligated to ensure that it is
|
||||||
|
available for as long as needed to satisfy these requirements.
|
||||||
|
|
||||||
|
e) Convey the object code using peer-to-peer transmission, provided
|
||||||
|
you inform other peers where the object code and Corresponding
|
||||||
|
Source of the work are being offered to the general public at no
|
||||||
|
charge under subsection 6d.
|
||||||
|
|
||||||
|
A separable portion of the object code, whose source code is excluded
|
||||||
|
from the Corresponding Source as a System Library, need not be
|
||||||
|
included in conveying the object code work.
|
||||||
|
|
||||||
|
A "User Product" is either (1) a "consumer product", which means any
|
||||||
|
tangible personal property which is normally used for personal, family,
|
||||||
|
or household purposes, or (2) anything designed or sold for incorporation
|
||||||
|
into a dwelling. In determining whether a product is a consumer product,
|
||||||
|
doubtful cases shall be resolved in favor of coverage. For a particular
|
||||||
|
product received by a particular user, "normally used" refers to a
|
||||||
|
typical or common use of that class of product, regardless of the status
|
||||||
|
of the particular user or of the way in which the particular user
|
||||||
|
actually uses, or expects or is expected to use, the product. A product
|
||||||
|
is a consumer product regardless of whether the product has substantial
|
||||||
|
commercial, industrial or non-consumer uses, unless such uses represent
|
||||||
|
the only significant mode of use of the product.
|
||||||
|
|
||||||
|
"Installation Information" for a User Product means any methods,
|
||||||
|
procedures, authorization keys, or other information required to install
|
||||||
|
and execute modified versions of a covered work in that User Product from
|
||||||
|
a modified version of its Corresponding Source. The information must
|
||||||
|
suffice to ensure that the continued functioning of the modified object
|
||||||
|
code is in no case prevented or interfered with solely because
|
||||||
|
modification has been made.
|
||||||
|
|
||||||
|
If you convey an object code work under this section in, or with, or
|
||||||
|
specifically for use in, a User Product, and the conveying occurs as
|
||||||
|
part of a transaction in which the right of possession and use of the
|
||||||
|
User Product is transferred to the recipient in perpetuity or for a
|
||||||
|
fixed term (regardless of how the transaction is characterized), the
|
||||||
|
Corresponding Source conveyed under this section must be accompanied
|
||||||
|
by the Installation Information. But this requirement does not apply
|
||||||
|
if neither you nor any third party retains the ability to install
|
||||||
|
modified object code on the User Product (for example, the work has
|
||||||
|
been installed in ROM).
|
||||||
|
|
||||||
|
The requirement to provide Installation Information does not include a
|
||||||
|
requirement to continue to provide support service, warranty, or updates
|
||||||
|
for a work that has been modified or installed by the recipient, or for
|
||||||
|
the User Product in which it has been modified or installed. Access to a
|
||||||
|
network may be denied when the modification itself materially and
|
||||||
|
adversely affects the operation of the network or violates the rules and
|
||||||
|
protocols for communication across the network.
|
||||||
|
|
||||||
|
Corresponding Source conveyed, and Installation Information provided,
|
||||||
|
in accord with this section must be in a format that is publicly
|
||||||
|
documented (and with an implementation available to the public in
|
||||||
|
source code form), and must require no special password or key for
|
||||||
|
unpacking, reading or copying.
|
||||||
|
|
||||||
|
7. Additional Terms.
|
||||||
|
|
||||||
|
"Additional permissions" are terms that supplement the terms of this
|
||||||
|
License by making exceptions from one or more of its conditions.
|
||||||
|
Additional permissions that are applicable to the entire Program shall
|
||||||
|
be treated as though they were included in this License, to the extent
|
||||||
|
that they are valid under applicable law. If additional permissions
|
||||||
|
apply only to part of the Program, that part may be used separately
|
||||||
|
under those permissions, but the entire Program remains governed by
|
||||||
|
this License without regard to the additional permissions.
|
||||||
|
|
||||||
|
When you convey a copy of a covered work, you may at your option
|
||||||
|
remove any additional permissions from that copy, or from any part of
|
||||||
|
it. (Additional permissions may be written to require their own
|
||||||
|
removal in certain cases when you modify the work.) You may place
|
||||||
|
additional permissions on material, added by you to a covered work,
|
||||||
|
for which you have or can give appropriate copyright permission.
|
||||||
|
|
||||||
|
Notwithstanding any other provision of this License, for material you
|
||||||
|
add to a covered work, you may (if authorized by the copyright holders of
|
||||||
|
that material) supplement the terms of this License with terms:
|
||||||
|
|
||||||
|
a) Disclaiming warranty or limiting liability differently from the
|
||||||
|
terms of sections 15 and 16 of this License; or
|
||||||
|
|
||||||
|
b) Requiring preservation of specified reasonable legal notices or
|
||||||
|
author attributions in that material or in the Appropriate Legal
|
||||||
|
Notices displayed by works containing it; or
|
||||||
|
|
||||||
|
c) Prohibiting misrepresentation of the origin of that material, or
|
||||||
|
requiring that modified versions of such material be marked in
|
||||||
|
reasonable ways as different from the original version; or
|
||||||
|
|
||||||
|
d) Limiting the use for publicity purposes of names of licensors or
|
||||||
|
authors of the material; or
|
||||||
|
|
||||||
|
e) Declining to grant rights under trademark law for use of some
|
||||||
|
trade names, trademarks, or service marks; or
|
||||||
|
|
||||||
|
f) Requiring indemnification of licensors and authors of that
|
||||||
|
material by anyone who conveys the material (or modified versions of
|
||||||
|
it) with contractual assumptions of liability to the recipient, for
|
||||||
|
any liability that these contractual assumptions directly impose on
|
||||||
|
those licensors and authors.
|
||||||
|
|
||||||
|
All other non-permissive additional terms are considered "further
|
||||||
|
restrictions" within the meaning of section 10. If the Program as you
|
||||||
|
received it, or any part of it, contains a notice stating that it is
|
||||||
|
governed by this License along with a term that is a further
|
||||||
|
restriction, you may remove that term. If a license document contains
|
||||||
|
a further restriction but permits relicensing or conveying under this
|
||||||
|
License, you may add to a covered work material governed by the terms
|
||||||
|
of that license document, provided that the further restriction does
|
||||||
|
not survive such relicensing or conveying.
|
||||||
|
|
||||||
|
If you add terms to a covered work in accord with this section, you
|
||||||
|
must place, in the relevant source files, a statement of the
|
||||||
|
additional terms that apply to those files, or a notice indicating
|
||||||
|
where to find the applicable terms.
|
||||||
|
|
||||||
|
Additional terms, permissive or non-permissive, may be stated in the
|
||||||
|
form of a separately written license, or stated as exceptions;
|
||||||
|
the above requirements apply either way.
|
||||||
|
|
||||||
|
8. Termination.
|
||||||
|
|
||||||
|
You may not propagate or modify a covered work except as expressly
|
||||||
|
provided under this License. Any attempt otherwise to propagate or
|
||||||
|
modify it is void, and will automatically terminate your rights under
|
||||||
|
this License (including any patent licenses granted under the third
|
||||||
|
paragraph of section 11).
|
||||||
|
|
||||||
|
However, if you cease all violation of this License, then your
|
||||||
|
license from a particular copyright holder is reinstated (a)
|
||||||
|
provisionally, unless and until the copyright holder explicitly and
|
||||||
|
finally terminates your license, and (b) permanently, if the copyright
|
||||||
|
holder fails to notify you of the violation by some reasonable means
|
||||||
|
prior to 60 days after the cessation.
|
||||||
|
|
||||||
|
Moreover, your license from a particular copyright holder is
|
||||||
|
reinstated permanently if the copyright holder notifies you of the
|
||||||
|
violation by some reasonable means, this is the first time you have
|
||||||
|
received notice of violation of this License (for any work) from that
|
||||||
|
copyright holder, and you cure the violation prior to 30 days after
|
||||||
|
your receipt of the notice.
|
||||||
|
|
||||||
|
Termination of your rights under this section does not terminate the
|
||||||
|
licenses of parties who have received copies or rights from you under
|
||||||
|
this License. If your rights have been terminated and not permanently
|
||||||
|
reinstated, you do not qualify to receive new licenses for the same
|
||||||
|
material under section 10.
|
||||||
|
|
||||||
|
9. Acceptance Not Required for Having Copies.
|
||||||
|
|
||||||
|
You are not required to accept this License in order to receive or
|
||||||
|
run a copy of the Program. Ancillary propagation of a covered work
|
||||||
|
occurring solely as a consequence of using peer-to-peer transmission
|
||||||
|
to receive a copy likewise does not require acceptance. However,
|
||||||
|
nothing other than this License grants you permission to propagate or
|
||||||
|
modify any covered work. These actions infringe copyright if you do
|
||||||
|
not accept this License. Therefore, by modifying or propagating a
|
||||||
|
covered work, you indicate your acceptance of this License to do so.
|
||||||
|
|
||||||
|
10. Automatic Licensing of Downstream Recipients.
|
||||||
|
|
||||||
|
Each time you convey a covered work, the recipient automatically
|
||||||
|
receives a license from the original licensors, to run, modify and
|
||||||
|
propagate that work, subject to this License. You are not responsible
|
||||||
|
for enforcing compliance by third parties with this License.
|
||||||
|
|
||||||
|
An "entity transaction" is a transaction transferring control of an
|
||||||
|
organization, or substantially all assets of one, or subdividing an
|
||||||
|
organization, or merging organizations. If propagation of a covered
|
||||||
|
work results from an entity transaction, each party to that
|
||||||
|
transaction who receives a copy of the work also receives whatever
|
||||||
|
licenses to the work the party's predecessor in interest had or could
|
||||||
|
give under the previous paragraph, plus a right to possession of the
|
||||||
|
Corresponding Source of the work from the predecessor in interest, if
|
||||||
|
the predecessor has it or can get it with reasonable efforts.
|
||||||
|
|
||||||
|
You may not impose any further restrictions on the exercise of the
|
||||||
|
rights granted or affirmed under this License. For example, you may
|
||||||
|
not impose a license fee, royalty, or other charge for exercise of
|
||||||
|
rights granted under this License, and you may not initiate litigation
|
||||||
|
(including a cross-claim or counterclaim in a lawsuit) alleging that
|
||||||
|
any patent claim is infringed by making, using, selling, offering for
|
||||||
|
sale, or importing the Program or any portion of it.
|
||||||
|
|
||||||
|
11. Patents.
|
||||||
|
|
||||||
|
A "contributor" is a copyright holder who authorizes use under this
|
||||||
|
License of the Program or a work on which the Program is based. The
|
||||||
|
work thus licensed is called the contributor's "contributor version".
|
||||||
|
|
||||||
|
A contributor's "essential patent claims" are all patent claims
|
||||||
|
owned or controlled by the contributor, whether already acquired or
|
||||||
|
hereafter acquired, that would be infringed by some manner, permitted
|
||||||
|
by this License, of making, using, or selling its contributor version,
|
||||||
|
but do not include claims that would be infringed only as a
|
||||||
|
consequence of further modification of the contributor version. For
|
||||||
|
purposes of this definition, "control" includes the right to grant
|
||||||
|
patent sublicenses in a manner consistent with the requirements of
|
||||||
|
this License.
|
||||||
|
|
||||||
|
Each contributor grants you a non-exclusive, worldwide, royalty-free
|
||||||
|
patent license under the contributor's essential patent claims, to
|
||||||
|
make, use, sell, offer for sale, import and otherwise run, modify and
|
||||||
|
propagate the contents of its contributor version.
|
||||||
|
|
||||||
|
In the following three paragraphs, a "patent license" is any express
|
||||||
|
agreement or commitment, however denominated, not to enforce a patent
|
||||||
|
(such as an express permission to practice a patent or covenant not to
|
||||||
|
sue for patent infringement). To "grant" such a patent license to a
|
||||||
|
party means to make such an agreement or commitment not to enforce a
|
||||||
|
patent against the party.
|
||||||
|
|
||||||
|
If you convey a covered work, knowingly relying on a patent license,
|
||||||
|
and the Corresponding Source of the work is not available for anyone
|
||||||
|
to copy, free of charge and under the terms of this License, through a
|
||||||
|
publicly available network server or other readily accessible means,
|
||||||
|
then you must either (1) cause the Corresponding Source to be so
|
||||||
|
available, or (2) arrange to deprive yourself of the benefit of the
|
||||||
|
patent license for this particular work, or (3) arrange, in a manner
|
||||||
|
consistent with the requirements of this License, to extend the patent
|
||||||
|
license to downstream recipients. "Knowingly relying" means you have
|
||||||
|
actual knowledge that, but for the patent license, your conveying the
|
||||||
|
covered work in a country, or your recipient's use of the covered work
|
||||||
|
in a country, would infringe one or more identifiable patents in that
|
||||||
|
country that you have reason to believe are valid.
|
||||||
|
|
||||||
|
If, pursuant to or in connection with a single transaction or
|
||||||
|
arrangement, you convey, or propagate by procuring conveyance of, a
|
||||||
|
covered work, and grant a patent license to some of the parties
|
||||||
|
receiving the covered work authorizing them to use, propagate, modify
|
||||||
|
or convey a specific copy of the covered work, then the patent license
|
||||||
|
you grant is automatically extended to all recipients of the covered
|
||||||
|
work and works based on it.
|
||||||
|
|
||||||
|
A patent license is "discriminatory" if it does not include within
|
||||||
|
the scope of its coverage, prohibits the exercise of, or is
|
||||||
|
conditioned on the non-exercise of one or more of the rights that are
|
||||||
|
specifically granted under this License. You may not convey a covered
|
||||||
|
work if you are a party to an arrangement with a third party that is
|
||||||
|
in the business of distributing software, under which you make payment
|
||||||
|
to the third party based on the extent of your activity of conveying
|
||||||
|
the work, and under which the third party grants, to any of the
|
||||||
|
parties who would receive the covered work from you, a discriminatory
|
||||||
|
patent license (a) in connection with copies of the covered work
|
||||||
|
conveyed by you (or copies made from those copies), or (b) primarily
|
||||||
|
for and in connection with specific products or compilations that
|
||||||
|
contain the covered work, unless you entered into that arrangement,
|
||||||
|
or that patent license was granted, prior to 28 March 2007.
|
||||||
|
|
||||||
|
Nothing in this License shall be construed as excluding or limiting
|
||||||
|
any implied license or other defenses to infringement that may
|
||||||
|
otherwise be available to you under applicable patent law.
|
||||||
|
|
||||||
|
12. No Surrender of Others' Freedom.
|
||||||
|
|
||||||
|
If conditions are imposed on you (whether by court order, agreement or
|
||||||
|
otherwise) that contradict the conditions of this License, they do not
|
||||||
|
excuse you from the conditions of this License. If you cannot convey a
|
||||||
|
covered work so as to satisfy simultaneously your obligations under this
|
||||||
|
License and any other pertinent obligations, then as a consequence you may
|
||||||
|
not convey it at all. For example, if you agree to terms that obligate you
|
||||||
|
to collect a royalty for further conveying from those to whom you convey
|
||||||
|
the Program, the only way you could satisfy both those terms and this
|
||||||
|
License would be to refrain entirely from conveying the Program.
|
||||||
|
|
||||||
|
13. Remote Network Interaction; Use with the GNU General Public License.
|
||||||
|
|
||||||
|
Notwithstanding any other provision of this License, if you modify the
|
||||||
|
Program, your modified version must prominently offer all users
|
||||||
|
interacting with it remotely through a computer network (if your version
|
||||||
|
supports such interaction) an opportunity to receive the Corresponding
|
||||||
|
Source of your version by providing access to the Corresponding Source
|
||||||
|
from a network server at no charge, through some standard or customary
|
||||||
|
means of facilitating copying of software. This Corresponding Source
|
||||||
|
shall include the Corresponding Source for any work covered by version 3
|
||||||
|
of the GNU General Public License that is incorporated pursuant to the
|
||||||
|
following paragraph.
|
||||||
|
|
||||||
|
Notwithstanding any other provision of this License, you have
|
||||||
|
permission to link or combine any covered work with a work licensed
|
||||||
|
under version 3 of the GNU General Public License into a single
|
||||||
|
combined work, and to convey the resulting work. The terms of this
|
||||||
|
License will continue to apply to the part which is the covered work,
|
||||||
|
but the work with which it is combined will remain governed by version
|
||||||
|
3 of the GNU General Public License.
|
||||||
|
|
||||||
|
14. Revised Versions of this License.
|
||||||
|
|
||||||
|
The Free Software Foundation may publish revised and/or new versions of
|
||||||
|
the GNU Affero General Public License from time to time. Such new versions
|
||||||
|
will be similar in spirit to the present version, but may differ in detail to
|
||||||
|
address new problems or concerns.
|
||||||
|
|
||||||
|
Each version is given a distinguishing version number. If the
|
||||||
|
Program specifies that a certain numbered version of the GNU Affero General
|
||||||
|
Public License "or any later version" applies to it, you have the
|
||||||
|
option of following the terms and conditions either of that numbered
|
||||||
|
version or of any later version published by the Free Software
|
||||||
|
Foundation. If the Program does not specify a version number of the
|
||||||
|
GNU Affero General Public License, you may choose any version ever published
|
||||||
|
by the Free Software Foundation.
|
||||||
|
|
||||||
|
If the Program specifies that a proxy can decide which future
|
||||||
|
versions of the GNU Affero General Public License can be used, that proxy's
|
||||||
|
public statement of acceptance of a version permanently authorizes you
|
||||||
|
to choose that version for the Program.
|
||||||
|
|
||||||
|
Later license versions may give you additional or different
|
||||||
|
permissions. However, no additional obligations are imposed on any
|
||||||
|
author or copyright holder as a result of your choosing to follow a
|
||||||
|
later version.
|
||||||
|
|
||||||
|
15. Disclaimer of Warranty.
|
||||||
|
|
||||||
|
THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
|
||||||
|
APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
|
||||||
|
HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
|
||||||
|
OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
|
||||||
|
THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
||||||
|
PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
|
||||||
|
IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
|
||||||
|
ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
|
||||||
|
|
||||||
|
16. Limitation of Liability.
|
||||||
|
|
||||||
|
IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
|
||||||
|
WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
|
||||||
|
THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
|
||||||
|
GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
|
||||||
|
USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
|
||||||
|
DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
|
||||||
|
PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
|
||||||
|
EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
|
||||||
|
SUCH DAMAGES.
|
||||||
|
|
||||||
|
17. Interpretation of Sections 15 and 16.
|
||||||
|
|
||||||
|
If the disclaimer of warranty and limitation of liability provided
|
||||||
|
above cannot be given local legal effect according to their terms,
|
||||||
|
reviewing courts shall apply local law that most closely approximates
|
||||||
|
an absolute waiver of all civil liability in connection with the
|
||||||
|
Program, unless a warranty or assumption of liability accompanies a
|
||||||
|
copy of the Program in return for a fee.
|
||||||
|
|
||||||
|
END OF TERMS AND CONDITIONS
|
||||||
|
|
||||||
|
How to Apply These Terms to Your New Programs
|
||||||
|
|
||||||
|
If you develop a new program, and you want it to be of the greatest
|
||||||
|
possible use to the public, the best way to achieve this is to make it
|
||||||
|
free software which everyone can redistribute and change under these terms.
|
||||||
|
|
||||||
|
To do so, attach the following notices to the program. It is safest
|
||||||
|
to attach them to the start of each source file to most effectively
|
||||||
|
state the exclusion of warranty; and each file should have at least
|
||||||
|
the "copyright" line and a pointer to where the full notice is found.
|
||||||
|
|
||||||
|
<one line to give the program's name and a brief idea of what it does.>
|
||||||
|
Copyright (C) <year> <name of author>
|
||||||
|
|
||||||
|
This program is free software: you can redistribute it and/or modify
|
||||||
|
it under the terms of the GNU Affero General Public License as published by
|
||||||
|
the Free Software Foundation, either version 3 of the License, or
|
||||||
|
(at your option) any later version.
|
||||||
|
|
||||||
|
This program is distributed in the hope that it will be useful,
|
||||||
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
GNU Affero General Public License for more details.
|
||||||
|
|
||||||
|
You should have received a copy of the GNU Affero General Public License
|
||||||
|
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
Also add information on how to contact you by electronic and paper mail.
|
||||||
|
|
||||||
|
If your software can interact with users remotely through a computer
|
||||||
|
network, you should also make sure that it provides a way for users to
|
||||||
|
get its source. For example, if your program is a web application, its
|
||||||
|
interface could display a "Source" link that leads users to an archive
|
||||||
|
of the code. There are many ways you could offer source, and different
|
||||||
|
solutions will be better for different programs; see section 13 for the
|
||||||
|
specific requirements.
|
||||||
|
|
||||||
|
You should also get your employer (if you work as a programmer) or school,
|
||||||
|
if any, to sign a "copyright disclaimer" for the program, if necessary.
|
||||||
|
For more information on this, and how to apply and follow the GNU AGPL, see
|
||||||
|
<https://www.gnu.org/licenses/>.
|
||||||
58
README.md
58
README.md
@ -12,6 +12,8 @@
|
|||||||
homed(内核零 IO) ← PluginSDK → 插件(所有 IO 能力)
|
homed(内核零 IO) ← PluginSDK → 插件(所有 IO 能力)
|
||||||
```
|
```
|
||||||
|
|
||||||
|
**v1.1.1 起媒体贯通插件边界**:插件与模型都能读写记忆里的图片/音频(`InsertWithMedia`、`InjectInputMedia`),媒体以 `[<mime> <短digest>] <描述>` 标记存在于纯文本记忆中——描述是可检索的语义记忆,digest 是回到字节的钥匙。
|
||||||
|
|
||||||
**v1.0.0 起外部插件是独立子进程**:经 stdio JSON-RPC(控制面)+ 共享内存段(数据面)+ 事件环(通知面)与内核通信。插件崩溃不影响内核且自动重启,换 `plugin.bin` 即生效的真热重载。
|
**v1.0.0 起外部插件是独立子进程**:经 stdio JSON-RPC(控制面)+ 共享内存段(数据面)+ 事件环(通知面)与内核通信。插件崩溃不影响内核且自动重启,换 `plugin.bin` 即生效的真热重载。
|
||||||
|
|
||||||
## 设计要点
|
## 设计要点
|
||||||
@ -193,6 +195,37 @@ internal/
|
|||||||
|
|
||||||
## 项目状态
|
## 项目状态
|
||||||
|
|
||||||
|
**v1.2.0** — 统一多模态向量空间 + 媒体升为图记忆一等节点 + 数据面全量迁到共享内存。
|
||||||
|
|
||||||
|
- **模型中立的统一向量空间**:内核不再适配任何具体模型,只提供公共 provider SPI
|
||||||
|
(`pkg/embedding`:`Modality` / `Input{Data,MIME}` / `Info{Dimension,Fingerprint,Modalities}`
|
||||||
|
+ 名字注册表),实现在 `providers/*`。默认 **Chinese-CLIP ViT-B/16** —— text 与 image
|
||||||
|
落在**同一空间**(512 维、指纹 `cd2a495cf990`、Apache-2.0、独立实测常驻约 1.15GB);
|
||||||
|
`qwen3vl` 保留(2048 维、常驻约 9.4GB,供内存充足或将来要视频的机器切回)。
|
||||||
|
文本检索仍由既有词向量 / TF-IDF 兜底:CLIP 双塔的**纯文本语义弱于 MLLM 型嵌入器**,
|
||||||
|
这是已知并写进文档的代价。
|
||||||
|
- **媒体是图数据库的一等节点与边**:**彻底删除**「用文本描述式索引图片」这套将就机制,
|
||||||
|
以及 `media_refs` 与媒体引用计数。记忆块遵循单层不变量——Context → Document → Graph
|
||||||
|
是块的**迁移**,不是复制、也不靠引用保活。
|
||||||
|
- **数据面全部走共享内存**(工具调用帧 / Cleaner / 输入输出通道 / 媒体块 / 文档与知识正文),
|
||||||
|
RPC 只传偏移描述符;**RPC 协议升到 2**,fd3 布局改变,**不支持滚动升级**——
|
||||||
|
内核与全部插件必须同批重建、同批安装,存量插件须用新版 `plugindev` 重编。
|
||||||
|
- 注入可声明 `InjectOptions{NoMemory, ContextPolicy}`(**默认仍记入记忆、默认不裁剪**);
|
||||||
|
裁剪必须显式声明,且先经插件注册的 `Cleaner`。SDK 1.2.0 相对 1.1.0 **纯追加**。
|
||||||
|
- **发行包默认启用** ONNX 向量空间,并把模型(754MB)与 ONNX Runtime(24MB)随
|
||||||
|
server/full 包发布;`homed` 放弃 Windows 原生支持改走 WSL2;jieba 词库内嵌进二进制。
|
||||||
|
- 修掉三个**安装链静默失败**:`initconfig` 因 `CGO_ENABLED=0` 是空操作(打印凭据却一个字节
|
||||||
|
没写)、全新安装被误判「已有配置」而整体跳过默认值播种(装完 0 插件)、deb 的 `postinst`
|
||||||
|
查错 unit 路径导致 `enable` 从未执行。
|
||||||
|
- 自本版起以 **AGPL-3.0-only** 发布(含网络条款;插件静态链接 SDK 故须同许可,见「许可」)。
|
||||||
|
|
||||||
|
> 以下历史条目保留原文以呈现演进,其中两条机制**已在 v1.2.0 移除**:
|
||||||
|
> 「媒体以 `[<mime> <短digest>] <描述>` 标记参与检索」(描述式索引)与「媒体引用计数式 GC」。
|
||||||
|
|
||||||
|
**v1.1.1** — 多模态贯通**插件边界**。v1.1.0 让记忆系统支持了二进制多媒体节点,但那条链路只对内核自己开放;本版打通到插件与模型。公开 SDK 新增媒体字段与三个媒体注入接口(配套 [SDK v1.1.0](https://gitcode.com/JianFeeeee/homeagent-sdk/releases/tag/v1.1.0),整条 1.1.x 线共用),内核实现对应四个 RPC。桥接层此前在**静默裁字段**:插件交进来的 `Confidence`/类型/`SentenceText` 全被丢弃、`Doc` 只留三个字段、`Remove` 不解引用(媒体永久算「被引用」,GC 收不掉)。`processTextInput`/`processMediaInput` 归一成一条 `processInput`,媒体路径由此获得它一直缺的去重、`no_memory`、通道 `Cleaner`、中断语义、`EventRawInput`。修掉三处真实缺陷:**用户发的图从来没出现在 WebUI 聊天记录里**(媒体路径发布 map 而订阅方断言 string)、**`memory_commit` 的 `sentence_text` 从未暴露给模型**(而它是媒体绑定链的必经环节)、**`PluginSDK` 两处并发竞态**(`-race` 实测 11 处,插件重载瞬间偶发 nil 解引用崩溃)。
|
||||||
|
|
||||||
|
**v1.1.0** — 记忆系统支持**二进制多媒体节点**。内容寻址媒体存储(CAS + SQLite 元数据 + 磁盘 blob,`Get` always 重校 digest),贯通 L0(上下文事件)/L2(文档)/L3(图谱句子)三层,引用计数式 GC(有引用者绝不删)。视觉模型生成的描述文本是持久语义记忆,blob 只是可被容量 GC 淘汰的缓存。
|
||||||
|
|
||||||
**v1.0.0** — 外部插件从 C ABI 动态库迁移到**子进程 + 共享内存**。首个不再加载 `.so`/`.dll` 的版本,与 0.9.x 不兼容(存量插件须用新版 `plugindev` 重编为 `plugin.bin`,**业务代码零改动**)。消除 6 类此前在生产造成故障的缺陷:热重载失效(`DF_1_NODELETE` 让 `dlclose` 成 no-op)、崩溃隔离缺失(插件 panic 带崩 homed)、stage lost update(副本模型丢失 35.8~36.8%)、cgo 超时不可中断(线程线性泄漏)、`output_send` 假成功(模型收到「已发送」而消息未送达)、Windows 能力断层(只见 3 个 stage 字段且无法写回)。三面通信:stdio JSON-RPC(控制)+ 共享内存段(数据)+ 事件环(通知);权限梯度显式化为三道闸。RPC 往返 p50 24.1µs,崩溃到恢复 <1s。
|
**v1.0.0** — 外部插件从 C ABI 动态库迁移到**子进程 + 共享内存**。首个不再加载 `.so`/`.dll` 的版本,与 0.9.x 不兼容(存量插件须用新版 `plugindev` 重编为 `plugin.bin`,**业务代码零改动**)。消除 6 类此前在生产造成故障的缺陷:热重载失效(`DF_1_NODELETE` 让 `dlclose` 成 no-op)、崩溃隔离缺失(插件 panic 带崩 homed)、stage lost update(副本模型丢失 35.8~36.8%)、cgo 超时不可中断(线程线性泄漏)、`output_send` 假成功(模型收到「已发送」而消息未送达)、Windows 能力断层(只见 3 个 stage 字段且无法写回)。三面通信:stdio JSON-RPC(控制)+ 共享内存段(数据)+ 事件环(通知);权限梯度显式化为三道闸。RPC 往返 p50 24.1µs,崩溃到恢复 <1s。
|
||||||
|
|
||||||
**v0.9.0** — C ABI v2:外部插件 Stage 回调支持写回(`invoke_stage` 增加 result 输出,插件可在 OnInput/AfterToolcall/PostAction 修改 RawMessage/LLMText/ToolResults 等并同步回内核),ABI 版本随内核 minor 对齐(v0.9.x → ABIVersion=2,`version_min=1` 向后兼容旧插件)。同步修复工具循环 zen 兼容补位误伤首轮 system 上下文的问题。配套 SDK 提供增强版 sanitizer 示例(坏 UTF-8/U+FFFD/ANSI 转义全链路清洗)。**该 ABI 已随 v1.0.0 退场。**
|
**v0.9.0** — C ABI v2:外部插件 Stage 回调支持写回(`invoke_stage` 增加 result 输出,插件可在 OnInput/AfterToolcall/PostAction 修改 RawMessage/LLMText/ToolResults 等并同步回内核),ABI 版本随内核 minor 对齐(v0.9.x → ABIVersion=2,`version_min=1` 向后兼容旧插件)。同步修复工具循环 zen 兼容补位误伤首轮 system 上下文的问题。配套 SDK 提供增强版 sanitizer 示例(坏 UTF-8/U+FFFD/ANSI 转义全链路清洗)。**该 ABI 已随 v1.0.0 退场。**
|
||||||
@ -218,7 +251,7 @@ internal/
|
|||||||
| **client** | waiter + 桌面 GUI | 连接远程 HomeAgent |
|
| **client** | waiter + 桌面 GUI | 连接远程 HomeAgent |
|
||||||
|
|
||||||
- Linux:`.deb`(amd64/arm64)、`.rpm`(x86_64)、`.tar.gz`
|
- Linux:`.deb`(amd64/arm64)、`.rpm`(x86_64)、`.tar.gz`
|
||||||
- Windows:`HomeAgent_v1.0.0_{Full,Server,Client}_win64.exe`(NSIS 安装向导)
|
- Windows:`HomeAgent_v1.2.0_{Full,Server,Client}_win64.exe`(NSIS 安装向导,含 AGPL 许可页)。自 v1.2.0 起因 `homed` 不再支持 Windows 原生(依赖 fd 继承与共享内存段内偏移解引用),安装器改为引导到 **WSL2**,并把 Linux 包送进发行版里按 Linux 方式安装。
|
||||||
- 免安装:`homeagent-bin-<os>_<arch>.tar.gz`(含 homed/waiter/initconfig)
|
- 免安装:`homeagent-bin-<os>_<arch>.tar.gz`(含 homed/waiter/initconfig)
|
||||||
- 校验:`SHA256SUMS`
|
- 校验:`SHA256SUMS`
|
||||||
|
|
||||||
@ -233,3 +266,26 @@ make install # 安装到系统
|
|||||||
```
|
```
|
||||||
|
|
||||||
依赖:Go 1.25+, CGo (go-sqlite3), Linux/Windows。
|
依赖:Go 1.25+, CGo (go-sqlite3), Linux/Windows。
|
||||||
|
|
||||||
|
## 许可
|
||||||
|
|
||||||
|
本项目以 **GNU Affero 通用公共许可证第 3 版(AGPL-3.0-only)** 发布,全文见 [LICENSE](LICENSE)。
|
||||||
|
|
||||||
|
它是 GPL 家族里**传染性最强**的一档:不仅分发时须提供完整对应源码,
|
||||||
|
**通过网络提供服务时也要向使用者提供源码**(§13 Remote Network Interaction)。
|
||||||
|
即:任何人把改过的 HomeAgent 对外提供网络服务,都必须让该服务的使用者拿到改动后的源码。
|
||||||
|
|
||||||
|
插件与本项目通过公开 SDK **静态链接**(SDK 源码会进入插件二进制),因此插件是本项目的
|
||||||
|
衍生作品,需以相同许可发布;子进程隔离不改变这一点,因为被链接的是 SDK 代码本身。
|
||||||
|
|
||||||
|
### 随包分发的第三方组件
|
||||||
|
|
||||||
|
| 组件 | 许可 | 位置 |
|
||||||
|
|---|---|---|
|
||||||
|
| Chinese-CLIP ViT-B/16(ONNX 产物) | Apache-2.0 | `/usr/lib/homeagent/models/chinese-clip-vit-b16-onnx/` |
|
||||||
|
| ONNX Runtime(`libonnxruntime.so`) | MIT | `/usr/lib/homeagent/onnxruntime/` |
|
||||||
|
| jieba 词库(内嵌进二进制) | MIT | 源码 `internal/memory/jiebadict/` |
|
||||||
|
| Go 依赖(go-sqlite3、gojieba、bubbletea 等) | MIT / BSD-3 / Apache-2.0 | 均为宽松许可,与 AGPL-3.0 兼容 |
|
||||||
|
|
||||||
|
这些组件**保持各自原有许可**,不在本项目的 AGPL 授权范围内;发行包把它们的许可全文放在
|
||||||
|
`/usr/share/doc/homeagent/licenses/`,并在 dep/rpm 元数据里声明本包许可为 `AGPL-3.0-only`。
|
||||||
|
|||||||
90
README_EN.md
90
README_EN.md
@ -12,6 +12,11 @@ Combined with a **three-layer memory architecture** (Context → Document → Gr
|
|||||||
homed (kernel, zero IO) ← PluginSDK → plugins (all IO capabilities)
|
homed (kernel, zero IO) ← PluginSDK → plugins (all IO capabilities)
|
||||||
```
|
```
|
||||||
|
|
||||||
|
**Since v1.1.1 media reaches the plugin boundary**: plugins and the model can both read and
|
||||||
|
write images/audio in memory (`InsertWithMedia`, `InjectInputMedia`). Media lives in plain-text
|
||||||
|
memory as a `[<mime> <short digest>] <description>` marker — the description is the searchable
|
||||||
|
semantic memory, the digest is the key back to the bytes.
|
||||||
|
|
||||||
**Since v1.0.0 external plugins are independent subprocesses**, communicating with the kernel over
|
**Since v1.0.0 external plugins are independent subprocesses**, communicating with the kernel over
|
||||||
stdio JSON-RPC (control plane) + a shared memory segment (data plane) + an event ring (notification
|
stdio JSON-RPC (control plane) + a shared memory segment (data plane) + an event ring (notification
|
||||||
plane). A plugin crash cannot take down the kernel and it restarts automatically; swapping
|
plane). A plugin crash cannot take down the kernel and it restarts automatically; swapping
|
||||||
@ -179,6 +184,62 @@ External plugin development: see [homeagent-sdk](https://gitcode.com/JianFeeeee/
|
|||||||
|
|
||||||
## Project Status
|
## Project Status
|
||||||
|
|
||||||
|
**v1.2.0** — unified multimodal vector space, media promoted to first-class graph memory, and the whole data plane moved into shared memory.
|
||||||
|
|
||||||
|
- **Model-neutral unified embedding space**: the kernel no longer adapts to any specific model.
|
||||||
|
It exposes only a public provider SPI (`pkg/embedding`: `Modality` / `Input{Data,MIME}` /
|
||||||
|
`Info{Dimension,Fingerprint,Modalities}` + a name registry), with implementations under
|
||||||
|
`providers/*`. Default: **Chinese-CLIP ViT-B/16** — text and image land in the **same space**
|
||||||
|
(512-dim, fingerprint `cd2a495cf990`, Apache-2.0, ~1.15GB RSS measured standalone);
|
||||||
|
`qwen3vl` is kept (2048-dim, ~9.4GB) for machines with headroom or future video. Text search
|
||||||
|
still falls back to the existing word-vector / TF-IDF path — a CLIP dual tower's pure-text
|
||||||
|
semantics are **weaker than an MLLM-style embedder**, a cost documented rather than hidden.
|
||||||
|
- **Media are first-class nodes and edges in the graph DB**: the "index images via generated
|
||||||
|
text descriptions" stopgap, `media_refs` and media reference counting are **removed**.
|
||||||
|
Memory blocks follow a single-layer invariant — Context → Document → Graph is a **migration**,
|
||||||
|
not a copy, and not kept alive by references.
|
||||||
|
- **The entire data plane goes through shared memory** (tool-call frames, Cleaners, input/output
|
||||||
|
lanes, media blocks, document and knowledge bodies); RPC carries only offset descriptors.
|
||||||
|
**RPC protocol is now 2**: the fd3 layout changed and there is **no rolling upgrade** —
|
||||||
|
kernel and all plugins must be rebuilt and installed together.
|
||||||
|
- Injections can declare `InjectOptions{NoMemory, ContextPolicy}` (**defaults: still recorded,
|
||||||
|
not pruned**); pruning must be requested explicitly and goes through the plugin's registered
|
||||||
|
`Cleaner`. SDK 1.2.0 is **purely additive** over 1.1.0.
|
||||||
|
- **Release packages enable the ONNX space by default** and bundle the model (754MB) plus
|
||||||
|
ONNX Runtime (24MB) in the server/full packages; `homed` drops native Windows support in
|
||||||
|
favour of WSL2; the jieba dictionary is embedded in the binary.
|
||||||
|
- Fixed three **silent install-chain failures**: `initconfig` was a no-op (`CGO_ENABLED=0` stub)
|
||||||
|
that printed credentials without writing any, fresh installs were misdetected as "already
|
||||||
|
configured" so default seeding was skipped entirely (0 plugins installed), and the deb
|
||||||
|
`postinst` looked for the unit in the wrong path so `enable` never ran.
|
||||||
|
- Licensed **AGPL-3.0-only** from this version on (network clause included; statically linked
|
||||||
|
plugins must match — see License).
|
||||||
|
|
||||||
|
> The historical entries below are kept verbatim to show the evolution; two mechanisms in them
|
||||||
|
> were **removed in v1.2.0**: text-description-based media indexing, and reference-counted media GC.
|
||||||
|
|
||||||
|
**v1.1.1** — Multimodal reaches the **plugin boundary**. v1.1.0 gave the memory system binary
|
||||||
|
multimedia nodes, but that path was open only to the kernel itself; this release opens it to
|
||||||
|
plugins and the model. The public SDK gains media fields and three media injection methods
|
||||||
|
(paired with [SDK v1.1.0](https://gitcode.com/JianFeeeee/homeagent-sdk/releases/tag/v1.1.0),
|
||||||
|
shared by the whole 1.1.x line), and the kernel implements the four matching RPCs. The bridge
|
||||||
|
layer had been **silently dropping fields**: `Confidence`/types/`SentenceText` handed in by a
|
||||||
|
plugin were discarded, `Doc` kept only three fields, and `Remove` never released references
|
||||||
|
(media stayed "referenced" forever, so GC could never reclaim it). `processTextInput` and
|
||||||
|
`processMediaInput` were unified into a single `processInput`, which finally gives the media
|
||||||
|
path the dedup, `no_memory`, channel `Cleaner`, interrupt semantics and correct `EventRawInput`
|
||||||
|
it had always lacked. Three real defects fixed: **user-sent images never appeared in the WebUI
|
||||||
|
chat log** (the media path published a map while the subscriber asserted a string),
|
||||||
|
**`memory_commit`'s `sentence_text` had never been exposed to the model** (though it is the
|
||||||
|
mandatory link in the media binding chain), and **two data races in `PluginSDK`** (11 reported
|
||||||
|
by `-race`; in production this showed up as sporadic nil-dereference crashes during plugin reload).
|
||||||
|
|
||||||
|
**v1.1.0** — Memory system supports **binary multimedia nodes**. Content-addressed media store
|
||||||
|
(CAS + SQLite metadata + on-disk blobs, `Get` always re-verifies the digest) wired through L0
|
||||||
|
(context events) / L2 (documents) / L3 (graph sentences), with reference-counted GC (referenced
|
||||||
|
items are never deleted). The description text produced by the vision model is the durable
|
||||||
|
semantic memory; the blob is only a cache that capacity GC may evict.
|
||||||
|
|
||||||
**v1.0.0** — External plugins moved from C ABI shared libraries to **subprocess + shared memory**. The first release that no longer loads `.so`/`.dll`, and it is incompatible with 0.9.x (existing plugins must be rebuilt into `plugin.bin` with the new `plugindev`, though **business code needs zero changes**). Eliminates 6 classes of defects that had caused production incidents: hot-reload silently failing (`DF_1_NODELETE` making `dlclose` a no-op), no crash isolation (a plugin panic took down homed), stage lost updates (35.8~36.8% loss under the copy model), uncancellable cgo timeouts (linear OS-thread leaks), `output_send` reporting false success (the model was told "sent" while the message never went out), and Windows capability degradation (only 3 stage fields visible, no write-back). Three communication planes: stdio JSON-RPC (control) + shared memory segment (data) + event ring (notification); the privilege gradient is now enforced by three explicit gates. RPC round-trip p50 24.1µs; crash-to-recovery under 1s.
|
**v1.0.0** — External plugins moved from C ABI shared libraries to **subprocess + shared memory**. The first release that no longer loads `.so`/`.dll`, and it is incompatible with 0.9.x (existing plugins must be rebuilt into `plugin.bin` with the new `plugindev`, though **business code needs zero changes**). Eliminates 6 classes of defects that had caused production incidents: hot-reload silently failing (`DF_1_NODELETE` making `dlclose` a no-op), no crash isolation (a plugin panic took down homed), stage lost updates (35.8~36.8% loss under the copy model), uncancellable cgo timeouts (linear OS-thread leaks), `output_send` reporting false success (the model was told "sent" while the message never went out), and Windows capability degradation (only 3 stage fields visible, no write-back). Three communication planes: stdio JSON-RPC (control) + shared memory segment (data) + event ring (notification); the privilege gradient is now enforced by three explicit gates. RPC round-trip p50 24.1µs; crash-to-recovery under 1s.
|
||||||
|
|
||||||
**v0.9.0** — C ABI v2: external plugin Stage callbacks can now write back (`invoke_stage` gained a result out-param; plugins may mutate RawMessage/LLMText/ToolResults etc. in OnInput/AfterToolcall/PostAction and have them synced to the core). ABI version now tracks core minor releases (v0.9.x → ABIVersion=2, `version_min=1` keeps old plugins loadable). Also fixes the tool-loop zen-compat placeholder that wrongly fired on first-turn system context tail. The SDK ships an enhanced sanitizer example (bad-UTF-8 / U+FFFD / ANSI-escape scrub across the whole pipeline). **This ABI retired with v1.0.0.**
|
**v0.9.0** — C ABI v2: external plugin Stage callbacks can now write back (`invoke_stage` gained a result out-param; plugins may mutate RawMessage/LLMText/ToolResults etc. in OnInput/AfterToolcall/PostAction and have them synced to the core). ABI version now tracks core minor releases (v0.9.x → ABIVersion=2, `version_min=1` keeps old plugins loadable). Also fixes the tool-loop zen-compat placeholder that wrongly fired on first-turn system context tail. The SDK ships an enhanced sanitizer example (bad-UTF-8 / U+FFFD / ANSI-escape scrub across the whole pipeline). **This ABI retired with v1.0.0.**
|
||||||
@ -204,7 +265,7 @@ External plugin development: see [homeagent-sdk](https://gitcode.com/JianFeeeee/
|
|||||||
| **client** | waiter + desktop GUI | Connecting to a remote HomeAgent |
|
| **client** | waiter + desktop GUI | Connecting to a remote HomeAgent |
|
||||||
|
|
||||||
- Linux: `.deb` (amd64/arm64), `.rpm` (x86_64), `.tar.gz`
|
- Linux: `.deb` (amd64/arm64), `.rpm` (x86_64), `.tar.gz`
|
||||||
- Windows: `HomeAgent_v1.0.0_{Full,Server,Client}_win64.exe` (NSIS installer)
|
- Windows: `HomeAgent_v1.2.0_{Full,Server,Client}_win64.exe` (NSIS installer, includes the AGPL license page). Since v1.2.0 `homed` no longer supports native Windows (it relies on fd inheritance and in-segment offset dereferencing), so the installer bootstraps **WSL2** and installs the Linux packages inside the distribution the same way a Linux host would.
|
||||||
- Portable: `homeagent-bin-<os>_<arch>.tar.gz` (homed/waiter/initconfig)
|
- Portable: `homeagent-bin-<os>_<arch>.tar.gz` (homed/waiter/initconfig)
|
||||||
- Verification: `SHA256SUMS`
|
- Verification: `SHA256SUMS`
|
||||||
|
|
||||||
@ -219,3 +280,30 @@ make install # Install to system
|
|||||||
```
|
```
|
||||||
|
|
||||||
Dependencies: Go 1.25+, CGo (go-sqlite3), Linux/Windows.
|
Dependencies: Go 1.25+, CGo (go-sqlite3), Linux/Windows.
|
||||||
|
|
||||||
|
## License
|
||||||
|
|
||||||
|
This project is released under the **GNU Affero General Public License, version 3
|
||||||
|
(AGPL-3.0-only)** — see [LICENSE](LICENSE).
|
||||||
|
|
||||||
|
This is the strongest copyleft in the GPL family: besides shipping the complete corresponding
|
||||||
|
source when you distribute the software, **you must also offer the source to users who interact
|
||||||
|
with it over a network** (§13, Remote Network Interaction). Anyone running a modified HomeAgent
|
||||||
|
as a network service therefore has to make the modified source available to that service's users.
|
||||||
|
|
||||||
|
Plugins are **statically linked** against this project through the public SDK (the SDK source
|
||||||
|
ends up inside the plugin binary), so plugins are derivative works and must be released under
|
||||||
|
the same license. Process isolation does not change this — what is linked is the SDK code itself.
|
||||||
|
|
||||||
|
### Third-party components shipped with the packages
|
||||||
|
|
||||||
|
| Component | License | Location |
|
||||||
|
|---|---|---|
|
||||||
|
| Chinese-CLIP ViT-B/16 (ONNX artifacts) | Apache-2.0 | `/usr/lib/homeagent/models/chinese-clip-vit-b16-onnx/` |
|
||||||
|
| ONNX Runtime (`libonnxruntime.so`) | MIT | `/usr/lib/homeagent/onnxruntime/` |
|
||||||
|
| jieba dictionary (embedded in the binary) | MIT | `internal/memory/jiebadict/` |
|
||||||
|
| Go dependencies (go-sqlite3, gojieba, bubbletea, …) | MIT / BSD-3 / Apache-2.0 | permissive, AGPL-3.0-compatible |
|
||||||
|
|
||||||
|
These components keep their own licenses and are not relicensed by this project. Full texts are
|
||||||
|
shipped in `/usr/share/doc/homeagent/licenses/`, and the package metadata declares this package
|
||||||
|
as `AGPL-3.0-only`.
|
||||||
|
|||||||
@ -189,6 +189,39 @@ All vectorization unified under `StaticEmbedder` (`internal/memory/static_embedd
|
|||||||
| `doc_query` | Search from Document |
|
| `doc_query` | Search from Document |
|
||||||
| `doc_commit` | Write to Document |
|
| `doc_commit` | Write to Document |
|
||||||
|
|
||||||
|
Since v1.1.1 `memory_commit` and `doc_commit` accept `media_digests`, and the kernel appends the
|
||||||
|
`[<mime> <short digest>] <description>` marker into the sentence/body — **the kernel builds the
|
||||||
|
marker, the model only supplies the digest**. Requiring the caller to know the format would mean a
|
||||||
|
single typo silently breaks reference binding with no error anywhere in the chain. `memory_commit`
|
||||||
|
also gained `sentence_text`: media references hang off a sentence, so with no sentence there is
|
||||||
|
nowhere to attach them.
|
||||||
|
|
||||||
|
### Media Memory (since v1.1.0)
|
||||||
|
|
||||||
|
`internal/memory/media/` — `Store`, content-addressed (CAS)
|
||||||
|
|
||||||
|
| Concern | Approach | Why |
|
||||||
|
|---|---|---|
|
||||||
|
| Addressing | sha256 digest; metadata in SQLite, blobs on disk | Identical bytes stored once; metadata must be queryable, blobs must not live in the database |
|
||||||
|
| Integrity | Every `Get` re-verifies the digest | Silently returning corrupt data on disk damage is far worse than an error |
|
||||||
|
| Write atomicity | `.tmp` + rename | A half-written file taken as complete content would permanently poison that digest |
|
||||||
|
| References | `owner_kind/owner_id/digest` composite primary key, `AddRef` idempotent | Three owner kinds: `context` (context events), `document`, `graph_sentence` |
|
||||||
|
| GC | Two-stage with `minAge`, **referenced items are never deleted** | Description text stays in the text layers while blobs may be evicted — semantic memory and byte cache are decoupled |
|
||||||
|
|
||||||
|
**How media is represented in plain-text memory** is the marker `[<mime> <short digest>] <description>`:
|
||||||
|
|
||||||
|
```
|
||||||
|
[image/png a1b2c3d4e5f6] a purple-blue-red three-band chart
|
||||||
|
```
|
||||||
|
|
||||||
|
Why it must ride on text: `Doc.Content`, `sentences.text` and text memory's `Input` are all strings
|
||||||
|
— there is no field to carry structured data. **The description text is the durable semantic
|
||||||
|
memory** (retrieval uses it); the digest is the key back to the bytes (reverse lookup uses it).
|
||||||
|
After capacity GC evicts a blob, the description remains in the L0/L2/L3 text.
|
||||||
|
|
||||||
|
The media store is **optional throughout**: with `core.memory.media.enabled=false` or no
|
||||||
|
configuration, the whole chain silently degrades to plain-text behaviour — no errors, no panics.
|
||||||
|
|
||||||
### Other Memory Layers
|
### Other Memory Layers
|
||||||
|
|
||||||
- **Social** (`internal/memory/social/social.go`) — Persona traits and relationship network, wraps GraphDB entity types
|
- **Social** (`internal/memory/social/social.go`) — Persona traits and relationship network, wraps GraphDB entity types
|
||||||
@ -296,7 +329,7 @@ Common ground:
|
|||||||
|
|
||||||
| Plane | Mechanism | Why this choice |
|
| Plane | Mechanism | Why this choice |
|
||||||
|---|---|---|
|
|---|---|---|
|
||||||
| Control | stdio JSON-RPC (NDJSON frames), 51 `core.*` methods | The process boundary *is* the ABI boundary—no need to maintain three platform-specific dynamic-library loaders |
|
| Control | stdio JSON-RPC (NDJSON frames), 55 `core.*` methods | The process boundary *is* the ABI boundary—no need to maintain three platform-specific dynamic-library loaders |
|
||||||
| Data | Shared memory segment, **one segment shared by all subprocesses** | One segment per plugin would degrade "kernel ctx → segment → plugin mutates → read back" into the copy model under concurrency, reproducing lost updates exactly |
|
| Data | Shared memory segment, **one segment shared by all subprocesses** | One segment per plugin would degrade "kernel ctx → segment → plugin mutates → read back" into the copy model under concurrency, reproducing lost updates exactly |
|
||||||
| Notification | Event ring + platform notify (Linux eventfd / macOS pipe / Windows Event) | The kernel must never block on a consumer: streaming output publishes per token, so any wait shows up as stutter |
|
| Notification | Event ring + platform notify (Linux eventfd / macOS pipe / Windows Event) | The kernel must never block on a consumer: streaming output publishes per token, so any wait shows up as stutter |
|
||||||
|
|
||||||
@ -334,8 +367,23 @@ sdk.Memory().Recall/Commit
|
|||||||
sdk.Knowledge().Search/Create
|
sdk.Knowledge().Search/Create
|
||||||
sdk.Settings().Get/Set/List
|
sdk.Settings().Get/Set/List
|
||||||
sdk.RegisterOutputChannel("qq", sdk.CapText|sdk.CapAudio|sdk.CapImage, "QQ channel, see output_send__qq_help for details", handler)
|
sdk.RegisterOutputChannel("qq", sdk.CapText|sdk.CapAudio|sdk.CapImage, "QQ channel, see output_send__qq_help for details", handler)
|
||||||
|
|
||||||
|
// v1.1.1 media APIs (all additive, no signature changes)
|
||||||
|
sdk.DocMemory().InsertWithMedia(doc, attachments) // attachments with Data land in CAS; Digest-only ones reference existing content
|
||||||
|
sdk.InjectInputMedia(source, channel, text, blocks) // media reaches the model in *this* turn
|
||||||
|
sdk.InjectInputMediaSync(...) // same, and waits for the reply
|
||||||
|
sdk.InjectInterruptMedia(...) // media-bearing interrupt, can preempt current processing
|
||||||
```
|
```
|
||||||
|
|
||||||
|
How media injection differs from `SetToolBlocks`: the latter is only callable inside a tool handler
|
||||||
|
and its media reaches the model with the **next** tool message; these three let a plugin
|
||||||
|
**initiate a turn that carries media** — it goes out with this turn's message and is automatically
|
||||||
|
stored in CAS with a memory reference attached. `Triple` and `Doc` gained `MediaDigests` /
|
||||||
|
`Attachments` correspondingly.
|
||||||
|
|
||||||
|
`internal/sdk/` is the bridge implementation for this layer and is not subject to the public
|
||||||
|
interface freeze (see `docs/git-branching.md` §6).
|
||||||
|
|
||||||
### Plugin Interface
|
### Plugin Interface
|
||||||
|
|
||||||
```go
|
```go
|
||||||
|
|||||||
@ -25,6 +25,19 @@ The significance lies in clear responsibility boundaries: the kernel focuses on
|
|||||||
|
|
||||||
Three progressive layers — context, cold archive, long-term graph memory — form an information decay and consolidation pipeline from short-term to persistent storage.
|
Three progressive layers — context, cold archive, long-term graph memory — form an information decay and consolidation pipeline from short-term to persistent storage.
|
||||||
|
|
||||||
|
**Media Memory (since v1.1.0)** — Images and audio are not attachments; they are a kind of node in all three layers:
|
||||||
|
- **Content-addressed store (CAS)**: addressed by digest, metadata in SQLite and blobs on disk, identical bytes
|
||||||
|
stored once. Every `Get` re-verifies the digest (silently returning corrupt data is worse than an error).
|
||||||
|
- **Reference-counted GC**: `owner_kind/owner_id/digest` is the primary key; context events, documents and graph
|
||||||
|
sentences each hold their own references. **Referenced items are never deleted** — only unowned content past
|
||||||
|
`minAge` is reclaimed.
|
||||||
|
- **The description text is the durable semantic memory**: what the vision model produced is written into
|
||||||
|
plain-text memory as a `[<mime> <short digest>] <description>` marker and participates in vector retrieval and
|
||||||
|
distillation; the blob is only a cache that capacity GC may evict. Months later "that purple-blue-red
|
||||||
|
three-band chart" is still findable — via the description, not the bytes.
|
||||||
|
- **Reaches the plugin boundary since v1.1.1**: plugins read and write media through `InsertWithMedia` /
|
||||||
|
`InjectInputMedia`; the model attaches media via the `media_digests` argument of `memory_commit` / `doc_commit`.
|
||||||
|
|
||||||
## What It Actually Does
|
## What It Actually Does
|
||||||
|
|
||||||
Code is in the project root, implemented in Go.
|
Code is in the project root, implemented in Go.
|
||||||
|
|||||||
@ -189,6 +189,36 @@ eventLoop() → processTextInput()
|
|||||||
| `doc_query` | 从 Document 搜索 |
|
| `doc_query` | 从 Document 搜索 |
|
||||||
| `doc_commit` | 写入 Document |
|
| `doc_commit` | 写入 Document |
|
||||||
|
|
||||||
|
`memory_commit` 与 `doc_commit` 自 v1.1.1 起接受 `media_digests`,并由内核把
|
||||||
|
`[<mime> <短digest>] <描述>` 标记补进句子/正文——**标记由内核拼,模型只给 digest**。
|
||||||
|
要求调用方知道格式,等于让一个拼写错误静默切断引用绑定而全链路无人报错。
|
||||||
|
`memory_commit` 同时新增 `sentence_text`:媒体引用挂在句子上,没有句子就无处可挂。
|
||||||
|
|
||||||
|
### 媒体记忆(v1.1.0 起)
|
||||||
|
|
||||||
|
`internal/memory/media/` — `Store`,内容寻址(CAS)
|
||||||
|
|
||||||
|
| 关注点 | 做法 | 为何 |
|
||||||
|
|---|---|---|
|
||||||
|
| 寻址 | sha256 digest,元数据在 SQLite、blob 在磁盘 | 相同字节只存一份;元数据要可查询,blob 不该进数据库 |
|
||||||
|
| 完整性 | 每次 `Get` 重校 digest | 磁盘损坏时静默返回脏数据比报错危险得多 |
|
||||||
|
| 写入原子性 | `.tmp` + rename | 半个文件被当成完整内容会永久污染那个 digest |
|
||||||
|
| 引用 | `owner_kind/owner_id/digest` 三元组主键,`AddRef` 幂等 | 三个 owner 类型:`context`(上下文事件)、`document`(文档)、`graph_sentence`(图谱句子) |
|
||||||
|
| GC | 两阶段 + `minAge`,**有引用者绝不删** | 描述文本留在文本层,blob 可淘汰——语义记忆与字节缓存分离 |
|
||||||
|
|
||||||
|
**媒体在纯文本记忆里的表示**是标记 `[<mime> <短digest>] <描述>`:
|
||||||
|
|
||||||
|
```
|
||||||
|
[image/png a1b2c3d4e5f6] 一张紫蓝红三色带图
|
||||||
|
```
|
||||||
|
|
||||||
|
之所以必须借文本承载:`Doc.Content`、`sentences.text`、文本记忆的 `Input` 全是字符串,
|
||||||
|
没有字段能挂结构化数据。**描述文本才是持久的语义记忆**(检索靠它),digest 是回到字节的
|
||||||
|
钥匙(反查靠它)。blob 被容量 GC 淘汰后,描述仍留在 L0/L2/L3 的文本里。
|
||||||
|
|
||||||
|
媒体存储**全程可选**:`core.memory.media.enabled=false` 或未配置时,整条链路静默退化为
|
||||||
|
纯文本行为,不报错不 panic。
|
||||||
|
|
||||||
### 其他记忆层
|
### 其他记忆层
|
||||||
|
|
||||||
- **Social** (`internal/memory/social/social.go`) — 人格特质和关系网,包装 GraphDB 实体类型
|
- **Social** (`internal/memory/social/social.go`) — 人格特质和关系网,包装 GraphDB 实体类型
|
||||||
@ -294,10 +324,14 @@ Lua 脚本插件加载:`internal/plugin/` → gopher-lua 解释器执行 `main
|
|||||||
|
|
||||||
| 面 | 机制 | 为何这么选 |
|
| 面 | 机制 | 为何这么选 |
|
||||||
|---|---|---|
|
|---|---|---|
|
||||||
| 控制面 | stdio JSON-RPC(NDJSON 帧),51 个 `core.*` method | 进程边界即 ABI 边界,无需维护三套平台特定的动态库加载代码 |
|
| 控制面 | stdio JSON-RPC(NDJSON 帧),55 个 `core.*` method | 进程边界即 ABI 边界,无需维护三套平台特定的动态库加载代码 |
|
||||||
| 数据面 | 共享内存段,**全部子进程共用一块** | 每插件一段会让「内核 ctx → 段 → 插件改 → 回读 ctx」在多插件下退化成副本模型,lost update 原样复现 |
|
| 数据面 | 共享内存段,**全部子进程共用一块** | 每插件一段会让「内核 ctx → 段 → 插件改 → 回读 ctx」在多插件下退化成副本模型,lost update 原样复现 |
|
||||||
| 通知面 | 事件环 + 平台通知(Linux eventfd / macOS pipe / Windows Event) | 内核发事件绕不等消费者,流式输出逐 token 发布时任何等待都会造成卡顿 |
|
| 通知面 | 事件环 + 平台通知(Linux eventfd / macOS pipe / Windows Event) | 内核发事件绕不等消费者,流式输出逐 token 发布时任何等待都会造成卡顿 |
|
||||||
|
|
||||||
|
v1.1.1 新增 4 个 method(51 → 55):`doc.insertWithMedia`、`io.injectMedia`、
|
||||||
|
`io.injectMediaSync`、`io.injectInterruptMedia`。**媒体块走 JSON 而非共享段二进制通道**——
|
||||||
|
data URL 本身已是 base64 文本,包进二进制传输省不了空间,还要跟其余 51 个 method 分道。
|
||||||
|
|
||||||
**子进程生命周期管理**:
|
**子进程生命周期管理**:
|
||||||
- 每子进程一根专职 `waitLoop`(`cmd.Wait()` 唯一调用点)——不依赖 stdout EOF,
|
- 每子进程一根专职 `waitLoop`(`cmd.Wait()` 唯一调用点)——不依赖 stdout EOF,
|
||||||
因为插件 fork 的孙子进程(browser 拉 chromium、editdoc 拉 python)继承同一 stdout,
|
因为插件 fork 的孙子进程(browser 拉 chromium、editdoc 拉 python)继承同一 stdout,
|
||||||
@ -330,8 +364,20 @@ sdk.Memory().Recall/Commit
|
|||||||
sdk.Knowledge().Search/Create
|
sdk.Knowledge().Search/Create
|
||||||
sdk.Settings().Get/Set/List
|
sdk.Settings().Get/Set/List
|
||||||
sdk.RegisterOutputChannel("qq", sdk.CapText|sdk.CapAudio|sdk.CapImage, "QQ消息通道,详见 output_send__qq_help", handler)
|
sdk.RegisterOutputChannel("qq", sdk.CapText|sdk.CapAudio|sdk.CapImage, "QQ消息通道,详见 output_send__qq_help", handler)
|
||||||
|
|
||||||
|
// v1.1.1 媒体接口(全部新增,无签名变更)
|
||||||
|
sdk.DocMemory().InsertWithMedia(doc, attachments) // 带 Data 的落进 CAS,只给 Digest 的引用已有内容
|
||||||
|
sdk.InjectInputMedia(source, channel, text, blocks) // 媒体在「本轮」就发给模型
|
||||||
|
sdk.InjectInputMediaSync(...) // 同上并同步等回复
|
||||||
|
sdk.InjectInterruptMedia(...) // 带媒体的中断,可抢占当前处理
|
||||||
```
|
```
|
||||||
|
|
||||||
|
媒体注入与 `SetToolBlocks` 的区别:后者只能在工具处理函数内部调用,且媒体要等**下一条**
|
||||||
|
tool message 才到模型手上;前三个是插件**主动发起一轮带媒体的对话**,媒体随本轮消息发出,
|
||||||
|
并自动落进 CAS、挂上媒体记忆引用。`Triple` 与 `Doc` 相应新增 `MediaDigests`、`Attachments`。
|
||||||
|
|
||||||
|
`internal/sdk/` 是这层的桥接实现,不受公开接口冻结约束(见 `docs/git-branching.md` §六)。
|
||||||
|
|
||||||
### Plugin 接口
|
### Plugin 接口
|
||||||
|
|
||||||
```go
|
```go
|
||||||
|
|||||||
@ -25,6 +25,17 @@ HomeAgent 是一个持续运行的个人智能 Agent 框架。
|
|||||||
|
|
||||||
三层递进:上下文 → 冷归档 → 长期图记忆,构成从短期到持久的信息衰减与整合管道。
|
三层递进:上下文 → 冷归档 → 长期图记忆,构成从短期到持久的信息衰减与整合管道。
|
||||||
|
|
||||||
|
**媒体记忆(v1.1.0 起)** — 图片/音频不是附属物,而是三层里的一类节点:
|
||||||
|
- **内容寻址存储(CAS)**:digest 寻址,元数据在 SQLite、blob 在磁盘,相同字节只存一份,
|
||||||
|
每次 `Get` 重校 digest(磁盘损坏静默返回脏数据比报错更危险)
|
||||||
|
- **引用计数 GC**:`owner_kind/owner_id/digest` 三元组为主键,上下文事件/文档/图谱句子各自持引用;
|
||||||
|
**有引用者绝不删除**,仅回收无主且超过 `minAge` 的内容
|
||||||
|
- **描述文本才是持久语义记忆**:视觉模型生成的描述以
|
||||||
|
`[<mime> <短digest>] <描述>` 标记形式写进纯文本记忆,参与向量检索与蒸馏;
|
||||||
|
blob 只是可被容量 GC 淘汰的缓存。几个月后“那张紫蓝红三色带图”仍可检索,靠的是描述而不是字节
|
||||||
|
- **v1.1.1 起贯通插件边界**:插件可通过 `InsertWithMedia` / `InjectInputMedia` 读写媒体,
|
||||||
|
模型可用 `memory_commit` / `doc_commit` 的 `media_digests` 参数关联媒体
|
||||||
|
|
||||||
## 它实际做了什么
|
## 它实际做了什么
|
||||||
|
|
||||||
代码位于项目仓库根目录,Go 语言实现。
|
代码位于项目仓库根目录,Go 语言实现。
|
||||||
|
|||||||
@ -25,9 +25,11 @@ import (
|
|||||||
luapkg "gitcode.com/JianFeeeee/HomeAgent/internal/lua"
|
luapkg "gitcode.com/JianFeeeee/HomeAgent/internal/lua"
|
||||||
"gitcode.com/JianFeeeee/HomeAgent/internal/memory"
|
"gitcode.com/JianFeeeee/HomeAgent/internal/memory"
|
||||||
"gitcode.com/JianFeeeee/HomeAgent/internal/memory/document"
|
"gitcode.com/JianFeeeee/HomeAgent/internal/memory/document"
|
||||||
|
"gitcode.com/JianFeeeee/HomeAgent/internal/memory/media"
|
||||||
"gitcode.com/JianFeeeee/HomeAgent/internal/memory/pipeline"
|
"gitcode.com/JianFeeeee/HomeAgent/internal/memory/pipeline"
|
||||||
"gitcode.com/JianFeeeee/HomeAgent/internal/memory/social"
|
"gitcode.com/JianFeeeee/HomeAgent/internal/memory/social"
|
||||||
"gitcode.com/JianFeeeee/HomeAgent/internal/memory/text"
|
"gitcode.com/JianFeeeee/HomeAgent/internal/memory/text"
|
||||||
|
"gitcode.com/JianFeeeee/HomeAgent/internal/memory/vector"
|
||||||
"gitcode.com/JianFeeeee/HomeAgent/internal/meta"
|
"gitcode.com/JianFeeeee/HomeAgent/internal/meta"
|
||||||
"gitcode.com/JianFeeeee/HomeAgent/internal/nlp"
|
"gitcode.com/JianFeeeee/HomeAgent/internal/nlp"
|
||||||
"gitcode.com/JianFeeeee/HomeAgent/internal/plugin"
|
"gitcode.com/JianFeeeee/HomeAgent/internal/plugin"
|
||||||
@ -41,10 +43,21 @@ import (
|
|||||||
sdk "gitcode.com/JianFeeeee/HomeAgent/internal/sdk"
|
sdk "gitcode.com/JianFeeeee/HomeAgent/internal/sdk"
|
||||||
"gitcode.com/JianFeeeee/HomeAgent/internal/supervisor"
|
"gitcode.com/JianFeeeee/HomeAgent/internal/supervisor"
|
||||||
"gitcode.com/JianFeeeee/HomeAgent/internal/tracker"
|
"gitcode.com/JianFeeeee/HomeAgent/internal/tracker"
|
||||||
|
"gitcode.com/JianFeeeee/HomeAgent/pkg/embedding"
|
||||||
"gitcode.com/JianFeeeee/HomeAgent/pkg/types"
|
"gitcode.com/JianFeeeee/HomeAgent/pkg/types"
|
||||||
|
|
||||||
|
// 空白导入内置 provider:它们各自在 init 里注册到 pkg/embedding。
|
||||||
|
// 想把核心换成自己的模型,只需替换这一行(或另建一个发行版 main)。
|
||||||
|
_ "gitcode.com/JianFeeeee/HomeAgent/providers/chineseclip"
|
||||||
|
_ "gitcode.com/JianFeeeee/HomeAgent/providers/qwen3vl"
|
||||||
)
|
)
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
|
// 平台门放在最前面:比 flag 解析还早,因为原生 Windows 上根本不应进入任何
|
||||||
|
// 初始化路径(会去建共享段、拉插件进程)。理由与 WSL 指引见
|
||||||
|
// platform_windows.go。
|
||||||
|
requireSupportedPlatform()
|
||||||
|
|
||||||
dataDir := flag.String("data", "", "data directory (default: auto-detect next to binary)")
|
dataDir := flag.String("data", "", "data directory (default: auto-detect next to binary)")
|
||||||
httpAddr := flag.String("webui", "", "webui listen address (default: webui.listen_addr from config)")
|
httpAddr := flag.String("webui", "", "webui listen address (default: webui.listen_addr from config)")
|
||||||
cliSocket := flag.String("socket", "", "cli unix socket path (default: <data>/cli.sock)")
|
cliSocket := flag.String("socket", "", "cli unix socket path (default: <data>/cli.sock)")
|
||||||
@ -295,6 +308,8 @@ func main() {
|
|||||||
ContextWindow: src.ContextWindow,
|
ContextWindow: src.ContextWindow,
|
||||||
MaxConcurrent: src.MaxConcurrent,
|
MaxConcurrent: src.MaxConcurrent,
|
||||||
Priority: src.Priority,
|
Priority: src.Priority,
|
||||||
|
Vision: src.Vision,
|
||||||
|
Audio: src.Audio,
|
||||||
}, luaVM, src.Name, src.Adapter)
|
}, luaVM, src.Name, src.Adapter)
|
||||||
providerMgr.Register(src.Name, luaProvider)
|
providerMgr.Register(src.Name, luaProvider)
|
||||||
if src.Adapter != "" {
|
if src.Adapter != "" {
|
||||||
@ -317,11 +332,64 @@ func main() {
|
|||||||
// 文档记忆 + 知识库
|
// 文档记忆 + 知识库
|
||||||
// ========================================================================
|
// ========================================================================
|
||||||
|
|
||||||
docStore := document.NewStore(filepath.Join(cfg.Daemon.DataDir, "memory", "documents"))
|
docStore := document.NewStore(filepath.Join(cfg.Daemon.DataDir, "memory", "documents"), memory.TokenizeWords)
|
||||||
if err := docStore.Start(); err != nil {
|
if err := docStore.Start(); err != nil {
|
||||||
log.Printf("[homed] warning: document store: %v", err)
|
log.Printf("[homed] warning: document store: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 媒体存储(内容寻址):记忆块的内容后端。
|
||||||
|
// 开关默认开;关闭后全部媒体接线静默跳过,对话行为与本特性上线前一致。
|
||||||
|
var mediaStore *media.Store
|
||||||
|
if cfgReg.GetBool("core.memory.media.enabled", true) {
|
||||||
|
mediaDir := cfgReg.GetString("core.memory.media.dir",
|
||||||
|
filepath.Join(cfg.Daemon.DataDir, "memory", "media"))
|
||||||
|
ms, err := media.New(mediaDir)
|
||||||
|
if err != nil {
|
||||||
|
// 媒体存储开不起来不该阻止启动——它是记忆增强,不是对话必需品
|
||||||
|
log.Printf("[homed] warning: media store: %v(媒体记忆已禁用)", err)
|
||||||
|
} else {
|
||||||
|
mediaStore = ms
|
||||||
|
defer mediaStore.Close()
|
||||||
|
st := mediaStore.Stats()
|
||||||
|
log.Printf("[homed] media store active: %v 条 / %v 字节",
|
||||||
|
st["count"], st["total_bytes"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 统一多模态向量空间。
|
||||||
|
//
|
||||||
|
// 核心**不**知道任何具体模型:它只按配置里的 provider 名从公共注册表
|
||||||
|
// (pkg/embedding)打开一个 provider,并把 options.* 原样交给它。模型文件
|
||||||
|
// 布局、预处理、解码、运行时全部属于 provider 内部实现。
|
||||||
|
// provider 名为空时禁用多模态向量检索,退回纯 fastText 文本路径。
|
||||||
|
var multimodalSpace vector.MultimodalEmbedder
|
||||||
|
if mmProvider := cfgReg.GetString("core.memory.multimodal_space.provider", ""); mmProvider != "" {
|
||||||
|
opts := map[string]string{}
|
||||||
|
const optPrefix = "core.memory.multimodal_space.options."
|
||||||
|
for _, key := range cfgReg.List("core.memory.multimodal_space.options.") {
|
||||||
|
opts[strings.TrimPrefix(key, optPrefix)] = cfgReg.GetString(key, "")
|
||||||
|
}
|
||||||
|
provider, err := embedding.Open(mmProvider, embedding.Config{Options: opts})
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("[homed] warning: 多模态向量 provider %q 打开失败: %v(多模态向量检索已禁用;已注册: %s)",
|
||||||
|
mmProvider, err, strings.Join(embedding.Names(), ", "))
|
||||||
|
} else if adapted, err := vector.AdaptProvider(provider); err != nil {
|
||||||
|
provider.Close()
|
||||||
|
log.Printf("[homed] warning: 多模态向量 provider %q 元数据不合法: %v(多模态向量检索已禁用)", mmProvider, err)
|
||||||
|
} else {
|
||||||
|
multimodalSpace = adapted
|
||||||
|
defer adapted.Close()
|
||||||
|
info := provider.Info()
|
||||||
|
// 指纹可能很长(模型文件哈希),日志里只取前 12 个字符便于对照。
|
||||||
|
shortFP := info.Fingerprint
|
||||||
|
if len(shortFP) > 12 {
|
||||||
|
shortFP = shortFP[:12]
|
||||||
|
}
|
||||||
|
log.Printf("[homed] multimodal space active: provider=%s dim=%d fp=%s modalities=%v",
|
||||||
|
mmProvider, info.Dimension, shortFP, info.Modalities)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
ks := knowledge.NewStore(filepath.Join(cfg.Daemon.DataDir, "knowledge"))
|
ks := knowledge.NewStore(filepath.Join(cfg.Daemon.DataDir, "knowledge"))
|
||||||
if err := ks.Start(); err != nil {
|
if err := ks.Start(); err != nil {
|
||||||
log.Printf("[homed] warning: knowledge store: %v", err)
|
log.Printf("[homed] warning: knowledge store: %v", err)
|
||||||
@ -354,6 +422,7 @@ func main() {
|
|||||||
pluginReg.SetMemory(memDB)
|
pluginReg.SetMemory(memDB)
|
||||||
pluginReg.SetTextMemory(textMem)
|
pluginReg.SetTextMemory(textMem)
|
||||||
pluginReg.SetDocStore(docStore)
|
pluginReg.SetDocStore(docStore)
|
||||||
|
pluginReg.SetMediaStore(mediaStore) // 插件写入的记忆也走媒体链路;nil 时静默降级
|
||||||
pluginReg.SetKnowledge(ks)
|
pluginReg.SetKnowledge(ks)
|
||||||
pluginReg.SetProviderManager(providerMgr)
|
pluginReg.SetProviderManager(providerMgr)
|
||||||
pluginReg.SetConfigRegistry(cfgReg)
|
pluginReg.SetConfigRegistry(cfgReg)
|
||||||
@ -429,6 +498,7 @@ func main() {
|
|||||||
Knowledge: ks,
|
Knowledge: ks,
|
||||||
SocialStore: socialStore,
|
SocialStore: socialStore,
|
||||||
TextMemory: textMem,
|
TextMemory: textMem,
|
||||||
|
MediaStore: mediaStore,
|
||||||
Personality: personality,
|
Personality: personality,
|
||||||
PluginReg: pluginReg,
|
PluginReg: pluginReg,
|
||||||
PluginDir: cfg.Plugin.Dir,
|
PluginDir: cfg.Plugin.Dir,
|
||||||
@ -439,6 +509,7 @@ func main() {
|
|||||||
ContextSavePath: filepath.Join(cfg.Daemon.DataDir, "memory", "context.json"),
|
ContextSavePath: filepath.Join(cfg.Daemon.DataDir, "memory", "context.json"),
|
||||||
EmbeddingModelPath: cfgReg.GetString("core.agent.embedding_model_path", ""),
|
EmbeddingModelPath: cfgReg.GetString("core.agent.embedding_model_path", ""),
|
||||||
Embedder: embedder,
|
Embedder: embedder,
|
||||||
|
MultimodalSpace: multimodalSpace,
|
||||||
StageHost: stageHost,
|
StageHost: stageHost,
|
||||||
EventBus: evBus,
|
EventBus: evBus,
|
||||||
ThinkingEnabled: cfg.LLM.ThinkingEnabled,
|
ThinkingEnabled: cfg.LLM.ThinkingEnabled,
|
||||||
|
|||||||
9
cmd/homed/platform_other.go
Normal file
9
cmd/homed/platform_other.go
Normal file
@ -0,0 +1,9 @@
|
|||||||
|
//go:build !windows
|
||||||
|
|
||||||
|
package main
|
||||||
|
|
||||||
|
// requireSupportedPlatform 在受支持的平台上不做任何事。
|
||||||
|
//
|
||||||
|
// 平台策略见 platform_windows.go:只有 homed 放弃 Windows 原生支持
|
||||||
|
// (插件体系依赖 fd 继承与共享内存段内偏移),Windows 用户走 WSL2。
|
||||||
|
func requireSupportedPlatform() {}
|
||||||
44
cmd/homed/platform_windows.go
Normal file
44
cmd/homed/platform_windows.go
Normal file
@ -0,0 +1,44 @@
|
|||||||
|
//go:build windows
|
||||||
|
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
)
|
||||||
|
|
||||||
|
// requireSupportedPlatform 在原生 Windows 上直接拒绝启动 homed。
|
||||||
|
//
|
||||||
|
// 为什么不做原生支持(不是「还没来得及做」,是设计上不做):
|
||||||
|
//
|
||||||
|
// homed 的插件体系建立在两个原语上——**继承的 fd**(Single memfd: 统一共享
|
||||||
|
// 内存区 + eventfd 通知)与**同段内相对偏移解引用**(各进程 mmap 到不同虚拟
|
||||||
|
// 基址,段内一律用偏移互相读写,这样插件回调才能就地改写内核看到的那份数据)。
|
||||||
|
//
|
||||||
|
// Windows 的等价物是命名内核对象(CreateFileMappingW / OpenEventW)+句柄表,
|
||||||
|
// 没有 fd 继承语义(os/exec 的 ExtraFiles 在 Windows 上直接不被支持),
|
||||||
|
// 生命周期与权限模型也按句柄而非进程继承来组织。要在其上重建这套语义,
|
||||||
|
// 等于再维护一套平台专属 ABI 与安全边界——而 C ABI 时代正是「三套 ABI 并存
|
||||||
|
// 导致改写型插件在某个平台上静默失效」的教训(§9.2)。
|
||||||
|
//
|
||||||
|
// 所以选择:**原生 Windows 不提供 homed**。Windows 用户跑 WSL2——
|
||||||
|
// WSL2 里就是普通 linux/amd64,走与我们测试矩阵完全相同的那条路径。
|
||||||
|
//
|
||||||
|
// 注意范围:只有 homed 如此。plugindev 工具链仍可在 Windows 上运行
|
||||||
|
// (在 Windows 上开发、为 WSL 构建 linux 插件是合理工作流)。
|
||||||
|
func requireSupportedPlatform() {
|
||||||
|
fmt.Fprintln(os.Stderr, "homed 不支持 Windows 原生运行。")
|
||||||
|
fmt.Fprintln(os.Stderr, "")
|
||||||
|
fmt.Fprintln(os.Stderr, "原因:子进程插件依赖 fd 继承 + 统一共享内存区的段内偏移解引用,")
|
||||||
|
fmt.Fprintln(os.Stderr, "而 Windows 的句柄模型无法表达这两者;强行适配等于再维护一套平台专属")
|
||||||
|
fmt.Fprintln(os.Stderr, "ABI——C ABI 时代三套 ABI 并存曾导致改写型插件在某个平台上静默失效。")
|
||||||
|
fmt.Fprintln(os.Stderr, "")
|
||||||
|
fmt.Fprintln(os.Stderr, "请改用 WSL2:")
|
||||||
|
fmt.Fprintln(os.Stderr, " 1. wsl --install -d Ubuntu # 安装 WSL2")
|
||||||
|
fmt.Fprintln(os.Stderr, " 2. 在 WSL 内下载 linux/amd64 的 homed 与插件(.hmap)")
|
||||||
|
fmt.Fprintln(os.Stderr, " 3. 在 WSL 内运行 homed:与 Linux 主机完全相同,无需额外配置")
|
||||||
|
fmt.Fprintln(os.Stderr, "")
|
||||||
|
fmt.Fprintln(os.Stderr, "数据目录可放在 /mnt/c/... 下以便与 Windows 侧共享,")
|
||||||
|
fmt.Fprintln(os.Stderr, "但不建议(跨文件系统 IO 慢、inotify 语义受限);推荐放在 WSL 内部路径。")
|
||||||
|
os.Exit(2)
|
||||||
|
}
|
||||||
@ -17,6 +17,36 @@ func randomSecret(n int) string {
|
|||||||
return hex.EncodeToString(b)
|
return hex.EncodeToString(b)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// must 让失败真正停下来。
|
||||||
|
//
|
||||||
|
// 这里曾经把所有 db.Exec 的返回值丢掉,配合 CGO_ENABLED=0 构建(go-sqlite3
|
||||||
|
// 退化成静态桩),得到的是一个**完全静默的空操作**:打印凭据、退出码 0、
|
||||||
|
// config.db 里一个字节都没写。调用方(安装脚本)无法区分成败,用户装完
|
||||||
|
// 照着 credentials.txt 登录必然失败。
|
||||||
|
func must(err error) {
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "initconfig: %v\n", err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// verify 回读刚写入的值。
|
||||||
|
//
|
||||||
|
// 只看 Exec 有没有报错不够:驱动被换掉(如上面的桩)、路径不对、写入被丢弃,
|
||||||
|
// 都可能返回 nil 而什么都没落下。这里把真实落盘的值读回来,与预期逐一比对,
|
||||||
|
// 不一致就非零退出——"初始化脚本说自己成功了"必须由数据库内容佐证。
|
||||||
|
func verify(db *sql.DB, table, key, want string) {
|
||||||
|
var got string
|
||||||
|
if err := db.QueryRow(fmt.Sprintf(`SELECT value FROM %s WHERE key = ?`, table), key).Scan(&got); err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "initconfig: 回读 %s.%s 失败: %v\n", table, key, err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
if got != want {
|
||||||
|
fmt.Fprintf(os.Stderr, "initconfig: %s.%s 与写入值不一致(读回 %q)\n", table, key, got)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
dataDir := flag.String("data", "", "data directory")
|
dataDir := flag.String("data", "", "data directory")
|
||||||
webuiUsername := flag.String("username", "admin", "webui username")
|
webuiUsername := flag.String("username", "admin", "webui username")
|
||||||
@ -33,16 +63,24 @@ func main() {
|
|||||||
|
|
||||||
dbPath := *dataDir + "/config.db"
|
dbPath := *dataDir + "/config.db"
|
||||||
db, err := sql.Open("sqlite3", dbPath)
|
db, err := sql.Open("sqlite3", dbPath)
|
||||||
if err != nil {
|
must(err)
|
||||||
fmt.Fprintf(os.Stderr, "open db: %v\n", err)
|
|
||||||
os.Exit(1)
|
|
||||||
}
|
|
||||||
defer db.Close()
|
defer db.Close()
|
||||||
|
|
||||||
db.Exec("PRAGMA journal_mode=WAL")
|
// 尽早验证数据库真的可用:sql.Open 是惰性的,不碰一次不会暴露驱动问题。
|
||||||
|
if _, err := db.Exec("PRAGMA journal_mode=WAL"); err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "initconfig: 打开数据库 %s 失败: %v\n", dbPath, err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
|
||||||
db.Exec(`CREATE TABLE IF NOT EXISTS config (key TEXT PRIMARY KEY, value TEXT NOT NULL)`)
|
if _, err := db.Exec(`CREATE TABLE IF NOT EXISTS config (key TEXT PRIMARY KEY, value TEXT NOT NULL)`); err != nil {
|
||||||
db.Exec(`INSERT OR IGNORE INTO config (key, value) VALUES (?, ?)`, "webui.listen_addr", ":8080")
|
fmt.Fprintf(os.Stderr, "initconfig: 创建 config 表失败: %v\n", err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
const listenAddr = ":8080"
|
||||||
|
if _, err := db.Exec(`INSERT OR IGNORE INTO config (key, value) VALUES (?, ?)`, "webui.listen_addr", listenAddr); err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "initconfig: 写入 webui.listen_addr 失败: %v\n", err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
|
||||||
pw := *webuiPassword
|
pw := *webuiPassword
|
||||||
if pw == "" {
|
if pw == "" {
|
||||||
@ -53,13 +91,28 @@ func main() {
|
|||||||
apiKey = randomSecret(16)
|
apiKey = randomSecret(16)
|
||||||
}
|
}
|
||||||
|
|
||||||
pt := "config_webui"
|
const pt = "config_webui"
|
||||||
db.Exec(fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s (key TEXT PRIMARY KEY, value TEXT NOT NULL)`, pt))
|
if _, err := db.Exec(fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s (key TEXT PRIMARY KEY, value TEXT NOT NULL)`, pt)); err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "initconfig: 创建 %s 表失败: %v\n", pt, err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
ws := fmt.Sprintf(`INSERT OR REPLACE INTO %s (key, value) VALUES (?, ?)`, pt)
|
ws := fmt.Sprintf(`INSERT OR REPLACE INTO %s (key, value) VALUES (?, ?)`, pt)
|
||||||
db.Exec(ws, "api_key", apiKey)
|
for _, kv := range [][2]string{
|
||||||
db.Exec(ws, "username", *webuiUsername)
|
{"api_key", apiKey},
|
||||||
db.Exec(ws, "password", pw)
|
{"username", *webuiUsername},
|
||||||
db.Exec(ws, "session_ttl_hours", "24")
|
{"password", pw},
|
||||||
|
{"session_ttl_hours", "24"},
|
||||||
|
} {
|
||||||
|
if _, err := db.Exec(ws, kv[0], kv[1]); err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "initconfig: 写入 %s.%s 失败: %v\n", pt, kv[0], err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
verify(db, pt, "api_key", apiKey)
|
||||||
|
verify(db, pt, "username", *webuiUsername)
|
||||||
|
verify(db, pt, "password", pw)
|
||||||
|
verify(db, "config", "webui.listen_addr", listenAddr)
|
||||||
|
|
||||||
fmt.Printf("API_KEY=%s\n", apiKey)
|
fmt.Printf("API_KEY=%s\n", apiKey)
|
||||||
fmt.Printf("WEBUI_USERNAME=%s\n", *webuiUsername)
|
fmt.Printf("WEBUI_USERNAME=%s\n", *webuiUsername)
|
||||||
|
|||||||
@ -6,9 +6,11 @@ Wants=network-online.target
|
|||||||
|
|
||||||
[Service]
|
[Service]
|
||||||
Type=simple
|
Type=simple
|
||||||
ExecStart=/usr/local/bin/homed -data /var/lib/homeagent
|
ExecStart=/usr/bin/homed -data /var/lib/homeagent
|
||||||
Restart=always
|
Restart=always
|
||||||
RestartSec=10
|
RestartSec=10
|
||||||
|
Environment=ONNXRUNTIME_DIR=/usr/lib/homeagent/onnxruntime
|
||||||
|
StateDirectory=homeagent
|
||||||
StartLimitBurst=3
|
StartLimitBurst=3
|
||||||
StartLimitInterval=60s
|
StartLimitInterval=60s
|
||||||
|
|
||||||
@ -26,7 +28,9 @@ DeviceAllow=/dev/dsp rw
|
|||||||
# Resource limits
|
# Resource limits
|
||||||
LimitNOFILE=65536
|
LimitNOFILE=65536
|
||||||
LimitNPROC=256
|
LimitNPROC=256
|
||||||
MemoryMax=2G
|
# Chinese-CLIP 本身实测约 1.15GB;加文档稠密索引、词向量、插件与
|
||||||
|
# ORT arena 后生产实例约 4.5GB。2GB 会在首次全量建索引时被 cgroup OOM。
|
||||||
|
MemoryMax=8G
|
||||||
CPUQuota=100%
|
CPUQuota=100%
|
||||||
|
|
||||||
[Install]
|
[Install]
|
||||||
|
|||||||
@ -15,6 +15,13 @@ BUILD_TIME="${BUILD_TIME:-$(date -u '+%Y-%m-%dT%H:%M:%SZ')}"
|
|||||||
GO="${GO:-$(command -v go 2>/dev/null || echo "go")}"
|
GO="${GO:-$(command -v go 2>/dev/null || echo "go")}"
|
||||||
LDFLAGS="-X gitcode.com/JianFeeeee/HomeAgent/internal/meta.Version=${VERSION} -X gitcode.com/JianFeeeee/HomeAgent/internal/meta.Commit=${COMMIT} -X gitcode.com/JianFeeeee/HomeAgent/internal/meta.BuildTime=${BUILD_TIME}"
|
LDFLAGS="-X gitcode.com/JianFeeeee/HomeAgent/internal/meta.Version=${VERSION} -X gitcode.com/JianFeeeee/HomeAgent/internal/meta.Commit=${COMMIT} -X gitcode.com/JianFeeeee/HomeAgent/internal/meta.BuildTime=${BUILD_TIME}"
|
||||||
|
|
||||||
|
# 版本与提交的**权威来源**是上面注入的 meta.Version / meta.Commit,不是 Go 自带的
|
||||||
|
# VCS 戳。后者不进 build cache key(Go 文档明确说明 VCS 变化不会触发重建),
|
||||||
|
# 命中缓存时会把上一次的 revision 一并带回来——实测发布分支的产物上就出现了
|
||||||
|
# 1715b5c(本机任何仓库都不存在的提交),用 `go version -m` 溯源会指向幽灵提交。
|
||||||
|
# 统一 -buildvcs=false:宁可没有这个信号,也不要一个错的。
|
||||||
|
# 溯源请用:`strings homed | grep -m1 '^<短 hash>$'`(meta.Commit 是字符串常量)。
|
||||||
|
|
||||||
TARGET="${1:-native}"
|
TARGET="${1:-native}"
|
||||||
COMPONENT="${2:-all}"
|
COMPONENT="${2:-all}"
|
||||||
|
|
||||||
@ -27,10 +34,19 @@ COMPONENT="${2:-all}"
|
|||||||
case "$TARGET" in
|
case "$TARGET" in
|
||||||
native) GOOS="" GOARCH="" ;;
|
native) GOOS="" GOARCH="" ;;
|
||||||
linux/amd64) GOOS=linux GOARCH=amd64 CC="${CC:-}" ;;
|
linux/amd64) GOOS=linux GOARCH=amd64 CC="${CC:-}" ;;
|
||||||
# arm64 刻意不设 CXX:设了会让 Go 用 aarch64 的 g++ 去链接,
|
# arm64 必须同时给 CXX:gojieba 是 C++,缺 CXX 时 cgo 用宿主 g++ 编出
|
||||||
# 而它对 host 产生的 .o 报 "file format not recognized"。
|
# x86-64 的 .o,链接时报 "Relocations in generic ELF (EM: 183)"(183 = aarch64)。
|
||||||
# gojieba 的 C++ 源仍由 CC 对应的 gcc 驱动编译(gcc 能编 C++)。
|
#
|
||||||
linux/arm64) GOOS=linux GOARCH=arm64 CC="${CC:-aarch64-linux-gnu-gcc}" ;;
|
# 此处曾有一条注释写着「arm64 刻意不设 CXX」,理由是设了会报
|
||||||
|
# "file format not recognized"。那个判断是错的:那个报错的真因是
|
||||||
|
# cmd/{homed,waiter}/*.syso(x86-64 COFF Windows 资源对象)被链进了目标,
|
||||||
|
# 与 CXX 无关。四组对照:
|
||||||
|
# syso 在 + 无 CXX → Relocations in generic ELF (EM: 183)
|
||||||
|
# syso 在 + 有 CXX → 000000.o: file format not recognized
|
||||||
|
# syso 隐藏 + 无 CXX → Relocations in generic ELF (EM: 183)
|
||||||
|
# syso 隐藏 + 有 CXX → 成功,ELF aarch64
|
||||||
|
# 本脚本的 hide_syso_for_target 已处理前一个条件,这里补上后一个。
|
||||||
|
linux/arm64) GOOS=linux GOARCH=arm64 CC="${CC:-aarch64-linux-gnu-gcc}" CXX="${CXX:-aarch64-linux-gnu-g++}" ;;
|
||||||
darwin/amd64) GOOS=darwin GOARCH=amd64 CC="${CC:-}" ;;
|
darwin/amd64) GOOS=darwin GOARCH=amd64 CC="${CC:-}" ;;
|
||||||
darwin/arm64) GOOS=darwin GOARCH=arm64 CC="${CC:-}" ;;
|
darwin/arm64) GOOS=darwin GOARCH=arm64 CC="${CC:-}" ;;
|
||||||
# Windows 必须同时给 CXX:gojieba 是 C++,缺 CXX 时 cgo 回退到宿主 g++,
|
# Windows 必须同时给 CXX:gojieba 是 C++,缺 CXX 时 cgo 回退到宿主 g++,
|
||||||
@ -46,7 +62,7 @@ case "$TARGET" in
|
|||||||
;;
|
;;
|
||||||
*)
|
*)
|
||||||
echo "Unknown target: $TARGET"
|
echo "Unknown target: $TARGET"
|
||||||
echo "Usage: $0 [native|linux/amd64|linux/arm64|darwin/amd64|darwin/arm64|windows/amd64|all]"
|
echo "Usage: $0 [native|linux/amd64|linux/arm64|darwin/amd64|darwin/arm64|windows/amd64|all] [all|homed|waiter|initconfig|gui|payload]"
|
||||||
echo " [all|homed|waiter|initconfig|gui]"
|
echo " [all|homed|waiter|initconfig|gui]"
|
||||||
exit 1
|
exit 1
|
||||||
esac
|
esac
|
||||||
@ -109,8 +125,21 @@ build_homed() {
|
|||||||
# Go 用 CC 驱动 CGO 编译与链接,用 CC 指定的交叉工具链来决定目标架构。
|
# Go 用 CC 驱动 CGO 编译与链接,用 CC 指定的交叉工具链来决定目标架构。
|
||||||
# 必须同时 export CC 给 Go 的 CGO 代码生成器,否则 CGO_ENABLED=1 下的
|
# 必须同时 export CC 给 Go 的 CGO 代码生成器,否则 CGO_ENABLED=1 下的
|
||||||
# 目标文件与 host 的 ld 不兼容(如 arm64 的 .o 给了 x86_64 的 ld)。
|
# 目标文件与 host 的 ld 不兼容(如 arm64 的 .o 给了 x86_64 的 ld)。
|
||||||
|
#
|
||||||
|
# HOMED_TAGS 默认带 onnxruntime:发行版**默认启用**本地向量空间。
|
||||||
|
# 不带这个标签时 providers/chineseclip 与 providers/qwen3vl 仍会注册,
|
||||||
|
# 但打开时报「requires build tag」并优雅降级(不静默假装成功)。
|
||||||
|
# 需要极简构建时可显式 HOMED_TAGS= 关掉。
|
||||||
|
#
|
||||||
|
# 运行期还需要 libonnxruntime.so(provider 按 /opt/onnxruntime、
|
||||||
|
# /usr/local/lib、/usr/lib 顺序查找);缺失时同样是「日志里的明确错误 +
|
||||||
|
# 降级」,不会假装启用。
|
||||||
local _cc="${CC:-cc}"
|
local _cc="${CC:-cc}"
|
||||||
CGO_ENABLED=1 CC="$_cc" "$GO" build -trimpath -installsuffix dynlink \
|
local _tags="${HOMED_TAGS-onnxruntime}"
|
||||||
|
local -a _tagargs=()
|
||||||
|
if [ -n "$_tags" ]; then _tagargs=(-tags "$_tags"); fi
|
||||||
|
CGO_ENABLED=1 CC="$_cc" "$GO" build -buildvcs=false -trimpath -installsuffix dynlink \
|
||||||
|
${_tagargs[@]+"${_tagargs[@]}"} \
|
||||||
-ldflags "$LDFLAGS" -o "$out" ./cmd/homed/
|
-ldflags "$LDFLAGS" -o "$out" ./cmd/homed/
|
||||||
echo " OK ($(file "$out" | sed 's/.*: //') | $(du -h "$out" | cut -f1))"
|
echo " OK ($(file "$out" | sed 's/.*: //') | $(du -h "$out" | cut -f1))"
|
||||||
}
|
}
|
||||||
@ -122,28 +151,74 @@ build_waiter() {
|
|||||||
if [ "$GOOS" = "windows" ]; then out="${out}.exe"; fi
|
if [ "$GOOS" = "windows" ]; then out="${out}.exe"; fi
|
||||||
|
|
||||||
echo "[BUILD] waiter ${plat} → $out"
|
echo "[BUILD] waiter ${plat} → $out"
|
||||||
CGO_ENABLED=0 "$GO" build -trimpath -installsuffix dynlink \
|
CGO_ENABLED=0 "$GO" build -buildvcs=false -trimpath -installsuffix dynlink \
|
||||||
-ldflags "$LDFLAGS" -o "$out" ./cmd/waiter/
|
-ldflags "$LDFLAGS" -o "$out" ./cmd/waiter/
|
||||||
echo " OK ($(du -h "$out" | cut -f1))"
|
echo " OK ($(du -h "$out" | cut -f1))"
|
||||||
}
|
}
|
||||||
|
|
||||||
# ---- initconfig (CGO-free 配置初始化器) ----
|
# ---- initconfig(必须 cgo:写 config.db 用的是 go-sqlite3)----
|
||||||
#
|
#
|
||||||
# NSIS 安装包(installer.nsi:220 File "..\build\initconfig.exe")与
|
# 这里**必须** CGO_ENABLED=1。此前写的是 CGO_ENABLED=0,而 cmd/initconfig 通过
|
||||||
# package-linux.sh 的 stage_variant 都引用它,但此前 build.sh 从不构建它——
|
# database/sql 使用 mattn/go-sqlite3:CGO_ENABLED=0 时该库退化成 static_mock.go
|
||||||
# Windows 安装包构建会直接失败在缺文件上。
|
# 里的桩,sql.Open 是懒的所以不报错、第一次 Exec 才失败;而 main.go 当时忽略
|
||||||
|
# 了所有错误——于是 initconfig 打印凭据、退出码 0、一个字节都没写进 config.db。
|
||||||
|
# 安装脚本把这份凭据写进 credentials.txt,用户照它登录必然失败,全程无报错。
|
||||||
|
#
|
||||||
|
# NSIS 安装包(installer.nsi)与 package-linux.sh 的 stage_variant 都引用它,
|
||||||
|
# 但此前 build.sh 从不构建它——Windows 安装包构建会直接失败在缺文件上。
|
||||||
build_initconfig() {
|
build_initconfig() {
|
||||||
local plat="${GOOS:-linux}/${GOARCH:-amd64}"
|
local plat="${GOOS:-linux}/${GOARCH:-amd64}"
|
||||||
local out="$BUILD_DIR/initconfig${SUFFIX:+_$SUFFIX}"
|
local out="$BUILD_DIR/initconfig${SUFFIX:+_$SUFFIX}"
|
||||||
if [ "$GOOS" = "windows" ]; then out="${out}.exe"; fi
|
if [ "$GOOS" = "windows" ]; then out="${out}.exe"; fi
|
||||||
|
|
||||||
echo "[BUILD] initconfig ${plat} → $out"
|
echo "[BUILD] initconfig ${plat} → $out"
|
||||||
CGO_ENABLED=0 "$GO" build -trimpath -installsuffix dynlink \
|
CGO_ENABLED=1 "$GO" build -buildvcs=false -trimpath -installsuffix dynlink \
|
||||||
-ldflags "$LDFLAGS" -o "$out" ./cmd/initconfig/
|
-ldflags "$LDFLAGS" -o "$out" ./cmd/initconfig/
|
||||||
echo " OK ($(du -h "$out" | cut -f1))"
|
echo " OK ($(du -h "$out" | cut -f1))"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# ---- linux-payload(给 Windows 安装器用的 Linux 包)----
|
||||||
|
#
|
||||||
|
# Windows 不再安装 homed.exe:homed 依赖 fd 继承 + 统一共享内存区的段内偏移
|
||||||
|
# 解引用,Windows 句柄模型无法表达(见 cmd/homed/platform_windows.go)。
|
||||||
|
# Windows 安装器改为引导到 WSL2,并把 **Linux 包**送进发行版里安装。
|
||||||
|
# 因此 Windows 安装包必须带上 Linux 产物——这一段就是把它暂存到
|
||||||
|
# build/linux-payload/(installer.nsi 从这里 File /r 打进安装包)。
|
||||||
|
#
|
||||||
|
# 复用 package-linux.sh 的产物,而不是在这里另行编译:WSL 里跑的就是普通
|
||||||
|
# linux/amd64,安装内容必须与 Linux 原生安装**完全一致**,否则又变成两个平台。
|
||||||
|
stage_linux_payload() {
|
||||||
|
local src="$PROJECT_ROOT/dist/linux"
|
||||||
|
local out="$BUILD_DIR/linux-payload"
|
||||||
|
|
||||||
|
rm -rf "$out"
|
||||||
|
mkdir -p "$out"
|
||||||
|
|
||||||
|
local found=0
|
||||||
|
for f in "$src"/*.deb "$src"/*.tar.gz; do
|
||||||
|
[ -f "$f" ] || continue
|
||||||
|
cp "$f" "$out/"
|
||||||
|
found=$((found + 1))
|
||||||
|
done
|
||||||
|
|
||||||
|
if [ "$found" -eq 0 ]; then
|
||||||
|
echo "[FAIL] build/linux-payload 为空:先运行 package-linux.sh 产出 dist/linux/*.deb|*.tar.gz" >&2
|
||||||
|
echo " (Windows 安装器会把这里的包送进 WSL 安装;空包等于装不上)" >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
echo "[BUILD] linux-payload ← $found 个包"
|
||||||
|
ls -1 "$out" | sed 's/^/ /'
|
||||||
|
}
|
||||||
|
|
||||||
# ---- gui (Electron) ----
|
# ---- gui (Electron) ----
|
||||||
|
#
|
||||||
|
# 输出目录必须用 --config.directories.output,**不能用 -o**:
|
||||||
|
# electron-builder 的 `-o` 是 `--mac`/`--macos` 的短别名(见 --help 的 Building 段),
|
||||||
|
# 不是 output。此前 `-o "$BUILD_DIR"` 被当成 macOS 的 target 列表,报
|
||||||
|
# ⨯ Unknown target: /home/program/trueagent/build
|
||||||
|
# (路径被 lowercase 后去匹配 target 名表,所以错误信息里的路径是全小写的,
|
||||||
|
# 这也是它看起来像「路径错」而实际是「参数位置错」的原因)。
|
||||||
|
# v1.0.1 与 v1.0.3 两次发布都因此手工组装过 GUI。
|
||||||
build_gui() {
|
build_gui() {
|
||||||
if [ -n "${GOOS:-}" ] && [ "$GOOS" != "$("$GO" env GOOS)" ]; then
|
if [ -n "${GOOS:-}" ] && [ "$GOOS" != "$("$GO" env GOOS)" ]; then
|
||||||
echo "[SKIP] gui ${GOOS}/${GOARCH} — electron-builder handles cross-platform natively; run 'all' on CI host"
|
echo "[SKIP] gui ${GOOS}/${GOARCH} — electron-builder handles cross-platform natively; run 'all' on CI host"
|
||||||
@ -161,22 +236,52 @@ build_gui() {
|
|||||||
# 不传 --config:electron-builder 默认从 package.json 的 "build" 键读配置。
|
# 不传 --config:electron-builder 默认从 package.json 的 "build" 键读配置。
|
||||||
# 传 --config package.json 会让它把**整个** package.json 当配置校验,
|
# 传 --config package.json 会让它把**整个** package.json 当配置校验,
|
||||||
# 于是 devDependencies / build / scripts 全被判为 "unknown property" 而失败。
|
# 于是 devDependencies / build / scripts 全被判为 "unknown property" 而失败。
|
||||||
(cd "$gui_dir" && npx electron-builder \
|
#
|
||||||
--linux --win --mac \
|
# GUI 失败不中断整体构建:homed/waiter/initconfig 是发布的主体,
|
||||||
--x64 --arm64 \
|
# 而 GUI 依赖 electron 运行时下载(离线机器、arm64 缺缓存都会失败)。
|
||||||
-p never \
|
# set -e 下若不接住,一个可选组件会让整轮跨平台构建全废。
|
||||||
-o "$BUILD_DIR")
|
if (cd "$gui_dir" && npx electron-builder \
|
||||||
echo " OK"
|
--linux --win --mac \
|
||||||
|
--x64 --arm64 \
|
||||||
|
-p never \
|
||||||
|
--config.directories.output="$BUILD_DIR"); then
|
||||||
|
echo " OK"
|
||||||
|
else
|
||||||
|
echo " WARN: gui 构建失败(可选组件,不影响 homed/waiter/initconfig)"
|
||||||
|
echo " Linux 包可用 deploy/packaging/package-linux.sh 内置的手工组装路径"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
# ---- dispatch ----
|
# ---- dispatch ----
|
||||||
case "$COMPONENT" in
|
if [ "${GOOS:-}" = "windows" ]; then
|
||||||
all) build_homed; build_waiter; build_initconfig; build_gui ;;
|
# Windows 目标:构建的**不是** homed——它已放弃 Windows 原生支持。
|
||||||
homed) build_homed ;;
|
# 需要的是:Linux 包(送进 WSL 安装)+ Windows 侧客户端(waiter CLI / GUI)。
|
||||||
waiter) build_waiter ;;
|
case "$COMPONENT" in
|
||||||
initconfig) build_initconfig ;;
|
all) build_waiter; stage_linux_payload; build_gui ;;
|
||||||
gui) build_gui ;;
|
waiter) build_waiter ;;
|
||||||
*)
|
payload) stage_linux_payload ;;
|
||||||
echo "Unknown component: $COMPONENT"
|
gui) build_gui ;;
|
||||||
exit 1
|
homed|initconfig)
|
||||||
esac
|
echo "homed/initconfig 不再提供 Windows 原生构建:请用 WSL2(或用 linux/amd64 目标)。" >&2
|
||||||
|
echo "原因见 cmd/homed/platform_windows.go。" >&2
|
||||||
|
exit 1
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
echo "Unknown component: $COMPONENT"
|
||||||
|
exit 1
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
else
|
||||||
|
case "$COMPONENT" in
|
||||||
|
all) build_homed; build_waiter; build_initconfig; build_gui ;;
|
||||||
|
homed) build_homed ;;
|
||||||
|
waiter) build_waiter ;;
|
||||||
|
initconfig) build_initconfig ;;
|
||||||
|
gui) build_gui ;;
|
||||||
|
*)
|
||||||
|
echo "Unknown component: $COMPONENT"
|
||||||
|
exit 1
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
fi
|
||||||
|
|||||||
@ -76,6 +76,10 @@ Function GenKey
|
|||||||
FunctionEnd
|
FunctionEnd
|
||||||
|
|
||||||
!insertmacro MUI_PAGE_WELCOME
|
!insertmacro MUI_PAGE_WELCOME
|
||||||
|
; 许可页:AGPL-3.0-only(全文在仓库根 LICENSE)。
|
||||||
|
; NSIS 的 File/!insertmacro 相对路径以**本 .nsi 所在目录**为基准解析,
|
||||||
|
; 而本文件在 deploy/packaging/,故仓库根是 ..\..\ 。
|
||||||
|
!insertmacro MUI_PAGE_LICENSE "..\..\LICENSE"
|
||||||
!insertmacro MUI_PAGE_DIRECTORY
|
!insertmacro MUI_PAGE_DIRECTORY
|
||||||
|
|
||||||
!if "${HAS_CREDENTIALS}" == "1"
|
!if "${HAS_CREDENTIALS}" == "1"
|
||||||
@ -216,17 +220,26 @@ FunctionEnd
|
|||||||
|
|
||||||
Section "Install" SEC_INSTALL
|
Section "Install" SEC_INSTALL
|
||||||
SetOutPath "$INSTDIR"
|
SetOutPath "$INSTDIR"
|
||||||
|
; WSL 引导脚本随安装包分发(它负责检测/引导 WSL 并把 Linux 包装进发行版)
|
||||||
|
File "..\..\deploy\packaging\windows\install-via-wsl.ps1"
|
||||||
CreateDirectory "$INSTDIR\data"
|
CreateDirectory "$INSTDIR\data"
|
||||||
CreateDirectory "$INSTDIR\data\log"
|
CreateDirectory "$INSTDIR\data\log"
|
||||||
CreateDirectory "$INSTDIR\data\plugins"
|
CreateDirectory "$INSTDIR\data\plugins"
|
||||||
CreateDirectory "$INSTDIR\data\adapters"
|
CreateDirectory "$INSTDIR\data\adapters"
|
||||||
|
|
||||||
|
; homed **不再装到 Windows**:插件体系依赖 fd 继承与统一共享内存区的段内偏移
|
||||||
|
; 解引用,Windows 的句柄模型无法表达(见 cmd/homed/platform_windows.go)。
|
||||||
|
; Windows 侧改为引导到 WSL2,把 **Linux 包**送进发行版里按 Linux 的方式安装。
|
||||||
|
; 所以这里带的是 linux/amd64 的 payload,不是 homed.exe。
|
||||||
!if "${HAS_CORE}" == "1"
|
!if "${HAS_CORE}" == "1"
|
||||||
File "..\..\build\initconfig.exe"
|
SetOutPath "$PLUGINSDIR\linux-payload"
|
||||||
File "..\..\build\homed.exe"
|
File /r "..\..\build\linux-payload\*.*"
|
||||||
|
SetOutPath "$INSTDIR"
|
||||||
!endif
|
!endif
|
||||||
|
|
||||||
!if "${HAS_WAITER}" == "1"
|
!if "${HAS_WAITER}" == "1"
|
||||||
|
; waiter 是 CLI 客户端:WSL 侧会装上 Linux 版;Windows 侧仍可保留原生版
|
||||||
|
; (它只是个客户端,不走插件体系)。
|
||||||
File "..\..\build\waiter.exe"
|
File "..\..\build\waiter.exe"
|
||||||
!endif
|
!endif
|
||||||
|
|
||||||
@ -237,11 +250,24 @@ Section "Install" SEC_INSTALL
|
|||||||
!endif
|
!endif
|
||||||
|
|
||||||
!if "${HAS_CORE}" == "1"
|
!if "${HAS_CORE}" == "1"
|
||||||
DetailPrint "初始化配置数据库..."
|
; 在 WSL2 里安装 homed。凭据(页面上收的那三个)透传进去,避免
|
||||||
nsExec::Exec '"$INSTDIR\initconfig.exe" -data "$INSTDIR\data" -username "$webuiUsername" -password "$webuiPassword" -apikey "$apiKey"'
|
; 「界面显示一份、config.db 里另一份」导致登录不上。
|
||||||
|
DetailPrint "检测 WSL 并在其中安装 HomeAgent..."
|
||||||
|
nsExec::ExecToStack 'powershell -NoProfile -ExecutionPolicy Bypass -File "$INSTDIR\install-via-wsl.ps1" -PayloadDir "$PLUGINSDIR\linux-payload" -ApiKey "$apiKey" -WebUIUser "$webuiUsername" -WebUIPass "$webuiPassword"'
|
||||||
Pop $0
|
Pop $0
|
||||||
|
Pop $1
|
||||||
${If} $0 != 0
|
${If} $0 != 0
|
||||||
DetailPrint "警告: 数据库初始化可能未成功完成"
|
; 退出码含义见 install-via-wsl.ps1:20/21 是「WSL 或发行版缺失,需要先装」,
|
||||||
|
; 属于可指引的用户动作,不当成安装失败来恐吓人。
|
||||||
|
${If} $0 == 20
|
||||||
|
MessageBox MB_ICONINFORMATION|MB_OK "未检测到 WSL。$\r$\n$\r$\n请在管理员 PowerShell 中执行:$\r$\n wsl --install$\r$\n$\r$\n然后重启 Windows,再重新运行本安装程序。"
|
||||||
|
${ElseIf} $0 == 21
|
||||||
|
MessageBox MB_ICONINFORMATION|MB_OK "WSL 已安装,但还没有发行版。$\r$\n$\r$\n请先执行:$\r$\n wsl --install -d Ubuntu$\r$\n$\r$\n完成首次初始化后再重新运行本安装程序。"
|
||||||
|
${Else}
|
||||||
|
MessageBox MB_ICONEXCLAMATION|MB_OK "WSL 内安装失败(退出码 $0)。$\r$\n$\r$\n可进入 WSL 手动排查:wsl -d Ubuntu$\r$\n安装脚本输出见上方日志。"
|
||||||
|
${EndIf}
|
||||||
|
${Else}
|
||||||
|
DetailPrint "HomeAgent 已在 WSL2 内安装完成"
|
||||||
${EndIf}
|
${EndIf}
|
||||||
!endif
|
!endif
|
||||||
|
|
||||||
|
|||||||
@ -3,7 +3,7 @@ Version: VERSION_PLACEHOLDER
|
|||||||
Architecture: ARCH_PLACEHOLDER
|
Architecture: ARCH_PLACEHOLDER
|
||||||
Maintainer: HomeAgent Team <team@homeagent.ai>
|
Maintainer: HomeAgent Team <team@homeagent.ai>
|
||||||
Installed-Size: INSTALLED_SIZE_PLACEHOLDER
|
Installed-Size: INSTALLED_SIZE_PLACEHOLDER
|
||||||
Depends: libc6 (>= 2.28)
|
Depends: libc6 (>= 2.28), libstdc++6, libgcc-s1
|
||||||
Section: utils
|
Section: utils
|
||||||
Priority: optional
|
Priority: optional
|
||||||
Homepage: https://github.com/trueagent/HomeAgent
|
Homepage: https://github.com/trueagent/HomeAgent
|
||||||
|
|||||||
@ -3,7 +3,7 @@ Version: VERSION_PLACEHOLDER
|
|||||||
Architecture: ARCH_PLACEHOLDER
|
Architecture: ARCH_PLACEHOLDER
|
||||||
Maintainer: HomeAgent Team <team@homeagent.ai>
|
Maintainer: HomeAgent Team <team@homeagent.ai>
|
||||||
Installed-Size: INSTALLED_SIZE_PLACEHOLDER
|
Installed-Size: INSTALLED_SIZE_PLACEHOLDER
|
||||||
Depends: libc6 (>= 2.28)
|
Depends: libc6 (>= 2.28), libstdc++6, libgcc-s1
|
||||||
Section: utils
|
Section: utils
|
||||||
Priority: optional
|
Priority: optional
|
||||||
Homepage: https://github.com/trueagent/HomeAgent
|
Homepage: https://github.com/trueagent/HomeAgent
|
||||||
|
|||||||
@ -2,24 +2,63 @@
|
|||||||
set -e
|
set -e
|
||||||
|
|
||||||
SERVICE_NAME="homeagent"
|
SERVICE_NAME="homeagent"
|
||||||
SERVICE_FILE="/lib/systemd/system/${SERVICE_NAME}.service"
|
|
||||||
HOMED_BIN="/usr/bin/homed"
|
HOMED_BIN="/usr/bin/homed"
|
||||||
DATA_DIR="/var/lib/homeagent"
|
DATA_DIR="/var/lib/homeagent"
|
||||||
|
SETUP_SH="/usr/lib/homeagent/setup.sh"
|
||||||
|
|
||||||
|
# unit 由本包装到 /etc/systemd/system/,而旧 postinst 只查
|
||||||
|
# /lib/systemd/system/(merged-usr 下等于 /usr/lib/systemd/system,那里没有
|
||||||
|
# 这个文件)——于是 daemon-reload 与 enable **从未执行过**:装完不会开机自启,
|
||||||
|
# 而 postinst 全程无报错。这里三个候选位置都看一下。
|
||||||
|
find_unit() {
|
||||||
|
for p in "/etc/systemd/system/${SERVICE_NAME}.service" \
|
||||||
|
"/usr/lib/systemd/system/${SERVICE_NAME}.service" \
|
||||||
|
"/lib/systemd/system/${SERVICE_NAME}.service"; do
|
||||||
|
if [ -f "$p" ]; then
|
||||||
|
printf '%s' "$p"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
case "$1" in
|
case "$1" in
|
||||||
configure)
|
configure)
|
||||||
if [ -f "$HOMED_BIN" ]; then
|
if [ -f "$HOMED_BIN" ]; then
|
||||||
mkdir -p "$DATA_DIR"
|
mkdir -p "$DATA_DIR"
|
||||||
|
|
||||||
# 初始化凭据和数据库
|
# 初始化凭据和数据库。
|
||||||
if [ -x /usr/lib/homeagent/setup.sh ]; then
|
#
|
||||||
HOMEAGENT_DATA="$DATA_DIR" /usr/lib/homeagent/setup.sh || true
|
# 这里不能再用 `|| true` 吞失败:setup.sh 靠 initconfig 写 config.db,
|
||||||
|
# 而 initconfig 曾因 CGO_ENABLED=0 静默空操作(凭据只进了
|
||||||
|
# credentials.txt、没进数据库),用户拿它登录必然失败,安装却一声不响。
|
||||||
|
# 失败必须看得见,并给出可直接执行的补救命令。
|
||||||
|
if [ -x "$SETUP_SH" ]; then
|
||||||
|
if ! HOMEAGENT_DATA="$DATA_DIR" "$SETUP_SH"; then
|
||||||
|
echo "E: homeagent 初始化失败——凭据可能未写入 config.db。" >&2
|
||||||
|
echo "E: 请手动重试:HOMEAGENT_DATA=$DATA_DIR $SETUP_SH" >&2
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
echo "W: 未找到 $SETUP_SH,跳过凭据初始化。" >&2
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# 注册 systemd 服务
|
# 注册 systemd 服务
|
||||||
if [ -f "$SERVICE_FILE" ]; then
|
if command -v systemctl >/dev/null 2>&1; then
|
||||||
systemctl daemon-reload 2>/dev/null || true
|
if find_unit >/dev/null; then
|
||||||
systemctl enable "$SERVICE_NAME" 2>/dev/null || true
|
systemctl daemon-reload 2>/dev/null || true
|
||||||
|
systemctl enable "$SERVICE_NAME" 2>/dev/null || true
|
||||||
|
# 首次安装就拉起来,装完即可用;升级时重启以真正加载新二进制
|
||||||
|
# (仅 enable 不会让已在运行的进程换用新文件)。
|
||||||
|
if [ -z "${2:-}" ]; then
|
||||||
|
systemctl start "$SERVICE_NAME" 2>/dev/null || \
|
||||||
|
echo "W: homeagent 服务未能启动,请检查:systemctl status $SERVICE_NAME" >&2
|
||||||
|
else
|
||||||
|
systemctl restart "$SERVICE_NAME" 2>/dev/null || \
|
||||||
|
echo "W: homeagent 服务未能重启,请检查:systemctl status $SERVICE_NAME" >&2
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
echo "W: 未找到 ${SERVICE_NAME}.service,未启用服务。" >&2
|
||||||
|
fi
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
;;
|
;;
|
||||||
|
|||||||
@ -8,16 +8,32 @@ CRED_FILE="${DATA_DIR}/credentials.txt"
|
|||||||
CONFIG_DB="${DATA_DIR}/config.db"
|
CONFIG_DB="${DATA_DIR}/config.db"
|
||||||
WAITER_CONF="${DATA_DIR}/waiter.yaml"
|
WAITER_CONF="${DATA_DIR}/waiter.yaml"
|
||||||
INITCONFIG_BIN="/usr/bin/initconfig"
|
INITCONFIG_BIN="/usr/bin/initconfig"
|
||||||
|
BUNDLED_MODEL_DIR="/usr/lib/homeagent/models/chinese-clip-vit-b16-onnx"
|
||||||
|
MODEL_LINK="${DATA_DIR}/models/chinese-clip-vit-b16-onnx"
|
||||||
|
|
||||||
# 如果已经初始化过,跳过
|
# 模型随 server/full 包安装到只读的 /usr/lib;配置默认仍指向 dataDir/models。
|
||||||
|
# 用符号链接把两者接起来,既不复制 754MB,也保持 dataDir 可迁移语义。
|
||||||
|
# 用户已有自定义目录时绝不覆盖;升级时既有链接自然指向新版包内容。
|
||||||
|
if [ -d "$BUNDLED_MODEL_DIR" ]; then
|
||||||
|
mkdir -p "${DATA_DIR}/models"
|
||||||
|
if [ ! -e "$MODEL_LINK" ] && [ ! -L "$MODEL_LINK" ]; then
|
||||||
|
ln -s "$BUNDLED_MODEL_DIR" "$MODEL_LINK"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
# 如果已经初始化过,只跳过凭据/数据库生成;上面的模型链接仍须在升级时补齐。
|
||||||
if [ -f "$CONFIG_DB" ] && [ -f "$CRED_FILE" ]; then
|
if [ -f "$CONFIG_DB" ] && [ -f "$CRED_FILE" ]; then
|
||||||
exit 0
|
exit 0
|
||||||
fi
|
fi
|
||||||
|
|
||||||
mkdir -p "$DATA_DIR"
|
mkdir -p "$DATA_DIR"
|
||||||
|
|
||||||
# 生成随机凭据
|
# 生成随机凭据。
|
||||||
API_KEY=$(cat /proc/sys/kernel/random/uuid 2>/dev/null | tr -d '-' || echo "homeagent$(date +%s)")
|
#
|
||||||
|
# 允许环境变量覆盖:安装器(包括 Windows 上的 WSL 引导安装)已经在界面上
|
||||||
|
# 向用户收过这些值,若不接受传入就只能两个地方各生成一份,用户看到的那份
|
||||||
|
# 与实际写入 config.db 的那份不一致——那种错会直接表现为「登录不上」。
|
||||||
|
API_KEY="${HOMEAGENT_API_KEY:-$(cat /proc/sys/kernel/random/uuid 2>/dev/null | tr -d '-' || echo "homeagent$(date +%s)")}"
|
||||||
WEBUI_USER="${WEBUI_USER:-admin}"
|
WEBUI_USER="${WEBUI_USER:-admin}"
|
||||||
WEBUI_PASS="${WEBUI_PASS:-$(openssl rand -hex 12 2>/dev/null || echo "homeagent")}"
|
WEBUI_PASS="${WEBUI_PASS:-$(openssl rand -hex 12 2>/dev/null || echo "homeagent")}"
|
||||||
|
|
||||||
|
|||||||
@ -5,23 +5,56 @@ PROJECT_ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
|
|||||||
BUILD_DIR="${PROJECT_ROOT}/build"
|
BUILD_DIR="${PROJECT_ROOT}/build"
|
||||||
DIST_DIR="${PROJECT_ROOT}/dist/linux"
|
DIST_DIR="${PROJECT_ROOT}/dist/linux"
|
||||||
VERSION="${VERSION:-$(git -C "$PROJECT_ROOT" describe --tags --dirty 2>/dev/null || echo "0.8.0")}"
|
VERSION="${VERSION:-$(git -C "$PROJECT_ROOT" describe --tags --dirty 2>/dev/null || echo "0.8.0")}"
|
||||||
|
|
||||||
|
# git describe 给出的是 v1.0.0-68-gba0b5a1-dirty 这类描述串,它不是合法的包版本:
|
||||||
|
# deb 的 Version 必须以数字开头,rpm 的 Version 不允许 '-'(那是版本/发布的分隔符)。
|
||||||
|
# 以前只有显式传 VERSION=1.0.3 才打得出来,默认路径一跑就死在 dpkg-deb 上——
|
||||||
|
# 而且死在 stage 之后,前面每条日志都是真的,只有最后一个产物没生成。
|
||||||
|
PKG_VERSION="${VERSION#v}"
|
||||||
|
case "$PKG_VERSION" in
|
||||||
|
[0-9]*) ;;
|
||||||
|
*) echo "ERROR: 包版本必须以数字开头(得到 '$VERSION')。请显式设置 VERSION=x.y.z 后重试。" >&2; exit 1 ;;
|
||||||
|
esac
|
||||||
|
PKG_VERSION="$(printf '%s' "$PKG_VERSION" | sed -e 's/-/+/g')"
|
||||||
PACKAGE_ROOT="${PROJECT_ROOT}/deploy/packaging/linux"
|
PACKAGE_ROOT="${PROJECT_ROOT}/deploy/packaging/linux"
|
||||||
GO="${GO:-$(command -v go 2>/dev/null || echo "go")}"
|
GO="${GO:-$(command -v go 2>/dev/null || echo "go")}"
|
||||||
|
|
||||||
ARCH="${1:-amd64}" # amd64 or arm64
|
ARCH="${1:-amd64}" # amd64 or arm64
|
||||||
|
|
||||||
|
# server/full 发行包默认带 Chinese-CLIP ONNX 产物与 ONNX Runtime。
|
||||||
|
# 二进制大资产不进 git:发布环境通过这两个目录提供已验证的产物;若缺失,
|
||||||
|
# server/full 打包必须明确失败,不能生成一个「默认启用但装完不能用」的假包。
|
||||||
|
CHINESECLIP_BUNDLE_DIR="${CHINESECLIP_BUNDLE_DIR:-$BUILD_DIR/model-assets/chinese-clip-vit-b16-onnx}"
|
||||||
|
ONNXRUNTIME_ASSET_DIR="${ONNXRUNTIME_ASSET_DIR:-$BUILD_DIR/runtime-assets/$ARCH}"
|
||||||
|
ONNXRUNTIME_LIB="${ONNXRUNTIME_LIB:-$ONNXRUNTIME_ASSET_DIR/libonnxruntime.so}"
|
||||||
|
ONNXRUNTIME_LICENSE="${ONNXRUNTIME_LICENSE:-$ONNXRUNTIME_ASSET_DIR/LICENSE}"
|
||||||
|
ONNXRUNTIME_NOTICES="${ONNXRUNTIME_NOTICES:-$ONNXRUNTIME_ASSET_DIR/ThirdPartyNotices.txt}"
|
||||||
|
|
||||||
|
# 打包 staging 会把 719MB 模型真的复制一份,临时目录必须落在构建目录所在的磁盘,
|
||||||
|
# 不能落在系统临时目录:本机 /tmp 是 9.8GB tmpfs,一次 full 包 staging 就能写满,
|
||||||
|
# 而且失败发生在 cp 进行到一半,报出来是 "No space left on device"——看上去像
|
||||||
|
# 资产/版本有问题,实际只是临时目录选错了文件系统。
|
||||||
|
STAGE_TMP="${BUILD_DIR}/.stage-tmp"
|
||||||
|
|
||||||
|
# electron 官方发布物用 x64/arm64 命名,而 Debian 用 amd64/arm64。
|
||||||
|
# 两者在 arm64 上恰好同名,amd64 上不同——此前缓存查找统一用 TAR_ARCH
|
||||||
|
# (amd64),于是 electron-v*-linux-x64.zip 永远命中不到,amd64 GUI 只能
|
||||||
|
# 靠"回退到 host node_modules"这条路组装。干净 worktree 里没有完整
|
||||||
|
# node_modules,GUI 就被静默跳过。故单独映射。
|
||||||
ACTION="${2:-all}" # all, build, deb, tar, rpm
|
ACTION="${2:-all}" # all, build, deb, tar, rpm
|
||||||
|
|
||||||
DEB_ARCH="$ARCH"
|
DEB_ARCH="$ARCH"
|
||||||
RPM_ARCH="$ARCH"
|
RPM_ARCH="$ARCH"
|
||||||
TAR_ARCH="$ARCH"
|
TAR_ARCH="$ARCH"
|
||||||
case "$ARCH" in
|
case "$ARCH" in
|
||||||
amd64) DEB_ARCH="amd64"; RPM_ARCH="x86_64"; TAR_ARCH="amd64" ;;
|
amd64) DEB_ARCH="amd64"; RPM_ARCH="x86_64"; TAR_ARCH="amd64"; ELECTRON_ARCH="x64" ;;
|
||||||
arm64) DEB_ARCH="arm64"; RPM_ARCH="aarch64"; TAR_ARCH="arm64" ;;
|
arm64) DEB_ARCH="arm64"; RPM_ARCH="aarch64"; TAR_ARCH="arm64"; ELECTRON_ARCH="arm64" ;;
|
||||||
*) echo "Unknown arch: $ARCH (use amd64 or arm64)"; exit 1 ;;
|
*) echo "Unknown arch: $ARCH (use amd64 or arm64)"; exit 1 ;;
|
||||||
esac
|
esac
|
||||||
|
|
||||||
echo "=== HomeAgent Linux Packager ==="
|
echo "=== HomeAgent Linux Packager ==="
|
||||||
echo "Version: $VERSION"
|
echo "Version: $VERSION"
|
||||||
|
[ "$PKG_VERSION" = "$VERSION" ] || echo "Package: $PKG_VERSION (normalized for deb/rpm)"
|
||||||
echo "Arch: $ARCH"
|
echo "Arch: $ARCH"
|
||||||
echo ""
|
echo ""
|
||||||
|
|
||||||
@ -83,7 +116,7 @@ restore_syso() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
# ensure both are always restored on exit
|
# ensure both are always restored on exit
|
||||||
restore_all() { restore_gomod; restore_syso; }
|
restore_all() { restore_gomod; restore_syso; rmdir "$STAGE_TMP" 2>/dev/null || true; }
|
||||||
trap restore_all EXIT
|
trap restore_all EXIT
|
||||||
|
|
||||||
# ---- build Go binaries via existing build.sh ----
|
# ---- build Go binaries via existing build.sh ----
|
||||||
@ -93,35 +126,42 @@ build_go() {
|
|||||||
prepare_gomod || true
|
prepare_gomod || true
|
||||||
hide_syso
|
hide_syso
|
||||||
|
|
||||||
bash "$PROJECT_ROOT/deploy/packaging/build.sh" "linux/$ARCH" "homed" 2>&1 || {
|
|
||||||
echo "WARNING: homed build failed (CGO/sqlite3 issue). Server/full packages may be incomplete."
|
|
||||||
}
|
|
||||||
bash "$PROJECT_ROOT/deploy/packaging/build.sh" "linux/$ARCH" "waiter" 2>&1 || {
|
|
||||||
echo "WARNING: waiter build failed."
|
|
||||||
}
|
|
||||||
bash "$PROJECT_ROOT/deploy/packaging/build.sh" "linux/$ARCH" "initconfig" 2>&1 || {
|
|
||||||
echo "WARNING: initconfig build failed(包内将缺少首次配置初始化器)。"
|
|
||||||
}
|
|
||||||
|
|
||||||
local suffix="linux_${ARCH}"
|
local suffix="linux_${ARCH}"
|
||||||
local homed_bin="$BUILD_DIR/homed_$suffix"
|
local homed_bin="$BUILD_DIR/homed_$suffix"
|
||||||
local waiter_bin="$BUILD_DIR/waiter_$suffix"
|
local waiter_bin="$BUILD_DIR/waiter_$suffix"
|
||||||
|
local initconfig_bin="$BUILD_DIR/initconfig_$suffix"
|
||||||
|
|
||||||
if [ ! -f "$homed_bin" ]; then
|
# 先删旧产物:否则本次构建失败后,残留文件会让「产物存在」判据假绿。
|
||||||
echo "ERROR: homed binary not found at $homed_bin"
|
rm -f "$homed_bin" "$waiter_bin" "$initconfig_bin"
|
||||||
exit 1
|
|
||||||
fi
|
bash "$PROJECT_ROOT/deploy/packaging/build.sh" "linux/$ARCH" "homed"
|
||||||
if [ ! -f "$waiter_bin" ]; then
|
test -x "$homed_bin"
|
||||||
echo "ERROR: waiter binary not found at $waiter_bin"
|
if ! go version -m "$homed_bin" | grep -Eq 'build[[:space:]]+-tags=.*onnxruntime'; then
|
||||||
exit 1
|
echo "ERROR: homed 不是 onnxruntime 构建,拒绝打 server/full 包:$homed_bin" >&2
|
||||||
|
return 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
echo " homed: $homed_bin ($(du -h "$homed_bin" | cut -f1))"
|
bash "$PROJECT_ROOT/deploy/packaging/build.sh" "linux/$ARCH" "waiter"
|
||||||
echo " waiter: $waiter_bin ($(du -h "$waiter_bin" | cut -f1))"
|
test -x "$waiter_bin"
|
||||||
|
bash "$PROJECT_ROOT/deploy/packaging/build.sh" "linux/$ARCH" "initconfig"
|
||||||
|
test -x "$initconfig_bin"
|
||||||
|
|
||||||
|
echo " homed: $homed_bin ($(du -h "$homed_bin" | cut -f1), onnxruntime)"
|
||||||
|
echo " waiter: $waiter_bin ($(du -h "$waiter_bin" | cut -f1))"
|
||||||
|
echo " initconfig: $initconfig_bin ($(du -h "$initconfig_bin" | cut -f1))"
|
||||||
echo ""
|
echo ""
|
||||||
}
|
}
|
||||||
|
|
||||||
# ---- build GUI (manual directory assembly, avoids electron-packager network issues) ----
|
# ---- build GUI (manual directory assembly, avoids electron-packager network issues) ----
|
||||||
|
#
|
||||||
|
# electron 运行时必须按**目标架构**取,不能用 host 的
|
||||||
|
# node_modules/electron/dist——那里永远是 host 架构(本机 x64)。
|
||||||
|
# v1.0.0 / v1.0.1 的 arm64 full/client 包都踩了这个坑:目录名带
|
||||||
|
# -arm64、homed/waiter 确实是 aarch64,但里面的 electron 是 x86-64,
|
||||||
|
# 在 arm64 机器上一启动就是 Exec format error(从未被交叉验证过)。
|
||||||
|
#
|
||||||
|
# 现在改为优先从 electron 缓存里取对应架构的 zip,并在最后做
|
||||||
|
# 一道强制校验:架构不符就删掉目录并跳过 GUI,宁可不发也不发坏包。
|
||||||
build_gui() {
|
build_gui() {
|
||||||
local gui_dir="$PROJECT_ROOT/cmd/gui"
|
local gui_dir="$PROJECT_ROOT/cmd/gui"
|
||||||
local gui_out="$BUILD_DIR/homeagent-gui-linux-${TAR_ARCH}"
|
local gui_out="$BUILD_DIR/homeagent-gui-linux-${TAR_ARCH}"
|
||||||
@ -133,22 +173,81 @@ build_gui() {
|
|||||||
|
|
||||||
echo ">>> Building GUI directory for linux/$ARCH..."
|
echo ">>> Building GUI directory for linux/$ARCH..."
|
||||||
|
|
||||||
if [ ! -d "$gui_dir/node_modules" ]; then
|
# 判据是 electron 包本身在不在,而不是 node_modules 目录在不在。
|
||||||
|
#
|
||||||
|
# npm install 失败(离线、网络受限)会留下一个只有一两个条目的空壳
|
||||||
|
# node_modules,目录存在但 electron 缺失。只看目录会以为"已安装",
|
||||||
|
# 于是 ever 读不到版本、缓存匹配退化、最后走到"host dist 也没有"而
|
||||||
|
# 静默跳过 GUI——包名和目录名全都正确,只是没有 GUI,没有任何一步报错。
|
||||||
|
if [ ! -f "$gui_dir/node_modules/electron/package.json" ]; then
|
||||||
|
if [ -d "$gui_dir/node_modules" ]; then
|
||||||
|
echo " node_modules 存在但 electron 缺失(疑似上次 npm install 未完成)"
|
||||||
|
fi
|
||||||
echo " npm install..."
|
echo " npm install..."
|
||||||
(cd "$gui_dir" && npm install --production)
|
if ! (cd "$gui_dir" && npm install --production); then
|
||||||
|
echo " WARNING: npm install 失败——离线环境下这是预期的。"
|
||||||
|
echo " GUI 需要 cmd/gui/node_modules/electron 或 ~/.cache/electron 缓存。"
|
||||||
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
local electron_dir="$gui_dir/node_modules/electron/dist"
|
# electron 版本优先从已安装的包里读,保证运行时与 app 依赖一致。
|
||||||
if [ ! -f "$electron_dir/electron" ]; then
|
# 读不到时退而从 package.json 的依赖声明里取数字部分(它可能写成
|
||||||
echo " WARNING: electron binary not found at $electron_dir. GUI will be skipped."
|
# "^33.0.0" 这类范围,只用于给缓存匹配一个提示,匹配不上仍会走通配)。
|
||||||
return
|
local ever
|
||||||
|
ever=$(python3 -c "import json;print(json.load(open('$gui_dir/node_modules/electron/package.json'))['version'])" 2>/dev/null || true)
|
||||||
|
if [ -z "$ever" ]; then
|
||||||
|
ever=$(python3 -c "
|
||||||
|
import json, re
|
||||||
|
d = json.load(open('$gui_dir/package.json'))
|
||||||
|
spec = (d.get('devDependencies', {}) or {}).get('electron') or (d.get('dependencies', {}) or {}).get('electron') or ''
|
||||||
|
m = re.search(r'(\\d+(?:\\.\\d+)*)', spec)
|
||||||
|
print(m.group(1) if m else '')
|
||||||
|
" 2>/dev/null || true)
|
||||||
|
[ -n "$ever" ] && echo " electron 版本取自 package.json 依赖声明: $ever(非精确)"
|
||||||
|
fi
|
||||||
|
|
||||||
|
mkdir -p "$gui_out"
|
||||||
|
|
||||||
|
# 优先:缓存里的目标架构 zip(~/.cache/electron/<hash>/electron-v<ver>-linux-<arch>.zip)
|
||||||
|
local zip=""
|
||||||
|
if [ -n "$ever" ]; then
|
||||||
|
zip=$(find "$HOME/.cache/electron" -name "electron-v${ever}-linux-${ELECTRON_ARCH}.zip" 2>/dev/null | head -1)
|
||||||
|
fi
|
||||||
|
if [ -z "$zip" ]; then
|
||||||
|
zip=$(find "$HOME/.cache/electron" -name "electron-v*-linux-${ELECTRON_ARCH}.zip" 2>/dev/null | head -1)
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ -n "$zip" ]; then
|
||||||
|
echo " electron runtime: $(basename "$zip")"
|
||||||
|
unzip -q -o "$zip" -d "$gui_out"
|
||||||
|
else
|
||||||
|
# 回退:仅当目标架构 == host 架构时才能用 host 的 dist
|
||||||
|
local host_arch
|
||||||
|
case "$(uname -m)" in
|
||||||
|
x86_64) host_arch=amd64 ;;
|
||||||
|
aarch64|arm64) host_arch=arm64 ;;
|
||||||
|
*) host_arch=unknown ;;
|
||||||
|
esac
|
||||||
|
if [ "$TAR_ARCH" != "$host_arch" ]; then
|
||||||
|
echo " WARNING: 缺 electron-v*-linux-${ELECTRON_ARCH}.zip 缓存,且目标架构与 host"
|
||||||
|
echo " ($host_arch) 不同——不能用 host 的 electron 冒充。跳过 GUI。"
|
||||||
|
echo " 解法:下载 electron-v${ever:-<ver>}-linux-${ELECTRON_ARCH}.zip 到"
|
||||||
|
echo " ~/.cache/electron/<任意子目录>/ 后重跑。"
|
||||||
|
rm -rf "$gui_out"
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
local electron_dir="$gui_dir/node_modules/electron/dist"
|
||||||
|
if [ ! -f "$electron_dir/electron" ]; then
|
||||||
|
echo " WARNING: electron binary not found at $electron_dir. GUI will be skipped."
|
||||||
|
rm -rf "$gui_out"
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
echo " electron runtime: host node_modules (同架构 $host_arch)"
|
||||||
|
cp -r "$electron_dir"/* "$gui_out/" 2>/dev/null
|
||||||
fi
|
fi
|
||||||
|
|
||||||
mkdir -p "$gui_out/resources/app/node_modules"
|
mkdir -p "$gui_out/resources/app/node_modules"
|
||||||
mkdir -p "$gui_out/resources/app/renderer"
|
mkdir -p "$gui_out/resources/app/renderer"
|
||||||
|
|
||||||
# copy electron runtime (binary + shared libs)
|
|
||||||
cp -r "$electron_dir"/* "$gui_out/" 2>/dev/null
|
|
||||||
rm -f "$gui_out/resources/default_app.asar" 2>/dev/null
|
rm -f "$gui_out/resources/default_app.asar" 2>/dev/null
|
||||||
|
|
||||||
# copy app source
|
# copy app source
|
||||||
@ -190,7 +289,28 @@ LAUNCHER
|
|||||||
chmod +x "$gui_out/homeagent-gui"
|
chmod +x "$gui_out/homeagent-gui"
|
||||||
chmod +x "$gui_out/electron"
|
chmod +x "$gui_out/electron"
|
||||||
|
|
||||||
echo " GUI built: $gui_out ($(du -sh "$gui_out" | cut -f1))"
|
# 最后一道强制校验:electron 二进制的实际架构必须匹配目标架构。
|
||||||
|
# 不做这步就会重现 v1.0.0/v1.0.1 的隐形坏包:包名、目录名、
|
||||||
|
# homed/waiter 全对,只有 electron 是错架构,直到用户在 arm64 机器上
|
||||||
|
# 双击才发现 Exec format error。
|
||||||
|
local want_pat
|
||||||
|
case "$TAR_ARCH" in
|
||||||
|
amd64) want_pat="x86-64" ;;
|
||||||
|
arm64) want_pat="aarch64" ;;
|
||||||
|
*) want_pat="" ;;
|
||||||
|
esac
|
||||||
|
if [ -n "$want_pat" ]; then
|
||||||
|
local got
|
||||||
|
got=$(file -b "$gui_out/electron" 2>/dev/null || echo "")
|
||||||
|
if ! printf '%s' "$got" | grep -q "$want_pat"; then
|
||||||
|
echo " ERROR: electron 架构不符——期望 $want_pat,实际: ${got%%,*}"
|
||||||
|
echo " 删除 GUI 目录并跳过(宁可不发,也不发装了跑不起来的包)。"
|
||||||
|
rm -rf "$gui_out"
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo " GUI built: $gui_out ($(du -sh "$gui_out" | cut -f1), $(file -b "$gui_out/electron" | cut -d, -f2 | tr -d ' '))"
|
||||||
echo ""
|
echo ""
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -215,6 +335,8 @@ stage_variant() {
|
|||||||
cp "$PROJECT_ROOT/deploy/homeagent.service" "$staging/etc/systemd/system/homeagent.service"
|
cp "$PROJECT_ROOT/deploy/homeagent.service" "$staging/etc/systemd/system/homeagent.service"
|
||||||
[ -f "$initconfig_bin" ] && cp "$initconfig_bin" "$staging/usr/bin/initconfig"
|
[ -f "$initconfig_bin" ] && cp "$initconfig_bin" "$staging/usr/bin/initconfig"
|
||||||
stage_setup "$staging"
|
stage_setup "$staging"
|
||||||
|
stage_license "$staging"
|
||||||
|
stage_multimodal_assets "$staging"
|
||||||
stage_gui "$staging"
|
stage_gui "$staging"
|
||||||
;;
|
;;
|
||||||
server)
|
server)
|
||||||
@ -223,9 +345,12 @@ stage_variant() {
|
|||||||
cp "$PROJECT_ROOT/deploy/homeagent.service" "$staging/etc/systemd/system/homeagent.service"
|
cp "$PROJECT_ROOT/deploy/homeagent.service" "$staging/etc/systemd/system/homeagent.service"
|
||||||
[ -f "$initconfig_bin" ] && cp "$initconfig_bin" "$staging/usr/bin/initconfig"
|
[ -f "$initconfig_bin" ] && cp "$initconfig_bin" "$staging/usr/bin/initconfig"
|
||||||
stage_setup "$staging"
|
stage_setup "$staging"
|
||||||
|
stage_license "$staging"
|
||||||
|
stage_multimodal_assets "$staging"
|
||||||
;;
|
;;
|
||||||
client)
|
client)
|
||||||
cp "$BUILD_DIR/waiter_$suffix" "$staging/usr/bin/waiter"
|
cp "$BUILD_DIR/waiter_$suffix" "$staging/usr/bin/waiter"
|
||||||
|
stage_license "$staging"
|
||||||
stage_gui "$staging"
|
stage_gui "$staging"
|
||||||
;;
|
;;
|
||||||
esac
|
esac
|
||||||
@ -260,6 +385,127 @@ stage_setup() {
|
|||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# 项目自身的许可:**所有变体**都要带(client 也分发 waiter 与 GUI)。
|
||||||
|
#
|
||||||
|
# deb 按 Debian 惯例给 /usr/share/doc/homeagent/copyright(DEP-5 机器可读格式),
|
||||||
|
# 同时把 LICENSE 全文放进去;rpm 的许可走 fpm 的 --license 元数据。
|
||||||
|
# 与 stage_multimodal_assets 的 licenses/ 分工:那里放**第三方**(模型/运行库)的
|
||||||
|
# 许可全文,这里放本项目自己的。
|
||||||
|
stage_license() {
|
||||||
|
local staging="$1"
|
||||||
|
local docdir="$staging/usr/share/doc/homeagent"
|
||||||
|
mkdir -p "$docdir"
|
||||||
|
cp "$PROJECT_ROOT/LICENSE" "$docdir/LICENSE"
|
||||||
|
cat > "$docdir/copyright" <<'EOF'
|
||||||
|
Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
|
||||||
|
Upstream-Name: HomeAgent
|
||||||
|
Source: https://gitcode.com/JianFeeeee/HomeAgent
|
||||||
|
|
||||||
|
Files: *
|
||||||
|
Copyright: HomeAgent contributors
|
||||||
|
License: AGPL-3.0-only
|
||||||
|
This program is free software: you can redistribute it and/or modify it under
|
||||||
|
the terms of the GNU Affero General Public License as published by the Free
|
||||||
|
Software Foundation, version 3 of the License.
|
||||||
|
.
|
||||||
|
This program is distributed in the hope that it will be useful, but WITHOUT
|
||||||
|
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
|
||||||
|
FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
|
||||||
|
details.
|
||||||
|
.
|
||||||
|
You should have received a copy of the GNU Affero General Public License along
|
||||||
|
with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||||
|
.
|
||||||
|
The license is AGPL-3.0-only: no later version may be chosen. Note the network
|
||||||
|
clause (§13 Remote Network Interaction) — offering modified versions of this
|
||||||
|
software to users over a network also requires offering them the source.
|
||||||
|
.
|
||||||
|
Full text: /usr/share/doc/homeagent/LICENSE
|
||||||
|
|
||||||
|
Files: usr/lib/homeagent/models/chinese-clip-vit-b16-onnx/*
|
||||||
|
Copyright: OFA-Sys / Chinese-CLIP authors
|
||||||
|
License: Apache-2.0
|
||||||
|
Full text: /usr/share/doc/homeagent/licenses/Chinese-CLIP-Apache-2.0.txt
|
||||||
|
Comment: pre-trained model artifacts; NOT covered by this package's AGPL grant
|
||||||
|
|
||||||
|
Files: usr/lib/homeagent/onnxruntime/*
|
||||||
|
Copyright: Microsoft Corporation
|
||||||
|
License: MIT
|
||||||
|
Full text: /usr/share/doc/homeagent/licenses/ONNX-Runtime-MIT.txt
|
||||||
|
Comment: license texts and third-party notices under licenses/ONNX-Runtime-*
|
||||||
|
EOF
|
||||||
|
chmod 644 "$docdir/LICENSE" "$docdir/copyright"
|
||||||
|
}
|
||||||
|
|
||||||
|
# server/full 的 ONNX 资产。模型与运行库是发行版能力的一部分,不是可选下载:
|
||||||
|
# 只要打 server/full 包,两者缺一就失败。client 包不运行 homed,故不携带。
|
||||||
|
stage_multimodal_assets() {
|
||||||
|
local staging="$1"
|
||||||
|
local model_dst="$staging/usr/lib/homeagent/models/chinese-clip-vit-b16-onnx"
|
||||||
|
local ort_dst="$staging/usr/lib/homeagent/onnxruntime"
|
||||||
|
local licenses="$staging/usr/share/doc/homeagent/licenses"
|
||||||
|
|
||||||
|
if [ ! -d "$CHINESECLIP_BUNDLE_DIR" ]; then
|
||||||
|
echo "ERROR: Chinese-CLIP 产物目录不存在:$CHINESECLIP_BUNDLE_DIR" >&2
|
||||||
|
echo "先运行 scripts/export_chineseclip_onnx.py,再通过 CHINESECLIP_BUNDLE_DIR 指向产物。" >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
for f in TextEncoder.onnx VisionEncoder.onnx embed_config.json vocab.txt reference.json SHA256SUMS; do
|
||||||
|
if [ ! -s "$CHINESECLIP_BUNDLE_DIR/$f" ]; then
|
||||||
|
echo "ERROR: Chinese-CLIP 产物缺少或为空:$CHINESECLIP_BUNDLE_DIR/$f" >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
if ! (cd "$CHINESECLIP_BUNDLE_DIR" && sha256sum -c SHA256SUMS); then
|
||||||
|
echo "ERROR: Chinese-CLIP SHA256SUMS 校验失败,拒绝打包。" >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ ! -s "$ONNXRUNTIME_LIB" ]; then
|
||||||
|
echo "ERROR: ONNX Runtime 不存在:$ONNXRUNTIME_LIB" >&2
|
||||||
|
echo "通过 ONNXRUNTIME_ASSET_DIR 或 ONNXRUNTIME_LIB 指向与目标架构匹配的资产。" >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
for notice in "$ONNXRUNTIME_LICENSE" "$ONNXRUNTIME_NOTICES"; do
|
||||||
|
if [ ! -s "$notice" ]; then
|
||||||
|
echo "ERROR: ONNX Runtime 许可证资产缺失:$notice" >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
local runtime_desc
|
||||||
|
runtime_desc=$(file -b "$ONNXRUNTIME_LIB")
|
||||||
|
case "$ARCH" in
|
||||||
|
amd64) printf '%s' "$runtime_desc" | grep -qE 'x86-64|x86_64' || {
|
||||||
|
echo "ERROR: ONNX Runtime 架构不是 amd64:$runtime_desc" >&2; return 1; } ;;
|
||||||
|
arm64) printf '%s' "$runtime_desc" | grep -qE 'aarch64|ARM aarch64' || {
|
||||||
|
echo "ERROR: ONNX Runtime 架构不是 arm64:$runtime_desc" >&2; return 1; } ;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
mkdir -p "$model_dst" "$ort_dst" "$licenses"
|
||||||
|
cp -a "$CHINESECLIP_BUNDLE_DIR/." "$model_dst/"
|
||||||
|
install -m 0755 "$ONNXRUNTIME_LIB" "$ort_dst/libonnxruntime.so"
|
||||||
|
|
||||||
|
# 许可证随二进制分发:Chinese-CLIP = Apache-2.0;ONNX Runtime = MIT,
|
||||||
|
# 同时携带其 ThirdPartyNotices(含 MKL/protobuf/zlib 等第三方条款)。
|
||||||
|
cp /usr/share/common-licenses/Apache-2.0 "$licenses/Chinese-CLIP-Apache-2.0.txt"
|
||||||
|
cp "$ONNXRUNTIME_LICENSE" "$licenses/ONNX-Runtime-MIT.txt"
|
||||||
|
cp "$ONNXRUNTIME_NOTICES" "$licenses/ONNX-Runtime-ThirdPartyNotices.txt"
|
||||||
|
cat > "$licenses/MODEL-SOURCES.txt" <<EOF
|
||||||
|
Chinese-CLIP ViT-B/16
|
||||||
|
upstream: https://huggingface.co/OFA-Sys/chinese-clip-vit-base-patch16
|
||||||
|
license: Apache-2.0
|
||||||
|
exported-by: scripts/export_chineseclip_onnx.py
|
||||||
|
dimensions: 512
|
||||||
|
modalities: text,image
|
||||||
|
|
||||||
|
ONNX Runtime
|
||||||
|
upstream: https://github.com/microsoft/onnxruntime
|
||||||
|
license: MIT (see ONNX-Runtime-MIT.txt and ONNX-Runtime-ThirdPartyNotices.txt)
|
||||||
|
EOF
|
||||||
|
|
||||||
|
echo " ONNX assets: model=$(du -sh "$model_dst" | cut -f1) runtime=$(du -h "$ort_dst/libonnxruntime.so" | cut -f1)"
|
||||||
|
}
|
||||||
|
|
||||||
# ---- create .deb ----
|
# ---- create .deb ----
|
||||||
build_deb() {
|
build_deb() {
|
||||||
local variant="$1"
|
local variant="$1"
|
||||||
@ -267,9 +513,9 @@ build_deb() {
|
|||||||
local deb_dir="${DIST_DIR}/deb"
|
local deb_dir="${DIST_DIR}/deb"
|
||||||
mkdir -p "$deb_dir"
|
mkdir -p "$deb_dir"
|
||||||
|
|
||||||
local pkg_name="homeagent-${variant}_${VERSION}_${DEB_ARCH}.deb"
|
local pkg_name="homeagent-${variant}_${PKG_VERSION}_${DEB_ARCH}.deb"
|
||||||
local deb_root
|
local deb_root
|
||||||
deb_root="$(mktemp -d)"
|
deb_root="$(mktemp -d "$STAGE_TMP/deb.XXXXXX")"
|
||||||
|
|
||||||
mkdir -p "$deb_root/DEBIAN"
|
mkdir -p "$deb_root/DEBIAN"
|
||||||
|
|
||||||
@ -277,7 +523,7 @@ build_deb() {
|
|||||||
local installed_size_kb
|
local installed_size_kb
|
||||||
installed_size_kb=$(du -sk "$staging" | cut -f1)
|
installed_size_kb=$(du -sk "$staging" | cut -f1)
|
||||||
|
|
||||||
sed -e "s/VERSION_PLACEHOLDER/$VERSION/g" \
|
sed -e "s/VERSION_PLACEHOLDER/$PKG_VERSION/g" \
|
||||||
-e "s/ARCH_PLACEHOLDER/$DEB_ARCH/g" \
|
-e "s/ARCH_PLACEHOLDER/$DEB_ARCH/g" \
|
||||||
-e "s/INSTALLED_SIZE_PLACEHOLDER/$installed_size_kb/g" \
|
-e "s/INSTALLED_SIZE_PLACEHOLDER/$installed_size_kb/g" \
|
||||||
"$control_file" > "$deb_root/DEBIAN/control"
|
"$control_file" > "$deb_root/DEBIAN/control"
|
||||||
@ -306,13 +552,13 @@ build_tar() {
|
|||||||
local tar_dir="${DIST_DIR}/tar"
|
local tar_dir="${DIST_DIR}/tar"
|
||||||
mkdir -p "$tar_dir"
|
mkdir -p "$tar_dir"
|
||||||
|
|
||||||
local archive_name="homeagent_${VERSION}_linux_${TAR_ARCH}.tar.gz"
|
local archive_name="homeagent_${PKG_VERSION}_linux_${TAR_ARCH}.tar.gz"
|
||||||
local archive_dir="homeagent-${VERSION}-linux-${TAR_ARCH}"
|
local archive_dir="homeagent-${PKG_VERSION}-linux-${TAR_ARCH}"
|
||||||
|
|
||||||
# build combined staging
|
# build combined staging
|
||||||
local staging
|
local staging
|
||||||
staging="$(mktemp -d)"
|
staging="$(mktemp -d "$STAGE_TMP/tar.XXXXXX")"
|
||||||
mkdir -p "$staging/usr/bin" "$staging/usr/lib/homeagent"
|
mkdir -p "$staging/usr/bin" "$staging/usr/lib/homeagent" "$staging/etc/systemd/system"
|
||||||
|
|
||||||
# copy all available binaries
|
# copy all available binaries
|
||||||
for bin in homed waiter initconfig; do
|
for bin in homed waiter initconfig; do
|
||||||
@ -323,6 +569,9 @@ build_tar() {
|
|||||||
# setup script
|
# setup script
|
||||||
local setup_src="$PROJECT_ROOT/deploy/packaging/linux/setup.sh"
|
local setup_src="$PROJECT_ROOT/deploy/packaging/linux/setup.sh"
|
||||||
[ -f "$setup_src" ] && cp "$setup_src" "$staging/usr/lib/homeagent/setup.sh"
|
[ -f "$setup_src" ] && cp "$setup_src" "$staging/usr/lib/homeagent/setup.sh"
|
||||||
|
cp "$PROJECT_ROOT/deploy/homeagent.service" "$staging/etc/systemd/system/homeagent.service"
|
||||||
|
stage_license "$staging"
|
||||||
|
stage_multimodal_assets "$staging"
|
||||||
|
|
||||||
# GUI if available
|
# GUI if available
|
||||||
local gui_src="$BUILD_DIR/homeagent-gui-linux-${TAR_ARCH}"
|
local gui_src="$BUILD_DIR/homeagent-gui-linux-${TAR_ARCH}"
|
||||||
@ -351,7 +600,7 @@ build_rpm() {
|
|||||||
local rpm_dir="${DIST_DIR}/rpm"
|
local rpm_dir="${DIST_DIR}/rpm"
|
||||||
mkdir -p "$rpm_dir"
|
mkdir -p "$rpm_dir"
|
||||||
|
|
||||||
local pkg_name="homeagent-${variant}-${VERSION}-1.${RPM_ARCH}.rpm"
|
local pkg_name="homeagent-${variant}-${PKG_VERSION}-1.${RPM_ARCH}.rpm"
|
||||||
|
|
||||||
# find fpm
|
# find fpm
|
||||||
local fpm_bin="$(command -v fpm 2>/dev/null || true)"
|
local fpm_bin="$(command -v fpm 2>/dev/null || true)"
|
||||||
@ -400,7 +649,7 @@ build_rpm() {
|
|||||||
-a "$RPM_ARCH" \
|
-a "$RPM_ARCH" \
|
||||||
--description "HomeAgent ${variant^} package" \
|
--description "HomeAgent ${variant^} package" \
|
||||||
--url "https://github.com/trueagent/HomeAgent" \
|
--url "https://github.com/trueagent/HomeAgent" \
|
||||||
--license "Proprietary" \
|
--license "AGPL-3.0-only" \
|
||||||
-C "$staging" \
|
-C "$staging" \
|
||||||
-p "$rpm_dir/$pkg_name" \
|
-p "$rpm_dir/$pkg_name" \
|
||||||
. 2>&1
|
. 2>&1
|
||||||
@ -411,7 +660,7 @@ build_rpm() {
|
|||||||
main() {
|
main() {
|
||||||
local target_arch="$ARCH"
|
local target_arch="$ARCH"
|
||||||
|
|
||||||
mkdir -p "$BUILD_DIR"
|
mkdir -p "$BUILD_DIR" "$STAGE_TMP"
|
||||||
|
|
||||||
case "$ACTION" in
|
case "$ACTION" in
|
||||||
all|build)
|
all|build)
|
||||||
@ -428,6 +677,10 @@ main() {
|
|||||||
|
|
||||||
mkdir -p "$DIST_DIR"
|
mkdir -p "$DIST_DIR"
|
||||||
|
|
||||||
|
# 上次成功构建留下的校验和必须在本次开工前删掉:本次若中途失败,脚本直接退出、
|
||||||
|
# 不重算 SHA256SUMS,旧的它会一直躺在 dist 里,看上去像在为这一批残缺产物背书。
|
||||||
|
rm -f "$DIST_DIR/SHA256SUMS"
|
||||||
|
|
||||||
for variant in full server client; do
|
for variant in full server client; do
|
||||||
echo ""
|
echo ""
|
||||||
echo "=============================================="
|
echo "=============================================="
|
||||||
@ -435,7 +688,7 @@ main() {
|
|||||||
echo "=============================================="
|
echo "=============================================="
|
||||||
|
|
||||||
local staging
|
local staging
|
||||||
staging=$(mktemp -d)
|
staging=$(mktemp -d "$STAGE_TMP/stage.XXXXXX")
|
||||||
stage_variant "$variant" "$staging"
|
stage_variant "$variant" "$staging"
|
||||||
|
|
||||||
case "$ACTION" in
|
case "$ACTION" in
|
||||||
@ -457,9 +710,19 @@ main() {
|
|||||||
echo "=== Done! Packages in: $DIST_DIR ==="
|
echo "=== Done! Packages in: $DIST_DIR ==="
|
||||||
echo ""
|
echo ""
|
||||||
echo "Summary:"
|
echo "Summary:"
|
||||||
find "$DIST_DIR" -type f \( -name "*.deb" -o -name "homeagent_*.tar.gz" -o -name "*.rpm" \) 2>/dev/null | sort | while read -r f; do
|
mapfile -t release_files < <(find "$DIST_DIR" -type f \( -name "*.deb" -o -name "homeagent_*.tar.gz" -o -name "*.rpm" \) 2>/dev/null | sort)
|
||||||
|
for f in "${release_files[@]}"; do
|
||||||
echo " $(du -h "$f" | cut -f1) $f"
|
echo " $(du -h "$f" | cut -f1) $f"
|
||||||
done
|
done
|
||||||
|
# 全部包生成之后一次计算,避免边打边算漏掉后生成的产物。
|
||||||
|
if [ ${#release_files[@]} -gt 0 ]; then
|
||||||
|
(
|
||||||
|
cd "$DIST_DIR"
|
||||||
|
find . -type f \( -name "*.deb" -o -name "homeagent_*.tar.gz" -o -name "*.rpm" \) \
|
||||||
|
-print0 | sort -z | xargs -0 sha256sum > SHA256SUMS
|
||||||
|
)
|
||||||
|
echo " SHA256SUMS: $DIST_DIR/SHA256SUMS"
|
||||||
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
main
|
main
|
||||||
|
|||||||
259
deploy/packaging/windows/install-via-wsl.ps1
Normal file
259
deploy/packaging/windows/install-via-wsl.ps1
Normal file
@ -0,0 +1,259 @@
|
|||||||
|
<#
|
||||||
|
.SYNOPSIS
|
||||||
|
在 WSL2 中安装 HomeAgent(homed + 插件 + WebUI)。
|
||||||
|
|
||||||
|
.DESCRIPTION
|
||||||
|
Windows 不再提供 homed 的原生安装。原因见 cmd/homed/platform_windows.go:
|
||||||
|
homed 的插件体系依赖「继承的 fd」与「统一共享内存区的段内偏移解引用」,
|
||||||
|
Windows 的句柄模型无法表达这两者;强行适配等于再维护一套平台专属 ABI,
|
||||||
|
而 C ABI 时代三套 ABI 并存正是「改写型插件在某个平台上静默失效」的根因。
|
||||||
|
|
||||||
|
本脚本因此把 Windows 安装流程变成一条引导链:
|
||||||
|
检测 WSL → 必要时引导安装 → 配置(默认版本 2 / systemd)
|
||||||
|
→ 把 **Linux 包** 送进发行版 → 在 WSL 内按 Linux 的方式安装。
|
||||||
|
|
||||||
|
它复用 Linux 侧的安装包与初始化脚本,不另写一套安装逻辑——
|
||||||
|
「WSL 里就是普通 linux/amd64」这一点必须保持成立,否则等于又开了第三个平台。
|
||||||
|
|
||||||
|
.PARAMETER PayloadDir
|
||||||
|
内含 Linux 安装包的目录(安装器把它解到临时目录后传进来)。
|
||||||
|
优先取 *.deb;没有 deb 时回退 *.tar.gz。
|
||||||
|
|
||||||
|
.PARAMETER Distro
|
||||||
|
目标发行版名。省略则用默认发行版;没有发行版时引导安装 Ubuntu。
|
||||||
|
|
||||||
|
.PARAMETER DataDir
|
||||||
|
WSL 内的数据目录。默认 /var/lib/homeagent(与 Linux 原生安装一致)。
|
||||||
|
不建议放 /mnt/c/...:跨文件系统 IO 慢,且 inotify 语义受限。
|
||||||
|
|
||||||
|
.NOTES
|
||||||
|
⚠️ 本脚本在开发环境(Linux)中只能做语法/逻辑审查,**未在真实 Windows + WSL
|
||||||
|
上执行过**。首次使用请逐段核对输出;下面每个阶段都打印了实际执行的命令,
|
||||||
|
便于定位到具体哪一步与预期不符。
|
||||||
|
#>
|
||||||
|
[CmdletBinding()]
|
||||||
|
param(
|
||||||
|
[Parameter(Mandatory = $true)][string]$PayloadDir,
|
||||||
|
[string]$Distro = "",
|
||||||
|
[string]$DataDir = "/var/lib/homeagent",
|
||||||
|
[string]$ApiKey = "",
|
||||||
|
[string]$WebUIUser = "",
|
||||||
|
[string]$WebUIPass = "",
|
||||||
|
[switch]$Uninstall
|
||||||
|
)
|
||||||
|
|
||||||
|
$ErrorActionPreference = "Stop"
|
||||||
|
$script:StageNo = 0
|
||||||
|
$script:DistroName = $Distro
|
||||||
|
|
||||||
|
function Write-Stage([string]$Text) {
|
||||||
|
$script:StageNo++
|
||||||
|
Write-Host ""
|
||||||
|
Write-Host ("=" * 64) -ForegroundColor DarkGray
|
||||||
|
Write-Host ("[$script:StageNo] $Text") -ForegroundColor Cyan
|
||||||
|
Write-Host ("=" * 64) -ForegroundColor DarkGray
|
||||||
|
}
|
||||||
|
|
||||||
|
function Write-Ok([string]$Text) { Write-Host " ✓ $Text" -ForegroundColor Green }
|
||||||
|
function Write-Warn2([string]$Text) { Write-Host " ! $Text" -ForegroundColor Yellow }
|
||||||
|
function Fail([string]$Text, [string]$Hint = "") {
|
||||||
|
Write-Host ""
|
||||||
|
Write-Host " 安装中止:$Text" -ForegroundColor Red
|
||||||
|
if ($Hint) { Write-Host " $Hint" -ForegroundColor Yellow }
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
# ── 0. 前置检查 ────────────────────────────────────────────────────────────
|
||||||
|
Write-Stage "前置检查"
|
||||||
|
|
||||||
|
$identity = [Security.Principal.WindowsPrincipal][Security.Principal.WindowsIdentity]::GetCurrent()
|
||||||
|
if (-not $identity.IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator)) {
|
||||||
|
# 装 WSL 与写 \\wsl$ 都需要管理员。不静默提权:用户应当看到发生了什么。
|
||||||
|
Fail "需要管理员权限" "请以管理员身份重新运行安装程序。"
|
||||||
|
}
|
||||||
|
Write-Ok "管理员权限"
|
||||||
|
|
||||||
|
if (-not (Get-Command wsl.exe -ErrorAction SilentlyContinue)) {
|
||||||
|
Write-Warn2 "未找到 wsl.exe"
|
||||||
|
Write-Host " homed 不再提供 Windows 原生版本,必须通过 WSL2 运行。"
|
||||||
|
Write-Host ""
|
||||||
|
Write-Host " 在管理员 PowerShell 中执行:" -ForegroundColor Yellow
|
||||||
|
Write-Host " wsl --install" -ForegroundColor White
|
||||||
|
Write-Host " 然后重启 Windows,再重新运行本安装程序。"
|
||||||
|
Write-Host ""
|
||||||
|
Write-Host " (Windows 10 需 2004+ 且启用虚拟机平台;Windows 11 开箱可用)"
|
||||||
|
exit 20
|
||||||
|
}
|
||||||
|
Write-Ok "wsl.exe 可用"
|
||||||
|
|
||||||
|
# ── 1. 检测 WSL 状态与发行版 ───────────────────────────────────────────────
|
||||||
|
Write-Stage "检测 WSL 与发行版"
|
||||||
|
|
||||||
|
# wsl -l -v 在「没有发行版」时返回非零,且输出是 UTF-16LE——直接解析会踩编码坑。
|
||||||
|
# 用 --status 取默认发行版,再单独枚举列表。
|
||||||
|
$distros = @()
|
||||||
|
try {
|
||||||
|
$raw = (& wsl.exe -l -q 2>$null | Out-String)
|
||||||
|
$distros = $raw -split "`r?`n" | ForEach-Object { $_.Trim() } | Where-Object { $_ -ne "" }
|
||||||
|
} catch {
|
||||||
|
$distros = @()
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($distros.Count -eq 0) {
|
||||||
|
Write-Warn2 "WSL 已安装,但没有任何发行版"
|
||||||
|
Write-Host ""
|
||||||
|
Write-Host " 请先安装发行版(推荐 Ubuntu):" -ForegroundColor Yellow
|
||||||
|
Write-Host " wsl --install -d Ubuntu" -ForegroundColor White
|
||||||
|
Write-Host ""
|
||||||
|
Write-Host " 首次启动 Ubuntu 会要求创建 Linux 用户名与密码,完成后重新运行本安装程序。"
|
||||||
|
exit 21
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($script:DistroName -eq "") {
|
||||||
|
try {
|
||||||
|
$script:DistroName = (& wsl.exe --status 2>$null | Select-String -Pattern "Default Distribution" |
|
||||||
|
ForEach-Object { ($_ -split ":")[1].Trim() })
|
||||||
|
} catch { }
|
||||||
|
if (-not $script:DistroName) { $script:DistroName = $distros[0] }
|
||||||
|
}
|
||||||
|
Write-Ok "发行版:$($script:DistroName)(共 $($distros.Count) 个:$($distros -join ', '))"
|
||||||
|
|
||||||
|
# ── 2. 确保是 WSL2 ─────────────────────────────────────────────────────────
|
||||||
|
Write-Stage "确保使用 WSL2"
|
||||||
|
|
||||||
|
# WSL1 没有真正的 Linux 内核、没有 systemd,且在共享内存/事件语义上与 WSL2 不同。
|
||||||
|
# homed 依赖 eventfd + mmap 语义,WSL1 会以难以诊断的方式失败,因此显式要求 WSL2。
|
||||||
|
try {
|
||||||
|
$verLine = (& wsl.exe -l -v 2>$null | Out-String) -split "`r?`n" |
|
||||||
|
Where-Object { $_ -match [regex]::Escape($script:DistroName) } | Select-Object -First 1
|
||||||
|
if ($verLine -match "\b1\b") {
|
||||||
|
Write-Warn2 "该发行版当前是 WSL1,正在升级为 WSL2 ..."
|
||||||
|
& wsl.exe --set-version $script:DistroName 2
|
||||||
|
if ($LASTEXITCODE -ne 0) { Fail "WSL2 升级失败" "可手动执行:wsl --set-version $($script:DistroName) 2" }
|
||||||
|
}
|
||||||
|
} catch { }
|
||||||
|
& wsl.exe --set-default-version 2 | Out-Null
|
||||||
|
Write-Ok "已使用 WSL2"
|
||||||
|
|
||||||
|
# ── 3. 准备 Linux 包 ───────────────────────────────────────────────────────
|
||||||
|
Write-Stage "准备 Linux 安装包"
|
||||||
|
|
||||||
|
$deb = Get-ChildItem -Path $PayloadDir -Filter "*.deb" -ErrorAction SilentlyContinue | Select-Object -First 1
|
||||||
|
$tar = Get-ChildItem -Path $PayloadDir -Filter "*.tar.gz" -ErrorAction SilentlyContinue | Select-Object -First 1
|
||||||
|
if ($deb) {
|
||||||
|
$pkg = $deb.FullName
|
||||||
|
$pkgKind = "deb"
|
||||||
|
} elseif ($tar) {
|
||||||
|
$pkg = $tar.FullName
|
||||||
|
$pkgKind = "tar"
|
||||||
|
} else {
|
||||||
|
Fail "在 $PayloadDir 下既没找到 .deb 也没找到 .tar.gz" "安装器应把 Linux 包解到该目录。"
|
||||||
|
}
|
||||||
|
Write-Ok "使用 $(Split-Path $pkg -Leaf)($pkgKind)"
|
||||||
|
|
||||||
|
# ── 4. 把包送进 WSL ────────────────────────────────────────────────────────
|
||||||
|
Write-Stage "把安装包送入 WSL"
|
||||||
|
|
||||||
|
# 走 /mnt/c 而不是 \\wsl$:前者是 WSL 稳定的对外通道,且不需要额外的 UNC 权限;
|
||||||
|
# 后者在某些 Windows 版本上对 Program Files 路径有重定向限制。
|
||||||
|
$winPath = (Resolve-Path $pkg).Path
|
||||||
|
$mntPath = "/mnt/" + $winPath.Substring(0, 1).ToLower() + ($winPath.Substring(2) -replace '\\', '/')
|
||||||
|
Write-Host " 源:$mntPath"
|
||||||
|
|
||||||
|
& wsl.exe -d $script:DistroName -u root -- bash -lc "mkdir -p /tmp/homeagent-install"
|
||||||
|
if ($LASTEXITCODE -ne 0) { Fail "无法在 WSL 内创建临时目录" "确认发行版可正常启动:wsl -d $($script:DistroName)" }
|
||||||
|
& wsl.exe -d $script:DistroName -u root -- bash -lc "cp '$mntPath' /tmp/homeagent-install/"
|
||||||
|
if ($LASTEXITCODE -ne 0) { Fail "复制安装包失败" }
|
||||||
|
Write-Ok "已送到 /tmp/homeagent-install/"
|
||||||
|
|
||||||
|
# ── 5. 在 WSL 内安装 ───────────────────────────────────────────────────────
|
||||||
|
Write-Stage "在 WSL 内安装 homed"
|
||||||
|
|
||||||
|
# 凭据经环境变量传给 setup.sh(它已支持 HOMEAGENT_API_KEY / WEBUI_USER / WEBUI_PASS)。
|
||||||
|
# 不传的话就会「界面显示一份、config.db 里另一份」,用户直接登录不上。
|
||||||
|
$credEnv = ""
|
||||||
|
if ($ApiKey) { $credEnv += "export HOMEAGENT_API_KEY='$ApiKey'; " }
|
||||||
|
if ($WebUIUser) { $credEnv += "export WEBUI_USER='$WebUIUser'; " }
|
||||||
|
if ($WebUIPass) { $credEnv += "export WEBUI_PASS='$WebUIPass'; " }
|
||||||
|
|
||||||
|
# 安装逻辑复用 Linux 侧:deb 走 apt(postinst 会调用 setup.sh 生成凭据与 config.db),
|
||||||
|
# tar 则解包到你同一套布局再执行同一份 setup.sh。刻意不在这里重写安装步骤——
|
||||||
|
# 「WSL 里就是普通 linux/amd64」必须保持成立,否则等于又开了第三个平台。
|
||||||
|
if ($pkgKind -eq "deb") {
|
||||||
|
$inWslPkg = "/tmp/homeagent-install/" + (Split-Path $pkg -Leaf)
|
||||||
|
& wsl.exe -d $script:DistroName -u root -- bash -lc @"
|
||||||
|
set -e
|
||||||
|
$credEnv
|
||||||
|
export HOMEAGENT_DATA='$DataDir'
|
||||||
|
apt-get update -qq
|
||||||
|
DEBIAN_FRONTEND=noninteractive apt-get install -y -qq '$inWslPkg'
|
||||||
|
"@
|
||||||
|
} else {
|
||||||
|
$inWslPkg = "/tmp/homeagent-install/" + (Split-Path $pkg -Leaf)
|
||||||
|
& wsl.exe -d $script:DistroName -u root -- bash -lc @"
|
||||||
|
set -e
|
||||||
|
$credEnv
|
||||||
|
mkdir -p /opt/homeagent /tmp/homeagent-extract
|
||||||
|
tar -xzf '$inWslPkg' -C /tmp/homeagent-extract
|
||||||
|
cd /tmp/homeagent-extract
|
||||||
|
# 与 deb 完全相同的布局:/usr/bin/homed + /usr/lib/homeagent/setup.sh。
|
||||||
|
# 两套安装若落到不同路径,之后的升级/排障就会出现「按文档找不到文件」。
|
||||||
|
install -m 0755 homed /usr/bin/homed
|
||||||
|
install -m 0755 waiter /usr/bin/waiter
|
||||||
|
[ -f initconfig ] && install -m 0755 initconfig /usr/bin/initconfig
|
||||||
|
if [ -f homeagent.service ]; then
|
||||||
|
install -m 0644 homeagent.service /etc/systemd/system/homeagent.service
|
||||||
|
fi
|
||||||
|
mkdir -p /usr/lib/homeagent
|
||||||
|
if [ -f setup.sh ]; then install -m 0755 setup.sh /usr/lib/homeagent/setup.sh; fi
|
||||||
|
export HOMEAGENT_DATA='$DataDir'
|
||||||
|
if [ -x /usr/lib/homeagent/setup.sh ]; then bash /usr/lib/homeagent/setup.sh; fi
|
||||||
|
"@
|
||||||
|
}
|
||||||
|
if ($LASTEXITCODE -ne 0) {
|
||||||
|
Fail "WSL 内安装失败(退出码 $LASTEXITCODE)" "可进入 WSL 手动排查:wsl -d $($script:DistroName)"
|
||||||
|
}
|
||||||
|
Write-Ok "安装完成"
|
||||||
|
|
||||||
|
# ── 6. 启动与自启 ──────────────────────────────────────────────────────────
|
||||||
|
Write-Stage "启动 homed 与自启配置"
|
||||||
|
|
||||||
|
& wsl.exe -d $script:DistroName -u root -- bash -lc @"
|
||||||
|
if command -v systemctl >/dev/null 2>&1 && systemctl list-unit-files 2>/dev/null | grep -q homeagent; then
|
||||||
|
systemctl enable homeagent 2>/dev/null || true
|
||||||
|
systemctl restart homeagent
|
||||||
|
echo ' ✓ systemd 服务 homeagent 已启动并设为自启'
|
||||||
|
else
|
||||||
|
# 没有 systemd(WSL2 默认可能没开):用 nohup 起,并把自启交给 Windows 侧的计划任务。
|
||||||
|
pkill -f '/usr/bin/homed' 2>/dev/null || true
|
||||||
|
nohup /usr/bin/homed -data '$DataDir' > /var/log/homeagent-boot.log 2>&1 &
|
||||||
|
echo ' ✓ 已用 nohup 启动(未检测到 systemd)'
|
||||||
|
fi
|
||||||
|
"@
|
||||||
|
|
||||||
|
$creds = & wsl.exe -d $script:DistroName -u root -- bash -lc "cat '$DataDir/credentials.txt' 2>/dev/null || true"
|
||||||
|
|
||||||
|
Write-Host ""
|
||||||
|
Write-Host "============================================================" -ForegroundColor Green
|
||||||
|
Write-Host " HomeAgent 已在 WSL2($($script:DistroName))内安装完成" -ForegroundColor Green
|
||||||
|
Write-Host "============================================================" -ForegroundColor Green
|
||||||
|
Write-Host ""
|
||||||
|
Write-Host " WebUI:http://localhost:8080" -ForegroundColor White
|
||||||
|
Write-Host " (WSL2 会把 WSL 内的端口映射到 Windows 的 localhost,无需额外配置)"
|
||||||
|
Write-Host ""
|
||||||
|
if ($creds) {
|
||||||
|
Write-Host " 初始凭据(也保存在 WSL 内 $DataDir/credentials.txt):" -ForegroundColor Yellow
|
||||||
|
Write-Host $creds
|
||||||
|
} else {
|
||||||
|
Write-Host " 未读到凭据文件,请进入 WSL 检查:cat $DataDir/credentials.txt" -ForegroundColor Yellow
|
||||||
|
}
|
||||||
|
Write-Host ""
|
||||||
|
Write-Host " 常用操作(在 PowerShell 中):"
|
||||||
|
Write-Host " 进入 WSL : wsl -d $($script:DistroName)"
|
||||||
|
Write-Host " 查看日志 : wsl -d $($script:DistroName) -u root -- journalctl -u homeagent -f"
|
||||||
|
Write-Host " 重启服务 : wsl -d $($script:DistroName) -u root -- systemctl restart homeagent"
|
||||||
|
Write-Host ""
|
||||||
|
Write-Host " 注意:WSL 实例不会随 Windows 启动而自动拉起。若需要开机自启,"
|
||||||
|
Write-Host " 可创建一个登录时触发的计划任务执行:"
|
||||||
|
Write-Host " wsl -d $($script:DistroName) -u root -- systemctl start homeagent"
|
||||||
|
exit 0
|
||||||
@ -66,19 +66,41 @@ def put_file(url: str, headers: dict, path: str) -> tuple[int, str]:
|
|||||||
return 0, f"{type(e).__name__}: {e}"
|
return 0, f"{type(e).__name__}: {e}"
|
||||||
|
|
||||||
|
|
||||||
|
def project_root() -> str:
|
||||||
|
"""向上找带 go.mod 的目录作为仓库根。
|
||||||
|
|
||||||
|
为何不数 dirname:本脚本初版在 scripts/(深度 1),移到 deploy/scripts/
|
||||||
|
(深度 2)后写死的两层 dirname 就指向了 deploy/dist/release,上传直接
|
||||||
|
FileNotFoundError。这正是 v0.7.2 那次 package/ → deploy/packaging/ 打断
|
||||||
|
PROJECT_ROOT 的同一个坑,改成按标记文件定位以后怎么挑位置都不会错。
|
||||||
|
"""
|
||||||
|
d = os.path.dirname(os.path.abspath(__file__))
|
||||||
|
while d != os.path.dirname(d):
|
||||||
|
if os.path.exists(os.path.join(d, "go.mod")):
|
||||||
|
return d
|
||||||
|
d = os.path.dirname(d)
|
||||||
|
# 实在找不到(脚本被单独拷出仓库)就回退到 cwd,给 ASSET_DIR 一个机会
|
||||||
|
return os.getcwd()
|
||||||
|
|
||||||
|
|
||||||
def main() -> int:
|
def main() -> int:
|
||||||
if len(sys.argv) < 3:
|
if len(sys.argv) < 3:
|
||||||
print(__doc__)
|
print(__doc__)
|
||||||
return 2
|
return 2
|
||||||
tag, token = sys.argv[1], sys.argv[2]
|
tag, token = sys.argv[1], sys.argv[2]
|
||||||
outdir = os.environ.get("ASSET_DIR") or os.path.join(
|
outdir = os.environ.get("ASSET_DIR") or os.path.join(
|
||||||
os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
|
project_root(), "dist", "release"
|
||||||
"dist",
|
|
||||||
"release",
|
|
||||||
)
|
)
|
||||||
|
if not os.path.isdir(outdir):
|
||||||
|
print(f"error: 资产目录不存在: {outdir}")
|
||||||
|
print(" 用 ASSET_DIR=<目录> 显式指定,或先跑构建生成 dist/release/")
|
||||||
|
return 2
|
||||||
files = sys.argv[3:] or sorted(
|
files = sys.argv[3:] or sorted(
|
||||||
f for f in os.listdir(outdir) if is_artifact(f)
|
f for f in os.listdir(outdir) if is_artifact(f)
|
||||||
)
|
)
|
||||||
|
if not files:
|
||||||
|
print(f"error: {outdir} 下没有可识别的发布产物")
|
||||||
|
return 2
|
||||||
print(f"repo={REPO} tag={tag} dir={outdir}", flush=True)
|
print(f"repo={REPO} tag={tag} dir={outdir}", flush=True)
|
||||||
failed = []
|
failed = []
|
||||||
for name in files:
|
for name in files:
|
||||||
|
|||||||
20
deploy/systemd/embed-sidecar.service
Normal file
20
deploy/systemd/embed-sidecar.service
Normal file
@ -0,0 +1,20 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=Jina v5-omni-nano Embedding Sidecar for HomeAgent
|
||||||
|
After=network.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=simple
|
||||||
|
User=root
|
||||||
|
WorkingDirectory=/home/newqqagent
|
||||||
|
ExecStart=/usr/local/bin/python3 /home/program/TrueAgent/scripts/embed_sidecar.py
|
||||||
|
Restart=on-failure
|
||||||
|
RestartSec=5
|
||||||
|
Environment=JINA_MODEL_DIR=/home/newqqagent/models/jina-v5-omni-nano
|
||||||
|
Environment=JINA_PORT=18999
|
||||||
|
Environment=JINA_DIMENSION=768
|
||||||
|
Environment=OMP_NUM_THREADS=8
|
||||||
|
Environment=MKL_NUM_THREADS=8
|
||||||
|
Environment=TOKENIZERS_PARALLELISM=false
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
124
docs/embedding-comparison.md
Normal file
124
docs/embedding-comparison.md
Normal file
@ -0,0 +1,124 @@
|
|||||||
|
# 检索方案对比报告(2026-09-09)
|
||||||
|
## 测试数据
|
||||||
|
- 文档库:492 篇生产文档(过滤 108 条健康检查测试文档)
|
||||||
|
- 媒体库:3 张生产图片(验证码、新闻截图、深色模式备忘录)
|
||||||
|
- 文本查询:10 组(精确匹配、语义、跨语言、模糊表达)
|
||||||
|
- 媒体查询:6 组(中文/英文查图片,3 张图片各 2 条)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 一、文本检索对比(文档库)
|
||||||
|
|
||||||
|
| 方案 | Hit@1 | Hit@5 | MRR | 平均延迟 |
|
||||||
|
|------|-------|-------|-----|----------|
|
||||||
|
| TF-IDF | 3/10 | 7/10 | 0.457 | 0.3ms |
|
||||||
|
| fastText(200k 中文+378k 英文) | 5/10 | 5/10 | 0.530 | 8.3ms |
|
||||||
|
| TF-IDF + fastText RRF | 4/10 | 7/10 | 0.552 | 12.3ms |
|
||||||
|
| **Jina v5-omni-nano** | **8/10** | **10/10** | **0.900** | **39.9ms** |
|
||||||
|
|
||||||
|
### 关键发现
|
||||||
|
|
||||||
|
1. **Jina 的优势来自"短语语义"能力**:
|
||||||
|
- "邮件代理是否已经成功接入" → TF-IDF rank 5,Jina rank 1
|
||||||
|
- "升级安装 QQ 插件包" → fastText rank 169,Jina rank 1(margin +0.30)
|
||||||
|
- "我所在城市的天气预报" → fastText rank 44,Jina rank 1
|
||||||
|
- "聊天输入区域文字多了会不会自动增高" → TF-IDF rank 1,Jina rank 1(margin +0.33)
|
||||||
|
|
||||||
|
2. **TF-IDF 在精确匹配上不可替代**:
|
||||||
|
- "长期文档记忆功能是否健康" → TF-IDF rank 3,Jina rank 1
|
||||||
|
- "重新加载全部扩展组件" → TF-IDF rank 0(完全未命中),Jina rank 2
|
||||||
|
- TF-IDF 的 Hit@5 70% 证明精确关键词召回仍有价值
|
||||||
|
|
||||||
|
3. **RRF 融合反而变差**:
|
||||||
|
- TF-IDF+fastText RRF MRR=0.552,低于 Jina 单路 0.900
|
||||||
|
- 原因:两种稀疏向量的排序在语义查询上高度重叠,RRF 无法弥补各自短板
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 二、图片检索对比(同 3 张图片,6 条查询)
|
||||||
|
|
||||||
|
| 方案 | Hit@1 | MRR | 平均 margin |
|
||||||
|
|------|-------|-----|-------------|
|
||||||
|
| CLIP ViT-B/32 | 4/6 | 0.806 | -0.008(负值!) |
|
||||||
|
| Jina v5-omni-nano | 4/6 | 0.833 | +0.024 |
|
||||||
|
|
||||||
|
### 逐条对比
|
||||||
|
|
||||||
|
| 查询 | CLIP rank | CLIP margin | Jina rank | Jina margin |
|
||||||
|
|------|-----------|-------------|-----------|-------------|
|
||||||
|
| 验证码图片(中) | 1 | +0.027 | 1 | +0.036 |
|
||||||
|
| 验证码图片(英) | 1 | +0.063 | 1 | +0.077 |
|
||||||
|
| 新闻截图(中) | 6 | -0.091 | 2 | -0.064 |
|
||||||
|
| 新闻截图(英) | 1 | +0.008 | 2 | -0.028 |
|
||||||
|
| 备忘录截图(中) | 3 | -0.045 | 1 | +0.045 |
|
||||||
|
| 备忘录截图(英) | 1 | +0.051 | 1 | +0.079 |
|
||||||
|
|
||||||
|
### 关键发现
|
||||||
|
|
||||||
|
1. **中文文本→图片**:Jina 明显优于 CLIP(MRR 0.833 vs 0.611)
|
||||||
|
- CLIP 中文查询余弦可低至 -0.076(完全反直觉)
|
||||||
|
- Jina 最差也是 +0.045,正样本始终高于负样本
|
||||||
|
|
||||||
|
2. **新闻截图是共同弱点**:
|
||||||
|
- CLIP 和 Jina 都被"深色模式备忘录"抢走新闻截图的排序
|
||||||
|
- 原因:新闻截图的文字描述含"深色"、"备忘录"等词,与备忘录图片的视觉特征重叠
|
||||||
|
- 这是描述质量 vs 视觉特征的竞争,不是模型问题
|
||||||
|
|
||||||
|
3. **margin 的实际意义**:
|
||||||
|
- CLIP 的平均 margin = -0.008(负值意味着正样本平均不如负样本)
|
||||||
|
- Jina 的平均 margin = +0.024(正样本始终略高于负样本)
|
||||||
|
- 但两者的 margin 都很小(< 0.1),生产环境仍需阈值校准
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 三、延迟与资源
|
||||||
|
|
||||||
|
| 方案 | 单次查询延迟 | 索引构建 | 内存 |
|
||||||
|
|------|-------------|----------|------|
|
||||||
|
| TF-IDF | 0.3ms | <1s | ~50MB |
|
||||||
|
| fastText | 8.3ms | <1s | ~200MB |
|
||||||
|
| CLIP ONNX | 26ms | N/A | ~600MB |
|
||||||
|
| Jina v5-omni CPU | 39.9ms | 78s(492篇) | ~4GB |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 四、结论与建议
|
||||||
|
|
||||||
|
### 核心判断
|
||||||
|
|
||||||
|
| 维度 | TF-IDF/fastText | CLIP | Jina v5-omni |
|
||||||
|
|------|-----------------|------|--------------|
|
||||||
|
| 文本精确匹配 | ★★★★★ | N/A | ★★★★ |
|
||||||
|
| 文本语义检索 | ★★ | N/A | ★★★★★ |
|
||||||
|
| 中文文本→图片 | 无能力 | ★ | ★★★★ |
|
||||||
|
| 英文文本→图片 | 无能力 | ★★★ | ★★★★ |
|
||||||
|
| 图片→图片 | 无能力 | ★★★ | ★★★★ |
|
||||||
|
| 多语言统一空间 | 无能力 | 有限 | ★★★★★ |
|
||||||
|
| 延迟 | ★★★★★ | ★★★ | ★★ |
|
||||||
|
|
||||||
|
### 架构建议
|
||||||
|
|
||||||
|
1. **保留 TF-IDF 作为精确召回的一级通道**:
|
||||||
|
- 0.3ms 延迟不可替代
|
||||||
|
- Hit@5 70% 证明在关键词匹配场景仍有价值
|
||||||
|
- 特别是"插件安装"、"设备查询"这类精确操作指令
|
||||||
|
|
||||||
|
2. **用 Jina 替换 fastText + CLIP 的稠密通道**:
|
||||||
|
- Jina 单路 MRR=0.90,超过 fastText+CLIP 融合
|
||||||
|
- 统一空间消除三条通道的维护成本
|
||||||
|
- 中文文本→图片从"无法检索"提升到"可检索"
|
||||||
|
|
||||||
|
3. **两路融合:TF-IDF + Jina RRF**(而非 TF-IDF + fastText RRF):
|
||||||
|
- TF-IDF 精确匹配 + Jina 语义覆盖
|
||||||
|
- RRF 避免跨空间分数归一化问题
|
||||||
|
- 预期 MRR > 0.90(精确匹配补 Jina 的语义盲区)
|
||||||
|
|
||||||
|
4. **图片检索仍需阈值校准**:
|
||||||
|
- Jina 的 margin 平均 +0.024,生产环境需设置合理阈值
|
||||||
|
- 建议:用真实正负样本对重新标定,而非沿用 CLIP 的 0.20 阈值
|
||||||
|
|
||||||
|
### 下一步
|
||||||
|
|
||||||
|
- 实现 TF-IDF + Jina RRF 融合,验证 MRR 是否能突破 0.90
|
||||||
|
- 用更多生产图片标定 Jina 的图片检索阈值
|
||||||
|
- 测试 fastText 词嵌入是否可以完全被 Jina 文本编码替代(L0 相关性计算)
|
||||||
@ -1,7 +1,8 @@
|
|||||||
# Git 分支管理规范
|
# Git 分支管理规范
|
||||||
|
|
||||||
> 生效:2026-08-31。适用:**本仓(TrueAgent/HomeAgent)与 third_party/homeagent-sdk(SDK 仓)**——两仓协作时分支策略必须一致,本规范两仓同用。
|
> 生效:2026-08-31,2026-09-04 修订(三级发布通道 + 单条发布分支),2026-09-06 修订(SDK 仓版本语义与发版联动,见 §七)。
|
||||||
> 核心原则一句话:**main 唯一长命、永远可部署;一切新工作在特性分支;版本发布走 release 分支 + tag;hotfix 只进 released 分支并 cherry-pick 回 main。**
|
> 适用:**本仓(TrueAgent/HomeAgent)与 third_party/homeagent-sdk(SDK 仓)**——两仓协作时分支策略必须一致,本规范两仓同用。
|
||||||
|
> 核心原则一句话:**main 唯一长命、永远可部署;一切新工作在特性分支;一个中版本一条发布分支,alpha/beta/正式由 tag 区分;hotfix 只进发布分支并 cherry-pick 回 main。**
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@ -11,18 +12,24 @@
|
|||||||
|---|---|---|---|---|
|
|---|---|---|---|---|
|
||||||
| `main` | **唯一长命分支** | — | — | ✅ **永远可部署** |
|
| `main` | **唯一长命分支** | — | — | ✅ **永远可部署** |
|
||||||
| `feature/xxx` | 短命(本次特性完成即删) | main | 合回 main | ❌ 不部署 |
|
| `feature/xxx` | 短命(本次特性完成即删) | main | 合回 main | ❌ 不部署 |
|
||||||
| `release/vX.Y.Z` | 中命(从切出到下个版本发布) | main | 打 tag → 构建发布 | ✅ **发布产物来源** |
|
| `release/vX.Y.x` | 中命(**整个中版本生命周期**) | main | 打 tag → 构建发布 | ✅ **发布产物来源** |
|
||||||
| hotfix(直接提交 release 分支) | 随 release 分支 | release 分支 | **cherry-pick 回 main** | ✅ |
|
| hotfix(直接提交发布分支) | 随发布分支 | 发布分支 | **cherry-pick 回 main** | ✅ |
|
||||||
|
|
||||||
```
|
```
|
||||||
main ──────────────── E ──────────────── G ────────────────(永远可部署)
|
main ──────────────── E ──────────────── G ────────────────(永远可部署)
|
||||||
│ ▲
|
│ ▲
|
||||||
│ feature/xxx │ cherry-pick(hotfix 逐个 pick 回)
|
│ feature/xxx │ cherry-pick(修复逐个 pick 回)
|
||||||
├── A ── B ──(合回)───────────────────┤
|
├── A ── B ──(合回)───────────────────┤
|
||||||
│ │
|
│ │
|
||||||
└── release/v1.2.0 release/v1.2.0
|
└── release/v1.0.x ────────────────────────────────────────────────
|
||||||
├─(tag v1.2.0)→ 构建发布 ├─(hotfix) F ← 版本特定严重 bug
|
│ │ │
|
||||||
└─ 退役(可删可留) └─ F 被 separately cherry-pick 到 main
|
├─(tag v1.0.0-alpha.1) 内部验证 │ │
|
||||||
|
├─(tag v1.0.0-beta.1) 小范围试用 │ │
|
||||||
|
├─(tag v1.0.0) 正式发布 │ │
|
||||||
|
├─(hotfix) F ─────────────────────┤ │
|
||||||
|
├─(tag v1.0.1) patch 发布 │ │
|
||||||
|
├─(hotfix) H ────────────────────────────────────┤
|
||||||
|
└─(tag v1.0.3) patch 发布
|
||||||
```
|
```
|
||||||
|
|
||||||
---
|
---
|
||||||
@ -32,8 +39,9 @@ main ──────────────── E ────────
|
|||||||
### 1. `main`(唯一长命分支)
|
### 1. `main`(唯一长命分支)
|
||||||
|
|
||||||
- **唯一长期存在且永远可部署**。任何时刻 `git checkout main` 出来都是可构建、可上线的状态。
|
- **唯一长期存在且永远可部署**。任何时刻 `git checkout main` 出来都是可构建、可上线的状态。
|
||||||
- 积攒**下一个版本**的功能:feature 分支完成即合回,main 持续向前。
|
- 积攒**下一个中版本**的功能:feature 分支完成即合回,main 持续向前。
|
||||||
- **main 上不直接开发**。所有改动经 feature 分支合入;hotfix 经 cherry-pick 注入。
|
- **main 上不直接开发**。所有改动经 feature 分支合入;hotfix 经 cherry-pick 注入。
|
||||||
|
- **main 的 `internal/meta.Version` 始终是下一个未发布版本**,不随 patch 发布变动。
|
||||||
- 合入门禁(**单人直推也遵守**,不强制 PR 但强制验证):
|
- 合入门禁(**单人直推也遵守**,不强制 PR 但强制验证):
|
||||||
- `make test` 全绿
|
- `make test` 全绿
|
||||||
- 涉及插件/工具链时:接口冻结检查 `git diff third_party/homeagent-sdk/sdk/` 为空
|
- 涉及插件/工具链时:接口冻结检查 `git diff third_party/homeagent-sdk/sdk/` 为空
|
||||||
@ -41,80 +49,160 @@ main ──────────────── E ────────
|
|||||||
|
|
||||||
### 2. `feature/xxx`(新特性/修复)
|
### 2. `feature/xxx`(新特性/修复)
|
||||||
|
|
||||||
- 命名:`feature/<短横线描述>`,如 `feature/plugin-proc-migration`、`feature/webui-narrow-fix`。
|
- 命名:`feature/<短横线描述>`,如 `feature/plugin-proc-migration`、`feature/memory-media`。
|
||||||
- **从 main 开出**:`git checkout -b feature/xxx main`。
|
- **从 main 开出**:`git checkout -b feature/xxx main`。
|
||||||
- 完成后合回 main:
|
- 完成后合回 main:
|
||||||
- 单人:直推(`git merge --no-ff` 保留特性边界,或 squash 成一个 commit,二选一在团队内固定)。
|
- 单人:直推(`git merge --no-ff` 保留特性边界,或 squash 成一个 commit,二选一在团队内固定)。
|
||||||
- 多人:走 PR(review 后合入)。
|
- 多人:走 PR(review 后合入)。
|
||||||
- 合回后删除 feature 分支(避免累积)。
|
- 合回后删除 feature 分支(避免累积)。
|
||||||
|
|
||||||
### 3. `release/vX.Y.Z`(发布)
|
### 3. `release/vX.Y.x`(发布分支:一个中版本一条)
|
||||||
|
|
||||||
- **从 main 的某个可部署点切出**:`git checkout -b release/v1.2.0 main`。
|
- **命名用 `x` 占位 patch 位**:`release/v1.0.x` 承载 1.0.0 → 1.0.1 → … → 1.0.N 全部发布,
|
||||||
- 切出后**冻结功能**——release 分支上只做:版本号 bump、发布准备、bug 修复、文档。
|
直到 `release/v1.1.x` 切出为止。**不要按 patch 号建分支**(`release/v1.0.1`、`release/v1.0.3` 各一条会把
|
||||||
- 打 tag → 构建发布安装包 → 上传(附件命名规范见历史记录)。
|
同一发布线切成互不相连的碎片,追溯时无法用一条分支看完整条线的演进)。
|
||||||
- **现网部署永远用 release tag 的构建产物**,不是 main 头部、更不是 feature。
|
- **从 main 的某个可部署点切出**:`git checkout -b release/v1.0.x main`。
|
||||||
|
- 切出后**冻结功能**——发布分支上只做:版本号 bump、发布准备、bug 修复、文档。
|
||||||
|
- **现网部署永远用发布分支上 tag 的构建产物**,不是 main 头部、更不是 feature。
|
||||||
|
|
||||||
### 4. hotfix(只属于此版本的严重 bug)
|
### 4. 三级发布通道(alpha / beta / 正式)
|
||||||
|
|
||||||
|
通道**由 tag 区分,不由分支区分**——三者共用同一条 `release/vX.Y.x`。
|
||||||
|
|
||||||
|
| 通道 | tag 形式 | 含义 | 受众 |
|
||||||
|
|---|---|---|---|
|
||||||
|
| alpha | `vX.Y.Z-alpha.N` | 功能齐了但未充分验证,可能有已知缺陷 | 仅内部/开发者自测 |
|
||||||
|
| beta | `vX.Y.Z-beta.N` | alpha 问题已修,等待真实环境暴露长尾问题 | 小范围试用、愿意承担风险的用户 |
|
||||||
|
| 正式 | `vX.Y.Z` | 通过验证,可上现网 | 所有用户 |
|
||||||
|
|
||||||
|
- **推进顺序**:alpha → beta → 正式,逐级向前,**每级都是同一条分支上的新 tag**。
|
||||||
|
这也是 semver 的标准预发布语义(`1.1.0-alpha.1 < 1.1.0-beta.1 < 1.1.0`),
|
||||||
|
包管理器与版本比较逻辑天然认得,无需额外约定。
|
||||||
|
- **允许跳级**:若改动小、验证充分(如仅一处已定位并有回归测试覆盖的内核修复),
|
||||||
|
可直接打正式 tag。跳级要在发布说明里写明理由。
|
||||||
|
- alpha/beta 的构建产物**可以上传 release 附件**,但必须在 gitcode release 上勾选
|
||||||
|
"预发布"标记,且发布说明首行标注通道与已知风险。
|
||||||
|
- **beta 未清零的严重问题不得进正式**:正式 tag 意味着"我们认为它能上 24/7 现网"。
|
||||||
|
- **发版动作只在发布分支上做**:版本号 bump、打 tag、构建产物、上传 release 附件,
|
||||||
|
全部发生在 `release/vX.Y.x` 上。**main 永远不是发版分支**——即使某个改动刚刚合进 main、
|
||||||
|
即使 main 此刻可部署,也不从 main 打 tag、不拿 main 的构建产物发布。
|
||||||
|
main 的版本号是「下一个未发布中版本」的路牌,不是任何一次发布的版本号。
|
||||||
|
|
||||||
|
### 5. hotfix(发布后发现的严重 bug)
|
||||||
|
|
||||||
- **场景**:版本已发布后,发现只存在于该版本(或该发布线)的严重 bug。
|
- **场景**:版本已发布后,发现只存在于该版本(或该发布线)的严重 bug。
|
||||||
- **动作**:直接把修复提交到 **release 分支**(不收进 main 的开发流)→ 该 release 分支重新构建、打 patch tag(如 `v1.2.1`)发布。
|
- **动作**:直接把修复提交到**发布分支** → 该分支重新构建、打下一个 patch tag(如 `v1.0.4`)发布。
|
||||||
- **关键:hotfix 必须 cherry-pick 回 main**:
|
- **关键:hotfix 必须 cherry-pick 回 main**:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# 在 release 分支上提交修复(代码部分与版本号 bump 分开提交)
|
# 在发布分支上提交修复(代码部分与版本号 bump 分开提交)
|
||||||
|
git checkout release/v1.0.x
|
||||||
git commit -m "fix(x): ..." # ① 修复本身
|
git commit -m "fix(x): ..." # ① 修复本身
|
||||||
git commit -m "chore: bump v1.2.1" # ② 版本号(此 commit 不 pick 回 main)
|
git commit -m "chore(release): bump v1.0.4" # ② 版本号(此 commit 不 pick 回 main)
|
||||||
|
git tag -a v1.0.4 -m "..."
|
||||||
|
|
||||||
# 回到 main,只挑修复本身
|
# 回到 main,只挑修复本身
|
||||||
git checkout main
|
git checkout main
|
||||||
git cherry-pick <修复commit的sha> # 只 pick ①,不 pick ②
|
git cherry-pick <修复①的sha> # 只 pick ①,不 pick ②
|
||||||
```
|
```
|
||||||
|
|
||||||
> **为什么 cherry-pick 而不是 merge**:release 分支只承载该版本特有的补丁,merge 会把 release 分支的版本号/发布相关改动一并带进 main 造成冲突。逐个 cherry-pick 修复 commit 让 main 精确地只获得修复本身。**版本号 bump 不要 pick 回 main**(main 的版本号应始终是下一个未发布版本)。
|
> **为什么 cherry-pick 而不是 merge**:发布分支只承载该版本特有的补丁,merge 会把
|
||||||
|
> 版本号/发布相关改动一并带进 main 造成冲突,并让 main 的 `meta.Version` 变成
|
||||||
|
> 已发布的旧版本号。逐个 cherry-pick 让 main 精确地只获得修复本身。
|
||||||
|
> **版本号 bump 不要 pick 回 main。**
|
||||||
|
|
||||||
- **hotfix 已逐个 pick 回 main ⇒ main 已含全部修复 ⇒ 无需再合并 release 回 main**。这是本规范刻意为之——除非 release 分支上有 main 想要的**功能级**改动(罕见),否则 release 永不 merge 回 main。
|
- **同时存在多个活跃 feature 分支时**:修复也要 pick 到那些分支,否则它们合回 main 时
|
||||||
|
可能带回旧代码。实践做法是修复落地当天就 pick 到全部活跃分支
|
||||||
|
(如 2026-09-04 的 stage 双重解锁修复同时 pick 到 `main` 与 `feature/memory-media`)。
|
||||||
|
|
||||||
### 5. release 分支退役
|
- **hotfix 已逐个 pick 回 main ⇒ main 已含全部修复 ⇒ 无需再合并发布分支回 main**。
|
||||||
|
这是本规范刻意为之——除非发布分支上有 main 想要的**功能级**改动(罕见),
|
||||||
|
否则发布分支永不 merge 回 main。
|
||||||
|
|
||||||
- **下个版本发布 = 此 release 分支生命周期结束**(不再维护)。
|
### 6. 发布分支退役
|
||||||
|
|
||||||
|
- **下个中版本发布 = 上一条发布分支生命周期结束**(`release/v1.1.x` 出现即 `release/v1.0.x` 退役)。
|
||||||
- 退役后可删可留:
|
- 退役后可删可留:
|
||||||
- 删除:保持仓库干净(tag 已保留全部历史,删分支不丢东西)。
|
- 删除:保持仓库干净(tag 已保留全部历史,删分支不丢东西)。
|
||||||
- 保留:便于追溯该发布线的历史构建(对 24/7 现网友好,推荐与本仓库一样保留已打 tag 的历史分支做对照)。
|
- 保留:便于追溯该发布线的历史构建(对 24/7 现网友好)。
|
||||||
- 本仓对现网多代版本并行维护时,保留近期 release 分支是合理的。
|
- **按 patch 号命名的历史发布分支应当合并/删除**:它们是本规范修订前的遗留形态,
|
||||||
|
内容已被对应的 `release/vX.Y.x` 完全包含,保留只会让"哪条才是这条线"变得含糊。
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 三、当前分支对齐(2026-08-31 执行)
|
## 三、当前分支对齐(2026-09-12 更新)
|
||||||
|
|
||||||
### 主仓(TrueAgent)
|
### 主仓(TrueAgent)
|
||||||
|
|
||||||
| 现存分支 | 状态 | 处理 |
|
| 分支 | 状态 | 处理 |
|
||||||
|---|---|---|
|
|---|---|---|
|
||||||
| `main` | `48b5c24` [origin/main] | ✅ 保持不变(规范基线) |
|
| `main` | 含全部回流修复;`meta.Version` = 下一个未发布中版本(现为 `1.3.0`) | ✅ 保持 |
|
||||||
| `feature/plugin-proc-migration` | 原 `update`,`69a138c`(领先 main 5:文档基线 + Part 0.1/0.2 + 本规范) | ✅ **已对齐重命名**(2026-08-31) |
|
| `release/v1.2.x` | **本条发布线**,`meta.Version` = `1.2.0`,vendored SDK 定版 `1.2.0`;已载入两个发布前修复(GUI 输出目录、知识库同名覆盖) | 🆕 2026-09-12 从 main 切出;**尚无 tag** |
|
||||||
| `backup-local`(SDK 仓) | `7092d15`(ahead 3, behind 14,含 `ignore example/recoverydiag` 敏感提交) | ⚠️ 遗留本地分支,功能已合入 main,**保留不删**(无远端,删除即永久丢失) |
|
| `release/v1.1.x` | 承载 `v1.1.0-beta.1` / `v1.1.0` / `v1.1.1` | 📦 已退役(§2.6:下个中版本发布即退役),保留供追溯 |
|
||||||
|
| `release/v1.0.x` | 承载 1.0.x 全部 tag | 📦 保留 |
|
||||||
|
| `feature/multimodal-embedding` | 已合入 main(`eb4762a`,43 提交,`--no-ff`) | ⏳ 待删(删远端分支需用户确认,§执行守则 3) |
|
||||||
|
|
||||||
|
> `feature/memory-media`、`feature/plugin-proc-migration` 均已从远端删除(旧表里的待删项已处理)。
|
||||||
|
|
||||||
### SDK 仓(homeagent-sdk)
|
### SDK 仓(homeagent-sdk)
|
||||||
|
|
||||||
| 现存分支 | 状态 | 处理 |
|
| 分支 | 状态 | 处理 |
|
||||||
|---|---|---|
|
|---|---|---|
|
||||||
| `main` | `61f307b` v1.2.0 | ✅ 保持不变 |
|
| `main` | `meta.Version` = 下一个未发布中版本(现为 **`1.2.0`**)——SDK **不跟 beta 发版**(§七.2),1.2.0 要等核心的**正式** tag 才定版(§七.3),在那之前路牌不得越过它。此阶段与核心 main(`1.3.0`)**故意不对称**,详见 §七.4 | ✅ 保持 |
|
||||||
| `update` | `5648519`(领先 main 1:Part 0.2 模板修复) | ⚠️ 与主仓 `update` 对齐重命名 |
|
| `release/v1.1.x` | `meta.Version` = `1.1.0`,承载 tag `v1.1.0` | ✅ 与核心对应 |
|
||||||
| `backup-local` | `7092d15`(ahead 3, behind 14,遗留调试分支) | ⚠️ 可选清理 |
|
| `release/v1.2.x` | **尚未创建** | ⏳ 随核心**正式** tag 一起建(§七.3:分支上把版本定为 `1.2.0` 再打 `v1.2.0`;beta 阶段不发 SDK) |
|
||||||
|
| `release/v1.0.0` | 旧 patch 号命名形态,内容已被 main 完全包含 | 📦 保留(供追溯 1.0 线构建) |
|
||||||
|
|
||||||
> `update` 整改工作分支按规范应为 `feature/plugin-proc-migration`(多进程插件化整改,8-9 周大特性)。
|
### 1.0.x 发布线 tag 历史
|
||||||
> 是否重命名由执行人确认;不重命名则视为偏离规范的既有分支,须在文档记录其存在。
|
|
||||||
|
| tag | 提交 | 通道 | 说明 |
|
||||||
|
|---|---|---|---|
|
||||||
|
| `v1.0.0` | `9b92a04` | 正式 | 外部插件从 C ABI 迁移到子进程 + 共享内存 |
|
||||||
|
| `v1.0.1` | `e671a8c` | 正式 | 多模态 bugfix(假成功、能力声明与回退链、see_video 帧数语义) |
|
||||||
|
| `v1.0.3` | `26dc76f` | 正式 | 内核 stage 协调器双重解锁(直接跳正式:单点修复 + 反向验证 + 全类审计) |
|
||||||
|
|
||||||
|
> `v1.0.2` 未使用:该号从未发布也无 tag,留空以免与任何本地构建混淆。
|
||||||
|
|
||||||
|
### 1.1.x 发布线 tag 历史
|
||||||
|
|
||||||
|
| tag | 提交 | 通道 | SDK | 说明 |
|
||||||
|
|---|---|---|---|---|
|
||||||
|
| `v1.1.0` | `579d7db` | 正式 | 1.0.0 | 记忆系统支持二进制多媒体节点(CAS 媒体存储 + L0/L2/L3 贯通) |
|
||||||
|
| `v1.1.0-beta.1` | `7a57a14` | beta | 不发 | 打包链路验证(GUI 架构污染 + 空壳 node_modules)。按 §七.2,beta 不伴随 SDK 发版 |
|
||||||
|
| `v1.1.1` | 见发布说明 | 正式 | **1.1.0** | 多模态贯通插件边界;SDK 首次随核心正式版发布 |
|
||||||
|
|
||||||
|
> `v1.1.0-beta.1` 的提交序在 `v1.1.0` **之后**(它多含一个打包修复),
|
||||||
|
> 而 semver 预发布语义里 `1.1.0-beta.1 < 1.1.0`。这是「一条发布分支 + tag 区分通道」的
|
||||||
|
> 已知代价:beta 是为验证**打包链路**而补打的,不代表源码更旧。发布说明里已注明。
|
||||||
|
|
||||||
|
### 1.2.x 发布线 tag 历史
|
||||||
|
|
||||||
|
| tag | 提交 | 通道 | SDK | 说明 |
|
||||||
|
|---|---|---|---|---|
|
||||||
|
| `v1.2.0-beta.1` | `215804c` | beta | 不发 | 统一多模态向量空间 + 媒体升为图记忆一等节点 + 数据面全量迁到共享内存(RPC 协议 **2**,与 1.x 不兼容)。按 §七.2,beta 不伴随 SDK 发版 |
|
||||||
|
| (正式 tag 待打) | — | — | — | 试运行 beta 无回退问题后打 `v1.2.0`,并同步 SDK 仓 `release/v1.2.x` + `v1.2.0` |
|
||||||
|
|
||||||
|
> 1.2.x 与存量插件**不兼容**:RPC 协议升到 2(fd3 布局改变),存量外部插件必须用
|
||||||
|
> 新版 plugindev 重编为 `plugin.bin`——**不支持滚动升级**,内核与插件须同批重建、同批安装。
|
||||||
|
> 按 §2.4,跳级直发正式版需在发布说明里列明「单点修复 / 反向验证 / 全类审计」三项;
|
||||||
|
> 本次改动面大(统一多模态向量空间 + 协议 2 + 数据面全量迁移),不满足跳级条件。
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 四、现网部署与版本对应(运维纪律)
|
## 四、现网部署与版本对应(运维纪律)
|
||||||
|
|
||||||
- **现网 homed 永远部署 `release/vX.Y.Z` 分支打出的 tag 构建**,路径见 `Makefile`(`make build` → `build/homed`)。
|
- **现网 homed 永远部署 `release/vX.Y.x` 分支上 tag 的构建产物**,路径见 `Makefile`(`make build` → `build/homed`)。
|
||||||
- systemd 服务(`/usr/local/bin/homed`)替换前:备份旧二进制 → 停服 → 替换 → 起服 → 健康检查(`scripts/verify_deploy.sh`)。
|
- systemd 服务(`/usr/local/bin/homed`)替换流程:
|
||||||
- **改造期间(update 整改)现网不得部署 main 或 feature 的中间态**——只有发版才用 release。
|
1. 备份旧二进制(`homed.bak.pre<版本>.<时间戳>`)
|
||||||
|
2. 备份配置库(**用 `sqlite3 .backup`,不用 `cp`**——WAL 模式下 cp 可能拿到不一致快照)
|
||||||
|
3. 记录当前插件建链清单,供重启后逐项比对
|
||||||
|
4. `install -m 0755` 替换(原子 rename,不会写坏正在运行的进程镜像)
|
||||||
|
5. `systemctl restart homeagent`
|
||||||
|
6. 健康检查:版本号、插件清单无缺失、`/api/v1/status`、一次真实对话、`fatal error` 计数为 0
|
||||||
|
- **改造期间现网不得部署 main 或 feature 的中间态**——只有发版才用发布分支的 tag。
|
||||||
|
- alpha/beta tag 的产物**不上现网**(现网是 24/7 服务,预发布通道的存在就是为了不拿它冒险)。
|
||||||
- 涉及 SDK 仓时:主仓 `go.mod` 的 `replace => ./third_party/homeagent-sdk` 指向本地 vendored 副本,
|
- 涉及 SDK 仓时:主仓 `go.mod` 的 `replace => ./third_party/homeagent-sdk` 指向本地 vendored 副本,
|
||||||
发版前确认 vendored SDK 与 SDK 仓 release tag 一致(两仓版本对齐是第一优先级)。
|
发版前确认 vendored SDK 与 SDK 仓 release tag 一致(**两仓中版本对齐是第一优先级**,见 §七)。
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@ -128,28 +216,112 @@ git checkout -b feature/xxx
|
|||||||
git checkout main && git merge --no-ff feature/xxx # 或 squash
|
git checkout main && git merge --no-ff feature/xxx # 或 squash
|
||||||
git branch -d feature/xxx
|
git branch -d feature/xxx
|
||||||
|
|
||||||
# 发布
|
# 开一条新中版本的发布线
|
||||||
git checkout -b release/v1.2.0 main
|
git checkout -b release/v1.1.x main
|
||||||
git commit -am "chore: bump v1.2.0" # 版本号
|
git commit -am "chore(release): bump v1.1.0-alpha.1"
|
||||||
git tag v1.2.0
|
git tag -a v1.1.0-alpha.1 -m "..." # alpha:内部验证
|
||||||
# ... 构建发布 ...
|
# ... 修问题 ...
|
||||||
|
git commit -am "chore(release): bump v1.1.0-beta.1"
|
||||||
|
git tag -a v1.1.0-beta.1 -m "..." # beta:小范围试用
|
||||||
|
# ... 真实环境验证 ...
|
||||||
|
git commit -am "chore(release): bump v1.1.0"
|
||||||
|
git tag -a v1.1.0 -m "..." # 正式
|
||||||
|
|
||||||
# hotfix(发布后)
|
# hotfix(发布后)——注意是同一条 release/v1.0.x,不新建分支
|
||||||
git checkout release/v1.2.0
|
git checkout release/v1.0.x
|
||||||
git commit -am "fix(x): 严重 bug" # ① 修复
|
git commit -am "fix(x): 严重 bug" # ① 修复
|
||||||
git commit -am "chore: bump v1.2.1" # ② 版本号
|
git commit -am "chore(release): bump v1.0.4" # ② 版本号
|
||||||
git tag v1.2.1
|
git tag -a v1.0.4 -m "..."
|
||||||
git checkout main
|
git checkout main
|
||||||
git cherry-pick <修复①的sha> # ③ 只挑修复
|
git cherry-pick <修复①的sha> # ③ 只挑修复
|
||||||
|
# 若有活跃 feature 分支,也 pick 过去
|
||||||
|
git checkout feature/xxx && git cherry-pick <main 上那个 pick 的 sha>
|
||||||
|
|
||||||
# release 退役(可选)
|
# 公开 SDK 接口改动(feature,不是 hotfix):先进 main,再 pick 到发布分支
|
||||||
git branch -d release/v1.2.0 # tag 已保存历史,删分支不丢东西
|
git checkout -b feature/sdk-xxx main
|
||||||
|
# ... 改 third_party/homeagent-sdk/sdk/ 与内核桥接层 ...
|
||||||
|
git checkout main && git merge --no-ff feature/sdk-xxx
|
||||||
|
git checkout release/v1.1.x
|
||||||
|
git cherry-pick <feature 的各 sha> # 只挑改动,不挑 main 的版本号
|
||||||
|
git commit -am "chore(release): bump v1.1.1" # 发布分支自己的版本号
|
||||||
|
git tag -a v1.1.1 -m "..."
|
||||||
|
# SDK 仓同步(仅在核心打正式 tag 时,见 §七.2/§七.3)
|
||||||
|
cd third_party/homeagent-sdk
|
||||||
|
git checkout -b release/v1.1.x main
|
||||||
|
git commit -am "chore(release): SDK 1.1.0(1.1.x 线全程共用)"
|
||||||
|
git tag -a v1.1.0 -m "..."
|
||||||
|
|
||||||
|
# 发布分支退役(下个中版本发布后,可选)
|
||||||
|
git branch -d release/v1.0.x # tag 已保存历史,删分支不丢东西
|
||||||
```
|
```
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 六、本规范与「接口冻结」约束的关系
|
## 六、本规范与「接口冻结」约束的关系
|
||||||
|
|
||||||
- feature 分支合回 main 的门禁(`git diff sdk/` 为空)是本仓特有的硬约束,独立于 Git 流程本身。
|
- feature 分支合回 main 的门禁(`git diff third_party/homeagent-sdk/sdk/` 为空)是本仓特有的硬约束,独立于 Git 流程本身。
|
||||||
- 插件多进程化整改(`feature/plugin-proc-migration` 或现 `update`)**不满足接口冻结不等于不能合并**——
|
- `internal/sdk` **不受冻结约束**,可自由扩展;冻结只针对公开 SDK 接口(`third_party/homeagent-sdk/sdk/`)。
|
||||||
接口冻结约束的是「公开 SDK 不变」,整改若突破需走变更评审(见 `docs/zh/plugin-interface-matrix.md` §七)。
|
- 若整改确需突破公开接口,走变更评审(见 `docs/zh/plugin-interface-matrix.md` §七),
|
||||||
|
并同步 `SDKCompatibleVersion` 与 SDK 仓的 release tag。
|
||||||
|
- **公开接口的改动本身是 feature,不是发布准备**:它必须走 `feature/xxx` → 合回 main 的路径,
|
||||||
|
再 cherry-pick 到发布分支。不允许把接口新增当成"发布分支上的 bug 修复"直接提交进 release
|
||||||
|
——发布分支冻结功能(§2.3),接口是最典型的功能面。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 七、SDK 仓的版本语义与发版联动
|
||||||
|
|
||||||
|
### 1. SDK 版本号跟随核心的中版本,patch 位恒为 `.0`
|
||||||
|
|
||||||
|
| 核心版本 | 对应 SDK 版本 |
|
||||||
|
|---|---|
|
||||||
|
| 1.1.0 / 1.1.1 / 1.1.2 / … / 1.1.N | **1.1.0**(全线共用,不随核心 patch 变动) |
|
||||||
|
| 1.2.0 起 | **1.2.0** |
|
||||||
|
|
||||||
|
- 核心的 patch 位(`x`)专用于 **bugfix 与漏洞修复**,这类改动不触碰公开 SDK 接口,
|
||||||
|
因此 SDK 版本号没有理由跟着动。
|
||||||
|
- **为什么不逐位对齐**:SDK 版本号是插件开发者的依赖声明。若核心每发一个 bugfix 就把 SDK
|
||||||
|
也推一个新号,开发者要么被迫跟版、要么怀疑自己版本过时,而接口其实一个字都没变。
|
||||||
|
让 SDK 号只在**接口可能变化的中版本边界**上跳,开发者只需关心「我在为哪个中版本写插件」。
|
||||||
|
- 因此「两仓版本对齐」在本规范里指**中版本对齐**(核心 1.1.x ↔ SDK 1.1.0),
|
||||||
|
不是三位全等。核心 1.1.1 配 SDK 1.1.0 就是对齐状态。
|
||||||
|
|
||||||
|
### 2. beta 阶段不发 SDK
|
||||||
|
|
||||||
|
- **核心的 alpha/beta tag 不伴随 SDK 仓发版**:SDK 仓在这一阶段**不打 tag、不建 release**。
|
||||||
|
- **为什么**:beta 是核心自己的测试阶段,此时 SDK 接口尚未固定。若此刻给 SDK 发版,
|
||||||
|
插件开发者会照着一个还会变的接口写代码——**那是无效开发**。接口没定就没有可依赖的契约,
|
||||||
|
发出去的版本号是一个假承诺。
|
||||||
|
- 这条约束的对象是 **SDK 仓的发版动作**,不是核心二进制里有没有 SDK 代码。
|
||||||
|
主仓 `go.mod` 用 `replace => ./third_party/homeagent-sdk`,任何核心构建都必然含 vendored
|
||||||
|
SDK 源码,这是构建机制决定的,不在本条约束范围内。
|
||||||
|
|
||||||
|
### 3. 正式发布时 SDK 随核心一起发
|
||||||
|
|
||||||
|
核心打**正式 tag**(`vX.Y.Z`,无预发布后缀)时,SDK 仓同步执行:
|
||||||
|
|
||||||
|
1. SDK 仓也有自己的 `release/vX.Y.x`(与核心同名,一个中版本一条);
|
||||||
|
2. 在该分支上把 `meta.Version` 定为 `X.Y.0`;
|
||||||
|
3. 打 tag `vX.Y.0`(首次进入该中版本时),并建 gitcode release;
|
||||||
|
4. 上传 5 平台 plugindev 产物 + `SHA256SUMS`。
|
||||||
|
|
||||||
|
同一中版本内的后续核心 patch(1.1.1 → 1.1.2 …)**不重复发 SDK**——SDK 已经是 1.1.0,
|
||||||
|
没有新东西要发。只有接口再次变化并进入下一个中版本时,SDK 才发 1.2.0。
|
||||||
|
|
||||||
|
### 4. 版本号在两仓 main 上的含义
|
||||||
|
|
||||||
|
两仓的 `main` 都遵守 §2.1:`meta.Version` 是**下一个未发布中版本**。
|
||||||
|
所以在 1.1.x 线发布期间,两仓 main 上的值都是 `1.2.0`——它标记「main 正在积攒 1.2 的东西」,
|
||||||
|
而不是「1.2.0 已经存在」。已发布的版本号一律看对应 `release/vX.Y.x` 分支与 tag。
|
||||||
|
|
||||||
|
**但两仓「同步推进」是有条件的**(这一点曾导致误判,现补写清楚):
|
||||||
|
推进的前提是**该中版本已经正式发布过**。具体到当前:
|
||||||
|
|
||||||
|
- 核心:切出 `release/v1.2.x` 后,1.2.0 就归发布线所有,main 立即推进到 `1.3.0`;
|
||||||
|
**即使 1.2.0 目前只有 beta tag**(beta 不上现网,但发布线已占住这个号)。
|
||||||
|
- SDK:因为 §七.2 **beta 不发 SDK**,SDK 1.2.0 要等核心的**正式** tag 才定版、
|
||||||
|
建 `release/v1.2.x`、打 `v1.2.0`(§七.3)。在那之前,SDK 的「下一个未发布中版本」
|
||||||
|
仍然是 `1.2.0`,其 main 不得越过它。
|
||||||
|
|
||||||
|
→ 因此在这一阶段,**核心 main = `1.3.0` 而 SDK main = `1.2.0` 是正确的**,
|
||||||
|
不是遗漏同步。(曾按本节的例子把 SDK main 也推到 1.3.0,等于宣称 SDK 1.2.0 已发布。)
|
||||||
|
|||||||
455
docs/zh/multimodal-space.md
Normal file
455
docs/zh/multimodal-space.md
Normal file
@ -0,0 +1,455 @@
|
|||||||
|
# 统一多模态向量空间
|
||||||
|
|
||||||
|
核心不绑定任何具体模型:它按 provider 名从公共注册表(`pkg/embedding`)打开一个
|
||||||
|
向量空间。仓库内自带两个:
|
||||||
|
|
||||||
|
| provider | 模态 | 维度 | 实测常驻 | 许可 | 适用 |
|
||||||
|
|---|---|---|---|---|---|
|
||||||
|
| `chineseclip` | text + image | 512 | **1.15 GB** | Apache-2.0 | 默认(内存受限 / 中文图文) |
|
||||||
|
| `qwen3vl` | text + image(视频已实现未纳入契约) | 2048 | 9.4 GB | Apache-2.0 | 内存充足 / 需要更强文本语义或视频 |
|
||||||
|
| `http` | 由外部服务决定 | 由外部服务决定 | 由外部服务决定 | — | 侧车部署(如 jina-v5-omni-nano,注意其 CC BY-NC 许可) |
|
||||||
|
|
||||||
|
下面第一节是 Qwen3-VL(2048 维,最强但最重),第二节是 Chinese-CLIP(512 维,
|
||||||
|
默认推荐)。两者互斥启用,改配置后重启生效。
|
||||||
|
|
||||||
|
文本、图像、**视频帧** 在同一模型、同一维度、同一 fingerprint 空间里被编码。
|
||||||
|
记忆系统用它做三件事:多模态图记忆的跨模态召回、multimodal doc 的向量融合、
|
||||||
|
multimodal context 的相关性裁剪/淘汰。
|
||||||
|
|
||||||
|
统一空间取代了此前「把图片交给视觉模型生成文字描述、再按描述检索」的做法。
|
||||||
|
那条链路有三个致命缺陷:描述是异步生成的(未生成前媒体等于不存在)、语义检索
|
||||||
|
实际上只搜描述文字、图库里的「媒体节点」只是描述文本的投影而不是媒体本身。
|
||||||
|
**不要再引入任何描述式索引。**
|
||||||
|
|
||||||
|
## 一、产物与获取
|
||||||
|
|
||||||
|
产物约 8 GB(含外部权重),**不进仓库**;用导出脚本自动拉取模型并导出:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 默认导出 图像 + 视频 G=2,3,4(即 4/6/8 帧)
|
||||||
|
python3 scripts/export_qwen3vl_embedding_onnx.py \
|
||||||
|
--out /home/newqqagent/models/qwen3-vl-embed-multimodal-onnx
|
||||||
|
|
||||||
|
# 只要 4 帧的视频档(省磁盘、省内存)
|
||||||
|
python3 scripts/export_qwen3vl_embedding_onnx.py --video-groups 2 --out ...
|
||||||
|
|
||||||
|
# 已下载过模型:跳过拉取
|
||||||
|
python3 scripts/export_qwen3vl_embedding_onnx.py \
|
||||||
|
--model-dir /path/to/Qwen3-VL-Embedding-2B \
|
||||||
|
--out /home/newqqagent/models/qwen3-vl-embed-multimodal-onnx
|
||||||
|
|
||||||
|
# 参考向量默认直接写进产物目录(<out>/qwen_reference.json),无需额外参数
|
||||||
|
python3 scripts/export_qwen3vl_embedding_onnx.py --model-dir ... --out ...
|
||||||
|
```
|
||||||
|
|
||||||
|
国内镜像:导出脚本沿用 `huggingface_hub` 的约定,直接 `export HF_ENDPOINT=https://hf-mirror.com` 即可。
|
||||||
|
依赖:`torch`(CPU 版即可)、`transformers>=4.57`、`onnx`、`onnxruntime`、`pillow`、`numpy`,
|
||||||
|
以及可选的 `huggingface_hub` / `modelscope`。显存不需要,内存建议 ≥ 16 GB(FP32 加载约 8 GB)。
|
||||||
|
|
||||||
|
导出脚本**会清空 --out 目录**后重写,避免旧图/旧外部权重污染 fingerprint
|
||||||
|
(fingerprint 变化会触发一次无意义的全量向量重算)。因此不要直接覆盖线上正在使用的目录,
|
||||||
|
先导出到新目录再切换。
|
||||||
|
|
||||||
|
### 产物契约(Go 侧按此读取)
|
||||||
|
|
||||||
|
| 文件 | 输入 | 输出 |
|
||||||
|
|---|---|---|
|
||||||
|
| `TokenEmbedding.onnx` | `input_ids` int64 `[1,seq]` | `hidden` float `[1,seq,2048]` |
|
||||||
|
| `Transformer.onnx` | `hidden`、`deepstack_0/1/2` `[1,seq,2048]`、`rotary_cos/sin` `[1,seq,128]`、`causal_mask` `[1,1,seq,seq]` | `embedding` `[1,2048]` |
|
||||||
|
| `Vision.onnx(+.data)` | `pixel_values` `[2304,1536]` | `deepstack_feature_0/1/2`、`vision_hidden_states` `[576,2048]` |
|
||||||
|
| `Vision_g{N}.onnx` | `pixel_values` `[N×2304,1536]` | 同上,`[N×576,2048]` |
|
||||||
|
|
||||||
|
外加 `tokenizer.json`、`tokenizer_config.json`、`chat_template.jinja`、`embed_config.json`、
|
||||||
|
`qwen_reference.json`。
|
||||||
|
|
||||||
|
`Vision.onnx` 是图像(单时间组);`Vision_g{N}.onnx` 是视频(N 个时间组 = 2N 帧)。
|
||||||
|
**没有 `Vision_g1.onnx`**——单组就是图像那张。
|
||||||
|
|
||||||
|
三段只是部署形式,不是三个向量空间:图文共用同一 token embedding、同一 28 层
|
||||||
|
Transformer、同一 last-token 池化。RoPE 与视觉特征散射故意留在 Go 计算,
|
||||||
|
因为旧式 tracer 会把 `seq=598 / visual=576` 烘焙进图里——签名上写着 dynamic
|
||||||
|
axis,实际却只能用导出的那个长度运行。
|
||||||
|
|
||||||
|
### ⚠️ max_length 必须按最大视频档推导
|
||||||
|
|
||||||
|
`embed_config.json` 的 `max_length` 是**整条序列**的上限,包含视觉占位符:
|
||||||
|
图像只需 598 token(1×576 + 模板),而视频是 G×576——G=2 就要 1190,G=4 要 2342。
|
||||||
|
沿用图像的 1024 会让处理器静默截断,然后在 transformers 内部报
|
||||||
|
`Mismatch in video token count between text and input_ids`。
|
||||||
|
导出脚本因此用 `max_length_for(video_groups) = max(1024, max(G)×576 + 256)` 自动推导,
|
||||||
|
并在构造视觉输入后显式断言视觉 token 数,把错误提前到导出阶段。
|
||||||
|
|
||||||
|
### 导出脚本自检(不可省)
|
||||||
|
|
||||||
|
脚本内部跑两道校验,任一道 cos < 0.999999 就以非零码退出:
|
||||||
|
|
||||||
|
1. 分段 PyTorch(三段组合)对比完整模型前向;
|
||||||
|
2. 用 onnxruntime 跑**导出后**的三段图,再对比完整模型前向。
|
||||||
|
|
||||||
|
「能加载」不等于「算得对」:形状错、输入名错、池化位置错的图都能正常 load。
|
||||||
|
|
||||||
|
## 一·补、text+image 默认空间:Chinese-CLIP ViT-B/16
|
||||||
|
|
||||||
|
**为什么它是默认**:text+image 只需要一个向量空间时,同时满足「小、可商用、中文原生」
|
||||||
|
的选项只有一个。
|
||||||
|
|
||||||
|
| | Chinese-CLIP | jina-v5-omni-nano | Qwen3-VL-Emb-2B |
|
||||||
|
|---|---|---|---|
|
||||||
|
| 参数量 | 188M | 1.04B | 2B |
|
||||||
|
| 产物 / 实测常驻 | **721MB / 1.15GB** | ~2GB / 2.23GB | 8GB / 9.4GB |
|
||||||
|
| 维度 | 512 | 768 | 2048 |
|
||||||
|
| 许可 | **Apache-2.0** | CC BY-NC(不可商用) | Apache-2.0 |
|
||||||
|
| 中文 | 原生(~2 亿中文图文对) | 多语言 | 多语言 |
|
||||||
|
| 文本语义 | 弱(双塔对比) | 好 | 最好 |
|
||||||
|
| 视频 | 无 | 有 | 有 |
|
||||||
|
|
||||||
|
**要诚实记录的代价**:CLIP 是双塔对比学习,text↔image 是强项,但**纯文本语义
|
||||||
|
(text↔text)明显弱于 MLLM 型嵌入器**。文本检索仍由既有词向量/TF-IDF 路径兜底,
|
||||||
|
本空间主要用于跨模态召回与相关性裁剪。需要更强文本语义或视频时切回 `qwen3vl`。
|
||||||
|
|
||||||
|
### 产物与获取
|
||||||
|
|
||||||
|
产物约 754MB,**不进仓库**;用导出脚本从官方权重导出(脚本入库,保证可复现):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
python3 scripts/export_chineseclip_onnx.py \
|
||||||
|
--model-dir /path/to/chinese-clip-vit-base-patch16 \
|
||||||
|
--out /home/newqqagent/models/chinese-clip-vit-b16-onnx
|
||||||
|
```
|
||||||
|
|
||||||
|
国内下载:本机 `huggingface.co` 走代理会被 reset,用 `hf-mirror.com` 且**不设代理**:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
curl -4 -L --retry 3 -o vocab.txt \
|
||||||
|
https://hf-mirror.com/OFA-Sys/chinese-clip-vit-base-patch16/resolve/main/vocab.txt
|
||||||
|
```
|
||||||
|
|
||||||
|
### 产物契约(Go 侧按此读取)
|
||||||
|
|
||||||
|
| 文件 | 输入 | 输出 |
|
||||||
|
|---|---|---|
|
||||||
|
| `TextEncoder.onnx` | `input_ids` int64 `[B,52]`、`attention_mask` int64 `[B,52]` | `text_features` float `[B,512]` |
|
||||||
|
| `VisionEncoder.onnx` | `pixel_values` float `[B,3,224,224]` | `image_features` float `[B,512]` |
|
||||||
|
|
||||||
|
外加 `embed_config.json`(维度/预处理/分词超参/文件名——provider 的唯一权威)、
|
||||||
|
`vocab.txt`、`reference.json`(冻结参考:逐文本 token id + 逐样本向量)、`SHA256SUMS`。
|
||||||
|
|
||||||
|
图像预处理:缩放到 224×224(双三次,复刻 PIL 系数)→ `(x/255 - mean) / std`,
|
||||||
|
不裁剪。文本:BERT WordPiece,`max_length=52`,补 `[PAD]`,超长截断尾部。
|
||||||
|
两个塔的输出**都没有在图中归一化**,归一化由 provider 负责(检索按余弦)。
|
||||||
|
|
||||||
|
### 启用
|
||||||
|
|
||||||
|
```bash
|
||||||
|
core.memory.multimodal_space.provider = chineseclip
|
||||||
|
core.memory.multimodal_space.options.model_dir = /home/newqqagent/models/chinese-clip-vit-b16-onnx
|
||||||
|
```
|
||||||
|
|
||||||
|
**新装默认就是这个**(`SeedDefaults` 写入 `chineseclip` + `<dataDir>/models/chinese-clip-vit-b16-onnx`),
|
||||||
|
发行版构建也默认带 `onnxruntime` 标签(`deploy/packaging/build.sh` 的 `HOMED_TAGS`,
|
||||||
|
需要极简构建时显式 `HOMED_TAGS=` 关闭)。
|
||||||
|
|
||||||
|
**老安装不会自动拿到**:播种判据是显式标记 `core.internal.seed_version`。
|
||||||
|
老安装(已播种过)下次启动只会被补上标记,**不会**被注入新默认值——
|
||||||
|
升级就静默加载 1.8GB 模型不是无副作用的事。要启用请显式写上面两个键。
|
||||||
|
|
||||||
|
> 这个判据曾经是「`config` 表为空才播种」。而发行包的 postinst 会先跑
|
||||||
|
> `initconfig`,它写一行 `webui.listen_addr` ——于是**全新安装**被误判为
|
||||||
|
> "已有配置",整个播种被跳过:没有 `core.plugin.dir`(装完 0 个插件)、
|
||||||
|
> 也没有多模态 provider(随包的模型与运行库成了死重量)。回归测试
|
||||||
|
> `TestSeedDefaultsAfterInitconfigPrepopulate` 与
|
||||||
|
> `TestSeedDefaultsDoesNotInjectIntoLegacyInstall` 钉住了这两种情形。
|
||||||
|
|
||||||
|
同样要求 `homed` 带 `onnxruntime` build tag。
|
||||||
|
|
||||||
|
### 随包分发(server / full 包自带模型与运行库)
|
||||||
|
|
||||||
|
模型与运行库是发行版能力的一部分,不做成「可选下载」:
|
||||||
|
|
||||||
|
| 内容 | 包内路径 |
|
||||||
|
|---|---|
|
||||||
|
| Chinese-CLIP 产物(754MB) | `/usr/lib/homeagent/models/chinese-clip-vit-b16-onnx/` |
|
||||||
|
| ONNX Runtime(24MB) | `/usr/lib/homeagent/onnxruntime/libonnxruntime.so` |
|
||||||
|
| 许可证 | `/usr/share/doc/homeagent/licenses/`(Apache-2.0、MIT、ThirdPartyNotices、模型来源) |
|
||||||
|
|
||||||
|
- `deploy/packaging/package-linux.sh` 的 `stage_multimodal_assets()` 在打 server/full 前
|
||||||
|
会校验产物 `SHA256SUMS`、逐文件非空、运行库架构与目标一致;**缺一即失败**,
|
||||||
|
不生成「默认启用但装完不能用」的假包。`client` 包不含(它不跑 homed)。
|
||||||
|
- 安装时 `setup.sh` 把包内模型目录软链到 `<dataDir>/models/chinese-clip-vit-b16-onnx`
|
||||||
|
(既不复制 754MB,也保持 dataDir 可迁移;已存在的自定义目录绝不覆盖)。
|
||||||
|
- 服务单元设 `Environment=ONNXRUNTIME_DIR=/usr/lib/homeagent/onnxruntime`;
|
||||||
|
provider 的查找顺序是 `ONNXRUNTIME_DIR` → `ONNX_ML_DIR` → 包内路径 →
|
||||||
|
`/opt/onnxruntime` → `/usr/local/lib` → `/usr/lib`。
|
||||||
|
- 构建机需自备产物:`build/model-assets/chinese-clip-vit-b16-onnx/` 与
|
||||||
|
`build/runtime-assets/<arch>/{libonnxruntime.so,LICENSE,ThirdPartyNotices.txt}`
|
||||||
|
(可用 `CHINESECLIP_BUNDLE_DIR` / `ONNXRUNTIME_ASSET_DIR` 覆盖)。
|
||||||
|
|
||||||
|
实测(从真实 deb 解包、按 postinst 顺序跑 `setup.sh`、再冷启动包内 homed):
|
||||||
|
`multimodal space active: provider=chineseclip dim=512 fp=cd2a495cf990 modalities=[text image]`,
|
||||||
|
并完成一次真实对话;`homeagent-server` 包 722MB(旧版 17MB),差额即模型与运行库。
|
||||||
|
|
||||||
|
#### ORT 环境是进程级单例(单主不析构)
|
||||||
|
|
||||||
|
进程内可能有多个 ORT 消费者(本 provider、`qwen3vl`、`internal/nlp` 的依存解析器)。
|
||||||
|
`onnxruntime_go` 的行为是:第二次 `InitializeEnvironment` 报错,而
|
||||||
|
`DestroyEnvironment` 会把别人正在用的环境一起拆掉。约定:
|
||||||
|
|
||||||
|
- 初始化前先 `IsInitialized()`,只有未初始化时才初始化;
|
||||||
|
- **任何消费者都不销毁环境**(环境随进程存活),只销毁自己的会话。
|
||||||
|
|
||||||
|
这个缺陷是「发行版默认带 onnxruntime 标签」后才暴露的:不带标签时多个消费者不会
|
||||||
|
同时存在(此前 `internal/nlp` 会重复初始化并降级,失败路径还会误销毁环境)。
|
||||||
|
|
||||||
|
### 模态范围
|
||||||
|
|
||||||
|
只声明 `text` 与 `image`。`audio`/`video` **明确返回 `ErrUnsupportedModality`**——
|
||||||
|
本空间没有它们的原生编码器,用别的模型向量冒充会污染整个向量空间
|
||||||
|
(这正是「音频明确 unsupported」那条纪律的落地)。
|
||||||
|
|
||||||
|
### 验证
|
||||||
|
|
||||||
|
Go 侧回归对着官方 PyTorch 参考(`reference.json`),模型目录由
|
||||||
|
`CHINESECLIP_MODEL_DIR` 指定,缺失时 skip:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
CHINESECLIP_MODEL_DIR=/home/newqqagent/models/chinese-clip-vit-b16-onnx \
|
||||||
|
go test -tags onnxruntime ./providers/chineseclip/ -v
|
||||||
|
```
|
||||||
|
|
||||||
|
实测结果:文本 5 个用例 `cos = 1.000000000000`(与官方逐位一致);
|
||||||
|
图像 4 个纯色用例 `cos = 1.000000`(自写 bicubic 与 PIL 在 6 位小数内一致);
|
||||||
|
另有跨模态判别、模态拒绝、指纹稳定性、产物缺失报错等用例。
|
||||||
|
|
||||||
|
### 两个已踩过的坑(都在测试里钉住了)
|
||||||
|
|
||||||
|
1. **分词器不能自己拼**。第一版探针用 `BertTokenizer(vocab_file=..., do_lower_case=True)`
|
||||||
|
手工分词,中文被整体切成 `[UNK]`,三个不同句子产出几乎相同的向量(余弦 0.98),
|
||||||
|
差点把「模型坏了」当成结论。官方配置是 `do_lower_case=true` + **删音标生效** +
|
||||||
|
**中文逐字切分**;Go 侧实现必须与官方**逐 token** 对齐(`TestTokenizerMatchesOfficialReference`)。
|
||||||
|
2. **参考向量是未归一化的原始输出**(模长 10~36)。用「点积当余弦 + 单侧下界」判定
|
||||||
|
会得到 13.6 而「通过」——测试里因此改成真余弦 + 双侧容差。
|
||||||
|
|
||||||
|
## 二、启用
|
||||||
|
|
||||||
|
核心不识别任何具体模型:它只按配置里的 **provider 名**从公共注册表
|
||||||
|
(`pkg/embedding`)打开一个 provider,并把 `options.*` 原样交给它。
|
||||||
|
模型文件布局、预处理、媒体解码、运行时都在 provider 内部。
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 配置库(config.db)或 WebUI 设置页
|
||||||
|
core.memory.multimodal_space.provider = qwen3vl
|
||||||
|
core.memory.multimodal_space.options.model_dir = /home/newqqagent/models/qwen3-vl-embed-multimodal-onnx
|
||||||
|
|
||||||
|
# 或换成一个外部向量服务(任何语言写的都行)
|
||||||
|
core.memory.multimodal_space.provider = http
|
||||||
|
core.memory.multimodal_space.options.endpoint = http://127.0.0.1:18999/embed
|
||||||
|
core.memory.multimodal_space.options.dimension = 2048
|
||||||
|
```
|
||||||
|
|
||||||
|
`options.*` 是 provider 自己的命名空间,核心不做任何解释(对 `qwen3vl` 是
|
||||||
|
`model_dir`,对 `http` 是 `endpoint`/`dimension`/`api_key`/…)。第三方 provider
|
||||||
|
可以定义自己的选项,无需改核心。
|
||||||
|
|
||||||
|
注意事项:
|
||||||
|
|
||||||
|
- 内置 provider `qwen3vl` 要求 `homed` 带 `onnxruntime` build tag 构建,且
|
||||||
|
`libonnxruntime.so` 可被找到(`/opt/onnxruntime/libonnxruntime.so` 等)。
|
||||||
|
未带 tag 时该 provider 会注册但打开时报「requires build tag」,而不是静默降级。
|
||||||
|
- `provider` 为空时禁用多模态向量检索,退回纯 fastText 文本路径。
|
||||||
|
- 改配置后需重启进程生效。
|
||||||
|
- 未配置时优雅降级:文档层退到 TF-IDF 稀疏检索,媒体块仍按结构边关联,只是没有跨模态召回。
|
||||||
|
|
||||||
|
## 二·补、给核心接自己的模型
|
||||||
|
|
||||||
|
核心只依赖一个很小的公共接口(`pkg/embedding`):
|
||||||
|
|
||||||
|
```go
|
||||||
|
// 输入对核心是不透明字节:modality 决定语义,Data+MIME 由 provider 解释。
|
||||||
|
type Input struct {
|
||||||
|
Modality Modality // text / image / audio / video / …
|
||||||
|
Purpose Purpose // query / document
|
||||||
|
Text string
|
||||||
|
Data []byte
|
||||||
|
MIME string
|
||||||
|
Metadata map[string]string
|
||||||
|
}
|
||||||
|
|
||||||
|
type Provider interface {
|
||||||
|
Embed(ctx context.Context, in Input) ([]float64, error)
|
||||||
|
Info() Info // Dimension, Fingerprint, Modalities
|
||||||
|
Close()
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
接入步骤:新建一个包,在 `init()` 里 `embedding.Register("your-model", factory)`,
|
||||||
|
再把这个包空白导入你的发行版 `main`(或替换内置 provider 的导入行)。
|
||||||
|
分词、预处理、解码、显存/内存管理、模型文件命名全部由你的 provider 决定。
|
||||||
|
|
||||||
|
两条原则值得强调:
|
||||||
|
|
||||||
|
- **能力是数据,不是接口方法**:支持哪些模态写在 `Info().Modalities` 里。
|
||||||
|
这样新增模态不需要改核心接口,核心也不需要为每个新模态做类型断言。
|
||||||
|
- **不支持的模态返回 `embedding.ErrUnsupportedModality`**,而不要拿别的模型顶替,
|
||||||
|
也不要降级成一个普通错误——调用方靠它区分「永远不会有向量」与「本次失败可重试」。
|
||||||
|
|
||||||
|
## 三、模态覆盖范围
|
||||||
|
|
||||||
|
### Qwen3-VL-Embedding-2B(本空间,2048 维)
|
||||||
|
|
||||||
|
模型卡明载支持 **Text / images / screenshots / videos**;`config.json` 有
|
||||||
|
`image_token_id` 与 `video_token_id`,**没有 `audio_token_id`/`audio_config`**。
|
||||||
|
|
||||||
|
| 模态 | 状态 | 说明 |
|
||||||
|
|---|---|---|
|
||||||
|
| 文本 | ✅ 原生 | `VectorizeDense` |
|
||||||
|
| 图像 | ✅ 原生 | `EmbedImageDense`,`Vision.onnx`,固定 768×768 |
|
||||||
|
| 视频 | ⚠️ 视觉侧已导出并校验,**Go 模板未完成** | `EmbedVideoDense` + `Vision_g{N}.onnx`;见下节 |
|
||||||
|
| 音频 | ❌ 本轮明确不做 | 决策结果;该模型也不具备(无 `audio_token_id`) |
|
||||||
|
|
||||||
|
### 视频:帧 → 时间组 → M-RoPE(均已实测对齐)
|
||||||
|
|
||||||
|
| 项 | 值 | 验证方式 |
|
||||||
|
|---|---|---|
|
||||||
|
| 占位符 | `<|video_pad|>` = **151656**(图像是 `<|image_pad|>` = 151655) | 处理器实测 |
|
||||||
|
| 模板 | 与图像同构,只换占位符 | `apply_chat_template` repr 逐字符比对 |
|
||||||
|
| 帧→槽位 | 组 g 的 tp0←帧2g、tp1←帧2g+1 | PyTorch `torch.equal == True`,maxdiff=0;反向对照 False |
|
||||||
|
| patch 布局 | `[G,24,24,2,2,3,2,16,16]`,即图像排列以 grid_t 为最外层堆叠 | 纯色视频于图像张量 `torch.equal == True` |
|
||||||
|
| 视觉 token | `G×576` | 处理器实测(G=2 → 1152) |
|
||||||
|
| M-RoPE | 每组独立:`base=start+24g`;`t=base`、`h=base+j/24`、`w=base+j%24` | 对应 `get_rope_index` 把 video grid 展开成 G 个 `t=1` 项 |
|
||||||
|
| 用错档 | onnxruntime 报 `InvalidArgument`(维度不符) | 实验实测,**不会静默算错** |
|
||||||
|
|
||||||
|
同步注意事项:
|
||||||
|
|
||||||
|
- **帧数必须恰好是 `2×G`**(G 取已导出的档)。奇数帧时只用得上前 `2×floor(n/2)` 帧,
|
||||||
|
多出的丢弃——不补重复帧,那会改变跳帧注意力看到的运动。
|
||||||
|
- **`video/*`(视频文件)不能直接喂给图像入口**:Go 侧没有视频解码器,
|
||||||
|
`EmbedImageDense(raw, "video/mp4")` 返回 `ErrModalityUnsupported`。调用方必须先抽帧。
|
||||||
|
- 视觉图按需懒加载(每张约 1.6GB),未用到的档位不占内存。
|
||||||
|
|
||||||
|
### 导出视频时踩过的两个坑(都已加断言)
|
||||||
|
|
||||||
|
两个坑都会让产物「看起来正常、实际是错的」,且都不会在导出时报错:
|
||||||
|
|
||||||
|
1. **处理器会静默重采样帧**。不给 `video_metadata` 时它回落到 `fps=24`,
|
||||||
|
把**任何**帧数都改成 `grid_t=2`:实测 4/6/8 帧全部得到 1152 个视觉 token。
|
||||||
|
修法:`processor(..., videos=[frames], do_sample_frames=False)`。
|
||||||
|
2. **`max_length` 只按图像算是不够的**。它是整条序列(含视觉占位符)的上限:
|
||||||
|
图像只需 598 token,而视频是 `G×576`——G=2 要 1190、G=4 要 2342。
|
||||||
|
沿用 1024 会截断并报
|
||||||
|
`Mismatch in video token count between text and input_ids`。
|
||||||
|
修法:`max_length_for(G) = max(1024, max(G)×576 + 256)`。
|
||||||
|
|
||||||
|
两个坑都会在导出脚本里显式断言(视觉 token 数、`video_grid_thw` 的组数),
|
||||||
|
把错误提前到导出阶段而不是留给运行时。
|
||||||
|
|
||||||
|
### 音频(本轮决策:不加)
|
||||||
|
|
||||||
|
**Qwen3-VL 不支持音频**,由模型卡与 `config.json` 双重确认:
|
||||||
|
|
||||||
|
```
|
||||||
|
模型卡:Supported Input Modalities: Text, images, screenshots, videos, and …
|
||||||
|
config:image_token_id ✓ / video_token_id ✓ / audio_token_id ✗ / audio_config ✗
|
||||||
|
```
|
||||||
|
|
||||||
|
本机有音频能力的是另一个模型(**jina-v5-omni-nano**,768 维,含
|
||||||
|
`modeling_llava_eurobert_audio.py` 与 `audio_token_id=128256`),与 Qwen 空间
|
||||||
|
**不同维度、不同坐标系,绝不可互相比较**。决定:**本轮不接入**;
|
||||||
|
其侧车(`scripts/embed_sidecar.py`)也仍只实现 `text`/`image`,`audio` 返回 400。
|
||||||
|
|
||||||
|
无论何时接入,都**不允许**:拿视觉塔去编码音频字节、或用另一个模型的向量
|
||||||
|
冒充某空间的音频向量——那会把两套坐标系混进同一空间,且错误是静默的。
|
||||||
|
音频在原空间返回 `vector.ErrModalityUnsupported`,使调用方区分
|
||||||
|
「永远不会有向量」与「本次失败可重试」。
|
||||||
|
|
||||||
|
Qwen3-VL 视觉塔把 `grid_thw` 当 Python 值消费(源码里是 `grid_thw.tolist()`),
|
||||||
|
legacy tracer(`dynamo=False`)会把它固化成常量:实测把 `grid_thw` 声明为图输入后,
|
||||||
|
导出的 ONNX 图里**根本没有该输入**,换帧数调用直接报 `Invalid input name: grid_thw`;
|
||||||
|
导出时的 TracerWarning 明确提示
|
||||||
|
`Converting a tensor to a Python list might cause the trace to be incorrect`。
|
||||||
|
|
||||||
|
因此视频的可行做法是:**在导出时固定时间组数 G,每个 G 一张 Vision 图**
|
||||||
|
(grid = `[G, 48, 48]`),Go 侧按实际帧数选用匹配的图;用 G=2 的图去喂 G=3 的
|
||||||
|
数据属于未定义行为。视频文件本身不能直接喂进本空间(`video/*` 返回
|
||||||
|
`ErrModalityUnsupported`),必须由上层先抽帧。
|
||||||
|
|
||||||
|
## 四、验证
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Go 侧:ONNX 路径(模型目录缺失时自动 skip)
|
||||||
|
QWEN_ONNX_MODEL_DIR=/home/newqqagent/models/qwen3-vl-embed-multimodal-onnx \
|
||||||
|
go test -tags onnxruntime ./internal/memory/qwen/ -v
|
||||||
|
|
||||||
|
# 排除二进制交付问题的替代:先单独验证模型与 CSV 无关的 ONNX 图
|
||||||
|
go vet -tags onnxruntime ./...
|
||||||
|
```
|
||||||
|
|
||||||
|
Go 测试覆盖:冻结参考向量(文本/图像各 12 维)、同输入确定性、不同输入敏感性、
|
||||||
|
图像与文本向量必须不同、以及音频/视频必须返回 `ErrModalityUnsupported`。
|
||||||
|
|
||||||
|
冻结参考向量由导出脚本写入**产物目录本身**(`<out>/qwen_reference.json`),
|
||||||
|
来源可追溯:同一脚本既产出模型,也产出「这个模型对固定输入应有的输出」。
|
||||||
|
重新导出后若参考值变化,说明权重或图结构变了,必须显式更新参考而不是放宽阈值。
|
||||||
|
|
||||||
|
> **参考向量是 L2 归一化后的值。** ONNX 图返回的是 final norm 之后的原始
|
||||||
|
> last hidden(量级约 100),而 Go 侧 `VectorizeDense` / `EmbedImageDense`
|
||||||
|
> 返回归一化向量。写参考时忘归一化,Go 测试会全线不匹配,而现象看起来
|
||||||
|
> 像“模型不对”,实际只是两边对“向量”的定义不同。
|
||||||
|
|
||||||
|
验证既有产物(不重新导出):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
python3 scripts/export_qwen3vl_embedding_onnx.py --verify-only --model-dir <model> \
|
||||||
|
--out /home/newqqagent/models/qwen3-vl-embed-multimodal-onnx
|
||||||
|
```
|
||||||
|
|
||||||
|
脚本会顺便把归一化后的参考向量写入该目录。
|
||||||
|
|
||||||
|
### 与现有部署产物的等价性
|
||||||
|
|
||||||
|
本仓库脚本对同一源模型导出时,`TokenEmbedding.onnx` 与 `Transformer.onnx` 与
|
||||||
|
线上在用的产物**逐字节相同**(sha256 一致);`Vision.onnx` 差异仅在打包形式:
|
||||||
|
旧产物把权重量到外部 `Vision.onnx.data`,新脚本内联在图里。两者数值等价。
|
||||||
|
|
||||||
|
注意这会带来一个**操作性**差异:Go 的结构指纹(`computeFingerprint`)把
|
||||||
|
`*.onnx.data` 的文件名与大小算在内,因此「外部权重版 ↔ 内联版」互换会让
|
||||||
|
fingerprint 变化,从而触发一次全量向量重算。重算不会**算错**(数值等价),
|
||||||
|
只是白花一次 CPU;若不想触发,就保持产物打包形式不变。
|
||||||
|
|
||||||
|
## 五、资源成本
|
||||||
|
|
||||||
|
- 产物磁盘约 8 GB;导出过程峰值内存约 10–12 GB(FP32 加载)。
|
||||||
|
- 单次 CPU 推理:文本约几十毫秒量级,图像(2304 patch 过 24 层视觉塔 + 28 层语言模型)
|
||||||
|
明显更重,因此入库时不阻塞对话,靠 `reembedStaleMedia` 在启动时并发迁移
|
||||||
|
(ONNX 路径 4 worker)。
|
||||||
|
- fingerprint 由三段图 + `embed_config.json` + 外部权重文件名/大小共同决定;
|
||||||
|
换模型或重新导出都会让它变化,从而触发历史向量重算——这是预期行为。
|
||||||
|
|
||||||
|
## 视频:当前状态(未完成,不得当作已验证)
|
||||||
|
|
||||||
|
**视觉侧**:`Vision_g2/g3/g4.onnx` 已导出,且每一档都与完整 PyTorch 模型逐档对过
|
||||||
|
(`cos` 分别为 1.000000119 / 1.000000119 / 1.000000000,覆盖度断言通过)。
|
||||||
|
|
||||||
|
**Go 侧模板**:与 HuggingFace processor 产出**不相等**,因此冻结回归
|
||||||
|
(`TestEmbedderVideoMatchesONNXReference`)当前**显式跳过**并注明原因,不算通过。
|
||||||
|
|
||||||
|
已定位的差异:processor 会按时间组插入字面时间戳文本。逐 token 实测:
|
||||||
|
|
||||||
|
```
|
||||||
|
<|vision_start|> <0.0 seconds> <|vision_start|> {576×<|video_pad|>} <|vision_end|>
|
||||||
|
<1.0 seconds> <|vision_start|> {576×<|video_pad|>} <|vision_end|>
|
||||||
|
```
|
||||||
|
|
||||||
|
而 Go 侧只生成 `<|vision_start|>{G×576 pads}<|vision_end|>`。同一输入下
|
||||||
|
Python `seq=1190`(1152 视觉 + **38** 文本),Go 侧只有 **22** 个文本 token。
|
||||||
|
|
||||||
|
注意两点:
|
||||||
|
|
||||||
|
- 时间戳文本**也占用 M-RoPE 位置**,所以 `TestVideoModelInputMRope` 的自洽断言
|
||||||
|
通过**不能**证明与官方实现一致(它是拿自己算的序列验自己算的位置)。
|
||||||
|
- 修复位置在 provider 内部(模型专属模板本就属于 provider),不是核心。
|
||||||
|
|
||||||
|
另外,公共 provider 契约把 `Data+MIME` 交给 provider 自行解码;本 provider
|
||||||
|
没有视频解码器(Go 标准库不含 H.264/MP4),因此 `Info().Modalities` **不声明 video**,
|
||||||
|
`Embed(video)` 返回 `ErrUnsupportedModality`。视频走 provider 自己的
|
||||||
|
`EmbedVideoDense`(接收已解码帧)。待核心有了对 provider 不透明的多帧容器后,
|
||||||
|
再把视频纳入公共契约。
|
||||||
@ -1,12 +1,16 @@
|
|||||||
# 外部插件接口不变矩阵(多进程化整改基线)
|
# 外部插件接口不变矩阵(多进程化整改基线)
|
||||||
|
|
||||||
> 状态:**完成 v2**(2026-09-03)——迁移已落地并上生产,内核 v1.0.0。
|
> 状态:**完成 v3**(2026-09-06)——v2 的迁移已上生产(内核 v1.0.0);v3 记录 v1.1.1 的公开接口**扩展**。
|
||||||
> 目的:钉死「暴露给外部插件的接口不变」这一约束的**合同面**——迁移前、迁移后外部插件看到/调用的 SDK 接口完全一致;
|
> 目的:钉死「暴露给外部插件的接口不变」这一约束的**合同面**——迁移前、迁移后外部插件看到/调用的 SDK 接口完全一致;
|
||||||
> 所有改造落在**核心(homed 侧)+ 工具链(plugindev)**,外部插件业务代码零改动,只需用新 plugindev 重编。
|
> 所有改造落在**核心(homed 侧)+ 工具链(plugindev)**,外部插件业务代码零改动,只需用新 plugindev 重编。
|
||||||
>
|
>
|
||||||
> **结果(已验证)**:`git diff third_party/homeagent-sdk/sdk/` 全程为空;17 个 `example/*/plugin.go` 逐字节未改
|
> **结果(已验证)**:`git diff third_party/homeagent-sdk/sdk/` 全程为空;17 个 `example/*/plugin.go` 逐字节未改
|
||||||
> (`git status example/` 无输出);生产 17 插件全部经子进程通道运行。
|
> (`git status example/` 无输出);生产 17 插件全部经子进程通道运行。
|
||||||
>
|
>
|
||||||
|
> ⚠️ **v1.1.x 起冻结约束被有意解除**,因为「接口不变」这条约束本身是为**迁移期**设的:
|
||||||
|
> 它要保的是「换运行模型不动业务代码」。迁移完成后,SDK 需要能随功能演进而扩展,
|
||||||
|
> 否则多模态这类能力永远到不了插件手上。解除的边界见 §九:**只增不减,签名不改**。
|
||||||
|
>
|
||||||
> 维护规则:每次改动公开 SDK 接口面 `third_party/homeagent-sdk/sdk/` 或模板 `tools/plugindev/templates/` 后,
|
> 维护规则:每次改动公开 SDK 接口面 `third_party/homeagent-sdk/sdk/` 或模板 `tools/plugindev/templates/` 后,
|
||||||
> 必须同步更新本矩阵。
|
> 必须同步更新本矩阵。
|
||||||
>
|
>
|
||||||
@ -67,7 +71,7 @@ type Plugin interface {
|
|||||||
|---|---|---|
|
|---|---|---|
|
||||||
| `Settings()` | `SettingsAPI` | **17 插件全部使用**(Get/Set/List/GetCore/SetCore/ListCore/DataDir/GetPlugin/SetPlugin/ListPlugin/RegisterDef/Defs/Dump/Plugins) |
|
| `Settings()` | `SettingsAPI` | **17 插件全部使用**(Get/Set/List/GetCore/SetCore/ListCore/DataDir/GetPlugin/SetPlugin/ListPlugin/RegisterDef/Defs/Dump/Plugins) |
|
||||||
| `Memory()` | `MemoryAPI`(Recall/Commit/Introspect/MergeEntities/Purge) | 低(controllable) |
|
| `Memory()` | `MemoryAPI`(Recall/Commit/Introspect/MergeEntities/Purge) | 低(controllable) |
|
||||||
| `DocMemory()` | `DocMemoryAPI`(Query/Insert/Remove/Stats) | 低 |
|
| `DocMemory()` | `DocMemoryAPI`(Query/Insert/**InsertWithMedia**/Remove/Stats) | 低(`InsertWithMedia` v1.1.0 新增) |
|
||||||
| `TextMemory()` | `TextMemoryAPI`(Append) | 0 当前 |
|
| `TextMemory()` | `TextMemoryAPI`(Append) | 0 当前 |
|
||||||
| `Knowledge()` | `KnowledgeAPI`(Search/Add/List) | 2 |
|
| `Knowledge()` | `KnowledgeAPI`(Search/Add/List) | 2 |
|
||||||
| `LLM()` | `LLMAPI`(ListSources/SetSource/CurrentSource) | 0 当前 |
|
| `LLM()` | `LLMAPI`(ListSources/SetSource/CurrentSource) | 0 当前 |
|
||||||
@ -85,7 +89,14 @@ type Plugin interface {
|
|||||||
| `InjectInterruptText` | `(source, channel, text string)` | example 使用 6 次 → case 6 |
|
| `InjectInterruptText` | `(source, channel, text string)` | example 使用 6 次 → case 6 |
|
||||||
| `InjectTextNoMemory` | `(source, channel, text string)` | → case 7 |
|
| `InjectTextNoMemory` | `(source, channel, text string)` | → case 7 |
|
||||||
| `InjectInputSync` | `(source, channel, text string) string` | → case 47(例:qq 闭环) |
|
| `InjectInputSync` | `(source, channel, text string) string` | → case 47(例:qq 闭环) |
|
||||||
| `SetToolBlocks` | `(blocks []ContentBlock)` | **当前空实现**(C ABI 无对应),迁移后经 arena 二进制注入可实现 |
|
| `SetToolBlocks` | `(blocks []ContentBlock)` | ✅ **v1.1.1 已落地**(`io.setToolBlocks`);同版补上 `PluginSDK` 侧一直缺失的便捷包装——接口里有、便捷方法里没有,插件此前只能自己去拿 injector |
|
||||||
|
| `InjectInputMedia` | `(source, channel, text string, blocks []ContentBlock)` | **v1.1.0 新增** → `io.injectMedia`。与 `SetToolBlocks` 的区别见下方说明 |
|
||||||
|
| `InjectInputMediaSync` | `(source, channel, text string, blocks []ContentBlock) string` | **v1.1.0 新增** → `io.injectMediaSync` |
|
||||||
|
| `InjectInterruptMedia` | `(source, channel, text string, blocks []ContentBlock)` | **v1.1.0 新增** → `io.injectInterruptMedia` |
|
||||||
|
|
||||||
|
**为何媒体注入不能搭 `SetToolBlocks` 的车**:后者只在**工具处理函数内部**可用,且媒体要等
|
||||||
|
**下一条 tool message** 才到模型手上。插件主动发起一轮带媒体的对话、以及中断注入,
|
||||||
|
需要各自的签名,且媒体在**本轮**就随消息发出,并自动落进 CAS、挂上媒体记忆引用。
|
||||||
| `RegisterStopHandler` / `RunStopHandlers` | `(func())` / `()` | 已有(qq 等 1 次) |
|
| `RegisterStopHandler` / `RunStopHandlers` | `(func())` / `()` | 已有(qq 等 1 次) |
|
||||||
| `RegisterOnRemoveHandler` / `RunOnRemoveHandlers` | `(func())` / `()` | example 使用 3 次 |
|
| `RegisterOnRemoveHandler` / `RunOnRemoveHandlers` | `(func())` / `()` | example 使用 3 次 |
|
||||||
| `Set*`(SetIOInjector/SetMemoryAPI/.../SetPluginMgrAPI) | — | 供 bridge/核心启动时接线,插件不直接调 |
|
| `Set*`(SetIOInjector/SetMemoryAPI/.../SetPluginMgrAPI) | — | 供 bridge/核心启动时接线,插件不直接调 |
|
||||||
@ -99,8 +110,12 @@ type Plugin interface {
|
|||||||
| `ChannelDef` | NoMemory/Cleaner(func) | 同上 |
|
| `ChannelDef` | NoMemory/Cleaner(func) | 同上 |
|
||||||
| `ToolCall` / `ToolResult` / `MemItem` | ID/Name/Plugin/Arguments;CallID/Name/Plugin/Success/Result;Role/Content/Score | 全部纯 JSON 可序列化 |
|
| `ToolCall` / `ToolResult` / `MemItem` | ID/Name/Plugin/Arguments;CallID/Name/Plugin/Success/Result;Role/Content/Score | 全部纯 JSON 可序列化 |
|
||||||
| `ContentBlock` / `ImageURL` / `AudioURL` | Type/Text/ImageURL/AudioURL;URL/Detail;URL | 全部可偏移化(迁移评估 3.3 已核实) |
|
| `ContentBlock` / `ImageURL` / `AudioURL` | Type/Text/ImageURL/AudioURL;URL/Detail;URL | 全部可偏移化(迁移评估 3.3 已核实) |
|
||||||
|
| `MediaAttachment`(**v1.1.0 新增**) | Digest/MIME/Data/Name/Description | 一个类型服务两个方向:给 `Data`+`MIME` 是新内容(CAS 按字节去重),只给 `Digest` 是引用已有内容。**读路径不回 `Data`**——一次检索可能命中几十份媒体,全塞回去会撑爆跨进程消息 |
|
||||||
| `Event` / `EventHandler` / `EventSubscriber` | Type/Source/Payload/Timestamp | 迁移后才对外部插件真正可用 |
|
| `Event` / `EventHandler` / `EventSubscriber` | Type/Source/Payload/Timestamp | 迁移后才对外部插件真正可用 |
|
||||||
| `Triple` / `Entity` / `Relation` / `Doc` / `TextEvent` / `PersonProfile` / `SocialRelation` / `Knowledge` / `ConfigDef` | — | 全部 JSON 可序列化 |
|
| `Triple` / `Entity` / `Relation` / `Doc` / `TextEvent` / `PersonProfile` / `SocialRelation` / `Knowledge` / `ConfigDef` | — | 全部 JSON 可序列化 |
|
||||||
|
| `Triple`(**v1.1.0 扩展**) | += `SentenceText` / `MediaDigests` | 媒体引用挂在**句子**上(`SentenceText` → `sentences` → `sentence_id` → `media_refs`),所以 `MediaDigests` 非空而 `SentenceText` 为空时内核会用媒体标记本身充当句子 |
|
||||||
|
| `Doc`(**v1.1.0 扩展**) | += `MediaDigests` / `Attachments` | `Query` 返回时由内核填充(仅元数据,不带字节) |
|
||||||
|
| `TextEvent`(**v1.1.0 扩展**) | += `Attachments` | 写入时内核把标记并进正文;`RecentEvents` 读回时从标记反解 |
|
||||||
|
|
||||||
**函数类型字段盘点(唯一无法跨进程序列化的东西)**:
|
**函数类型字段盘点(唯一无法跨进程序列化的东西)**:
|
||||||
- `ToolDef.Cleaner func(string) string`
|
- `ToolDef.Cleaner func(string) string`
|
||||||
@ -252,7 +267,9 @@ Part 0.2 先做了过渡补丁(只回传真正变更的字段);Part 4 的
|
|||||||
| 能力 | 迁移前 | 迁移后 | 实际结果 |
|
| 能力 | 迁移前 | 迁移后 | 实际结果 |
|
||||||
|---|---|---|---|
|
|---|---|---|---|
|
||||||
| 事件订阅 `Events().Subscribe`(case 23/24) | ❌ 空实现 | ✅ 事件环(EvtRing + eventfd + 独立游标) | ✅ 已接线(当前零用户) |
|
| 事件订阅 `Events().Subscribe`(case 23/24) | ❌ 空实现 | ✅ 事件环(EvtRing + eventfd + 独立游标) | ✅ 已接线(当前零用户) |
|
||||||
| `SetToolBlocks` 多模态注入 | ❌ 空实现 | ✅ 二进制落 arena,Slice 描述符回传 | ⚠️ method 已定义,内核侧仍未实现 |
|
| `SetToolBlocks` 多模态注入 | ❌ 空实现 | ✅ `io.setToolBlocks` | ✅ **v1.1.1 已落地**(走 JSON 而非共享段二进制通道,理由见 §九) |
|
||||||
|
| 媒体入记忆(`InsertWithMedia`、`Triple.MediaDigests`) | ❌ 不存在 | ✅ CAS + 引用计数 GC | ✅ **v1.1.0 类型 / v1.1.1 内核实现** |
|
||||||
|
| 插件主动发起带媒体的一轮对话(`InjectInputMedia*`) | ❌ 不存在 | ✅ 媒体在本轮就到模型手上 | ✅ **v1.1.1** |
|
||||||
| `ContextMsgs`/`ReasoningContent`/`TokenUsage`/`Memory`/`Extra`/`Errors` | ❌ 看不到 | ✅ 共享内存全字段 | ✅ 18 字段全可见可写 |
|
| `ContextMsgs`/`ReasoningContent`/`TokenUsage`/`Memory`/`Extra`/`Errors` | ❌ 看不到 | ✅ 共享内存全字段 | ✅ 18 字段全可见可写 |
|
||||||
| 插件崩溃隔离 | ❌ panic 带崩 homed | ✅ 子进程独立崩溃 | ✅ 测试 + 生产验证 |
|
| 插件崩溃隔离 | ❌ panic 带崩 homed | ✅ 子进程独立崩溃 | ✅ 测试 + 生产验证 |
|
||||||
| 热重载 `.so` | ❌ `DF_1_NODELETE` no-op | ✅ 同路径替换 `.bin` 即生效 | ✅ 生产实测 |
|
| 热重载 `.so` | ❌ `DF_1_NODELETE` no-op | ✅ 同路径替换 `.bin` 即生效 | ✅ 生产实测 |
|
||||||
@ -291,6 +308,8 @@ C 结构体不好传函数指针(那是运气,任何人给 dispatch 加个 c
|
|||||||
3. ✅ **阶段 5**:17 个外部插件全部 `.bin` 化、cabi 删除(-3198 行);
|
3. ✅ **阶段 5**:17 个外部插件全部 `.bin` 化、cabi 删除(-3198 行);
|
||||||
`go build ./...` 与全仓 `go test ./...` 均通过。
|
`go build ./...` 与全仓 `go test ./...` 均通过。
|
||||||
4. ✅ **全程**:`git diff third_party/homeagent-sdk/sdk/` 为零——接口冻结的硬证据。
|
4. ✅ **全程**:`git diff third_party/homeagent-sdk/sdk/` 为零——接口冻结的硬证据。
|
||||||
|
5. ⚠️ **v1.1.x 起该检查项不再适用**:冻结是迁移期的约束,迁移完成即到期(见 §九)。
|
||||||
|
取代它的门禁是「存量插件零改动零重编」——见 §九的验证方式。
|
||||||
|
|
||||||
生产端到端(2026-09-03,真实 QQ 消息):
|
生产端到端(2026-09-03,真实 QQ 消息):
|
||||||
|
|
||||||
@ -304,6 +323,100 @@ tool output_send__qq result: 已通过 [qq] 通道发送: map[status:sent]
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
## 九、v1.1.x 的接口扩展规则(冻结解除后的替代约束)
|
||||||
|
|
||||||
|
冻结约束是为**迁移期**设的:它要保的是「换运行模型不动业务代码」。迁移完成后继续冻结,
|
||||||
|
等于让 SDK 永远停在迁移那天的能力面——多模态这类功能永远到不了插件手上。
|
||||||
|
|
||||||
|
取代它的是三条更弱但仍然硬的约束:
|
||||||
|
|
||||||
|
### 1. 只增不减,签名不改
|
||||||
|
|
||||||
|
新增字段、新增方法可以;**改已有方法的签名、删字段、改字段语义不行**。
|
||||||
|
|
||||||
|
实例:v1.1.0 想让插件能给三元组关联媒体,两条路——改 `Commit` 的签名加一个参数,
|
||||||
|
或新增 `CommitWithMedia`。选了后者。改签名会让每个调 `Commit` 的插件编译失败,
|
||||||
|
而那些插件根本不关心媒体。
|
||||||
|
|
||||||
|
### 2. 新增方法必须是「插件调用、内核实现」方向
|
||||||
|
|
||||||
|
这是**存量插件不需要重编**的技术原因:`IOInjector` 新增三个方法后,插件只是
|
||||||
|
*多了可以调的东西*,没有新的实现义务。反过来若在 `Plugin` 接口上加方法,
|
||||||
|
每个存量插件都会因未实现而编译失败。
|
||||||
|
|
||||||
|
因此 `SDKCompatibleVersion` 与 SDK 的 `CoreVersion` 都不必随之跃迁:
|
||||||
|
1.1.0 的 SDK 配 1.0.0 编的插件仍然成立。
|
||||||
|
|
||||||
|
### 3. 生成模板必须同步接线,否则是**全体外部插件编译失败**
|
||||||
|
|
||||||
|
公开接口加方法时,`tools/plugindev/templates/proc_main.go.tmpl` 里的 `procIO` /
|
||||||
|
`procDocMemory` 若不实现新方法,就不满足接口——**每个外部插件都编不过**,是硬失败
|
||||||
|
不是软降级。v1.1.1 这一层是被 `go test` 抓出来的(`internal/plugin/proc` 的两个
|
||||||
|
E2E 用例编译失败),不是靠人工检查发现的。
|
||||||
|
|
||||||
|
完整接线链共六处:`protocol.go` 的 method 常量 → `capability.go` 的能力归属 →
|
||||||
|
`corehandler.go` 的分派分支 → `proc_core.go` 的委托 → `proc_main.go.tmpl` 的模板实现 →
|
||||||
|
测试替身(`fakeCoreSDK`、`injectCapture`、`capability_test.go` 的手工方法清单)。
|
||||||
|
还要同步 `yaegi/mocksdk`——它没有任何代码对着编译,所以漂移不会被编译器抓到
|
||||||
|
(v1.1.1 修的时候发现它的 `Triple` 用的是 `Predicate`,而公开 SDK 一直叫 `Relation`)。
|
||||||
|
|
||||||
|
### 验证方式(取代「diff 为零」)
|
||||||
|
|
||||||
|
| 检查 | 命令 | v1.1.1 结果 |
|
||||||
|
|---|---|---|
|
||||||
|
| 存量插件源码零改动 | `cd example/<n> && go vet ./...`(17 个) | ✅ 17/17 通过 |
|
||||||
|
| 旧产物仍能建链 | 用 SDK 0.9.2 编的 `plugin.bin` 跑 `TestRealPlugin_*` | ✅ 4/4 通过(握手校验 `ProtocolVersion=1`,不是 SDK 版本) |
|
||||||
|
| 模板已接线 | `cd tools/plugindev && go test ./...` | ✅ `TestProcTemplate_CoversAllCoreMethods` 含新 method |
|
||||||
|
| 并发安全 | `go test ./sdk/ -race -count=5` | ✅ 零 DATA RACE(13 例压测) |
|
||||||
|
|
||||||
|
### v1.2.x 的接口扩展(2026-09-12)
|
||||||
|
|
||||||
|
1.2.0 把「记不记入记忆 / 要不要据此裁剪上下文」从**只有工具与通道能声明**,扩到**注入侧也能声明**:
|
||||||
|
|
||||||
|
| 新增 | 方向 | 说明 |
|
||||||
|
|---|---|---|
|
||||||
|
| `InjectOptions{NoMemory, ContextPolicy, CleanerName}` | 新增类型 | 单次注入的行为声明 |
|
||||||
|
| `ContextPolicyNone` / `ContextPolicyPrune` + `ValidContextPolicy` | 新增常量/函数 | 取值只有 `""` / `none` / `prune`;`prune` 必须显式声明 |
|
||||||
|
| 六个 `*Opts` 变体(Text / InterruptText / InputSync / InputMedia / InputMediaSync / InterruptMedia) | 插件调用、内核实现 | 旧的三参数方法保留为**零值糖**,与 `InjectOptions{}` 逐键等价 |
|
||||||
|
| `ChannelDef.ContextPolicy` + `ChannelDef` 的 JSON tag | 结构体字段 | 通道也可声明裁剪;补 tag 是因为通道定义要跨进程传给内核,而 `Cleaner` 是函数必须忽略——无 tag 时新增字段会被**静默丢掉** |
|
||||||
|
|
||||||
|
签名层面零变更(六个方法全是新增),满足第 1、2 条。
|
||||||
|
|
||||||
|
**但「接口纯追加」不等于「无需重编」**:1.2.0 同时把插件运行协议升到 2
|
||||||
|
(fd3 布局改变,不支持滚动升级),`ProtocolVersion` 不匹配会在握手时被明确拒绝
|
||||||
|
并提示用配套 plugindev 重编。两件事必须分开说,否则会被误读成「既然纯追加就还能用旧产物」。
|
||||||
|
|
||||||
|
#### 这次扩展自己抓出来的两处漂移(都是本节第 3 条要防的那类)
|
||||||
|
|
||||||
|
1. **模板接线守卫红了**:`TestProcTemplate_CoversAllCoreMethods` 要求模板出现内核提供的
|
||||||
|
每一个 method id,而注入标志位落地后模板不再发 `io.injectTextNoMem`(旧模板发它,
|
||||||
|
现在走 `io.injectText` + `NoMemory` 标志位)。内核保留该 id 是**刻意的向后兼容面**
|
||||||
|
(用那时模板编出的二进制仍在外面),不是漏接线——所以改的是判据:把它移入显式的
|
||||||
|
`deprecated` 表,并加**反向保护**(条目一旦重新出现在模板里就报错,避免这张表
|
||||||
|
退化成「永久豁免」的垃圾抽屉)。
|
||||||
|
2. **mocksdk 缺一个方法**:拿公共 SDK `IOInjector` 的 14 个方法名与 mock 的方法集
|
||||||
|
**机械求差**,差集恰好是旧的三参数 `InjectInputSync`——通道类插件(qq / a2a)完成
|
||||||
|
「入站 → agent 处理 → 回复取回」闭环要调的那个。`git log -S` 证实它**从来就缺**,
|
||||||
|
不是本次引入;补齐后差集为空。(上次漂的是 `Triple.Predicate` vs `Relation`,同一类问题。)
|
||||||
|
|
||||||
|
#### 验证(1.2.0,本机实测)
|
||||||
|
|
||||||
|
| 检查 | 命令 | 结果 |
|
||||||
|
|---|---|---|
|
||||||
|
| 存量插件源码零改动 | 逐个 `cd example/<n> && go vet ./...` | ✅ 17/17 通过(`luademo` 是 Lua、无 `go.mod`,跳过) |
|
||||||
|
| 模板已接线 | `cd tools/plugindev && go test ./...` | ✅ 全绿(修复前为红;反向保护另用「把 id 塞回模板」验证过会报错) |
|
||||||
|
| 并发安全 | `go test -race -count=5 ./sdk/` | ✅ ok |
|
||||||
|
| mocksdk 未漂移 | 方法集求差(14 个方法) | ✅ 差集为空 |
|
||||||
|
|
||||||
|
### 为何媒体块走 JSON 而不是共享段二进制通道
|
||||||
|
|
||||||
|
`SetToolBlocks` 的原设计是「二进制落 arena,Slice 描述符回传」。实际落地时改走 JSON:
|
||||||
|
data URL 本身已是 base64 文本,包进二进制传输省不了空间,还要让这四个 method 跟其余
|
||||||
|
51 个分道扬镳。共享段的价值在于**并发改写同一份状态**(StageContext 的 lost update),
|
||||||
|
而媒体块是单向传递的不可变数据,没有这个问题。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
## 八、关联文档
|
## 八、关联文档
|
||||||
|
|
||||||
- `docs/zh/架构迁移评估.md` — 完整论证(§3.2 method id 平移、§3.3 数据面、§3.4 SDK 封装、§3.5 回调型资源、§3.8 能力对齐)
|
- `docs/zh/架构迁移评估.md` — 完整论证(§3.2 method id 平移、§3.3 数据面、§3.4 SDK 封装、§3.5 回调型资源、§3.8 能力对齐)
|
||||||
|
|||||||
3
go.mod
3
go.mod
@ -18,6 +18,7 @@ require (
|
|||||||
github.com/charmbracelet/bubbletea v1.3.10
|
github.com/charmbracelet/bubbletea v1.3.10
|
||||||
github.com/charmbracelet/lipgloss v1.1.0
|
github.com/charmbracelet/lipgloss v1.1.0
|
||||||
golang.org/x/sys v0.38.0
|
golang.org/x/sys v0.38.0
|
||||||
|
golang.org/x/text v0.3.8
|
||||||
)
|
)
|
||||||
|
|
||||||
require (
|
require (
|
||||||
@ -40,8 +41,6 @@ require (
|
|||||||
github.com/muesli/termenv v0.16.0 // indirect
|
github.com/muesli/termenv v0.16.0 // indirect
|
||||||
github.com/rivo/uniseg v0.4.7 // indirect
|
github.com/rivo/uniseg v0.4.7 // indirect
|
||||||
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
|
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
|
||||||
golang.org/x/text v0.3.8 // indirect
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
replace gitcode.com/JianFeeeee/homeagent-sdk => ./third_party/homeagent-sdk
|
replace gitcode.com/JianFeeeee/homeagent-sdk => ./third_party/homeagent-sdk
|
||||||
|
|||||||
@ -235,6 +235,31 @@ type RoutableProvider interface {
|
|||||||
Priority() int // AUTO 跨源选择的优先级,大者优先
|
Priority() int // AUTO 跨源选择的优先级,大者优先
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ModalProvider 声明自身的多模态能力。单独抽接口而不合进 Provider:
|
||||||
|
// 第三方 Provider 实现无需改动,未实现时按纯文本处理(保守侧)。
|
||||||
|
type ModalProvider interface {
|
||||||
|
SupportsVision() bool
|
||||||
|
SupportsAudio() bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// ProviderSupportsVision 安全判定任意 Provider 能否看图。
|
||||||
|
// 未实现 ModalProvider 的一律返回 false:宁可多走一次文字回退,
|
||||||
|
// 也不能把图默默扔给一个会把它剥掉的上游。
|
||||||
|
func ProviderSupportsVision(p Provider) bool {
|
||||||
|
if mp, ok := p.(ModalProvider); ok {
|
||||||
|
return mp.SupportsVision()
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// ProviderSupportsAudio 安全判定任意 Provider 能否听音频。
|
||||||
|
func ProviderSupportsAudio(p Provider) bool {
|
||||||
|
if mp, ok := p.(ModalProvider); ok {
|
||||||
|
return mp.SupportsAudio()
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
// ModelContextWindow 返回模型的最大上下文窗口(token 数)
|
// ModelContextWindow 返回模型的最大上下文窗口(token 数)
|
||||||
// 标称窗口 ≠ 有效窗口:接近满时注意力涣散,调用方应取 70-80% 为目标利用率
|
// 标称窗口 ≠ 有效窗口:接近满时注意力涣散,调用方应取 70-80% 为目标利用率
|
||||||
func ModelContextWindow(model string) int {
|
func ModelContextWindow(model string) int {
|
||||||
@ -284,6 +309,11 @@ type BaseConfig struct {
|
|||||||
ContextWindow int `json:"context_window"`
|
ContextWindow int `json:"context_window"`
|
||||||
MaxConcurrent int `json:"max_concurrent"`
|
MaxConcurrent int `json:"max_concurrent"`
|
||||||
Priority int `json:"priority"`
|
Priority int `json:"priority"`
|
||||||
|
|
||||||
|
// Vision/Audio 声明这条链路能否真正处理多模态内容块。
|
||||||
|
// 网关可能静默剥离 image_url 后仍返回 200,所以不能从响应推断能力。
|
||||||
|
Vision bool `json:"vision"`
|
||||||
|
Audio bool `json:"audio"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// LuaAdaptedProvider 使用 Lua 脚本做请求/响应变换,直接发起 HTTP 调用
|
// LuaAdaptedProvider 使用 Lua 脚本做请求/响应变换,直接发起 HTTP 调用
|
||||||
@ -343,6 +373,11 @@ func (p *LuaAdaptedProvider) Name() string { return p.name }
|
|||||||
func (p *LuaAdaptedProvider) Model() string { return p.cfg.Model }
|
func (p *LuaAdaptedProvider) Model() string { return p.cfg.Model }
|
||||||
func (p *LuaAdaptedProvider) Priority() int { return p.cfg.Priority }
|
func (p *LuaAdaptedProvider) Priority() int { return p.cfg.Priority }
|
||||||
|
|
||||||
|
// SupportsVision/SupportsAudio 实现 ModalProvider,值来自部署时声明
|
||||||
|
// (core.llm.sources.<name>.vision / .audio)。
|
||||||
|
func (p *LuaAdaptedProvider) SupportsVision() bool { return p.cfg.Vision }
|
||||||
|
func (p *LuaAdaptedProvider) SupportsAudio() bool { return p.cfg.Audio }
|
||||||
|
|
||||||
func (p *LuaAdaptedProvider) Chat(ctx context.Context, req *CompletionRequest) (*CompletionResponse, error) {
|
func (p *LuaAdaptedProvider) Chat(ctx context.Context, req *CompletionRequest) (*CompletionResponse, error) {
|
||||||
if p.cfg.Model != "" && (req.Model == "" || req.Model == "AUTO") {
|
if p.cfg.Model != "" && (req.Model == "" || req.Model == "AUTO") {
|
||||||
req.Model = p.cfg.Model
|
req.Model = p.cfg.Model
|
||||||
|
|||||||
@ -14,8 +14,10 @@ import (
|
|||||||
"gitcode.com/JianFeeeee/HomeAgent/internal/knowledge"
|
"gitcode.com/JianFeeeee/HomeAgent/internal/knowledge"
|
||||||
"gitcode.com/JianFeeeee/HomeAgent/internal/memory"
|
"gitcode.com/JianFeeeee/HomeAgent/internal/memory"
|
||||||
"gitcode.com/JianFeeeee/HomeAgent/internal/memory/document"
|
"gitcode.com/JianFeeeee/HomeAgent/internal/memory/document"
|
||||||
|
"gitcode.com/JianFeeeee/HomeAgent/internal/memory/media"
|
||||||
"gitcode.com/JianFeeeee/HomeAgent/internal/memory/social"
|
"gitcode.com/JianFeeeee/HomeAgent/internal/memory/social"
|
||||||
"gitcode.com/JianFeeeee/HomeAgent/internal/memory/text"
|
"gitcode.com/JianFeeeee/HomeAgent/internal/memory/text"
|
||||||
|
"gitcode.com/JianFeeeee/HomeAgent/internal/memory/vector"
|
||||||
"gitcode.com/JianFeeeee/HomeAgent/internal/plugin"
|
"gitcode.com/JianFeeeee/HomeAgent/internal/plugin"
|
||||||
"gitcode.com/JianFeeeee/HomeAgent/internal/tracker"
|
"gitcode.com/JianFeeeee/HomeAgent/internal/tracker"
|
||||||
"gitcode.com/JianFeeeee/HomeAgent/pkg/types"
|
"gitcode.com/JianFeeeee/HomeAgent/pkg/types"
|
||||||
@ -50,6 +52,11 @@ type Agent struct {
|
|||||||
// 文本记忆(原始对话日志)
|
// 文本记忆(原始对话日志)
|
||||||
textMem *text.Memory
|
textMem *text.Memory
|
||||||
|
|
||||||
|
// 媒体存储(内容寻址):对话里出现的图片/音频按 sha256 落盘去重。
|
||||||
|
// 它是记忆块的内容存储,不单独做生命周期管理:块的创建/迁移/删除
|
||||||
|
// 由记忆系统本身决定。为 nil 时全部媒体接线静默跳过。
|
||||||
|
mediaStore *media.Store
|
||||||
|
|
||||||
// 人格设定
|
// 人格设定
|
||||||
personality *agentPkg.Personality
|
personality *agentPkg.Personality
|
||||||
|
|
||||||
@ -83,10 +90,17 @@ type Agent struct {
|
|||||||
selfInputCh chan selfInputMsg
|
selfInputCh chan selfInputMsg
|
||||||
|
|
||||||
// 子任务异步执行
|
// 子任务异步执行
|
||||||
childMu sync.Mutex
|
childMu sync.Mutex
|
||||||
childNextID int64
|
childNextID int64
|
||||||
childResults map[string]string
|
// childTasks 记录子任务状态:运行中 / 结果 / 是否已交付。
|
||||||
childRunning map[string]bool // 运行中的子任务(child_result 查询时区分'运行中'与'不存在')
|
//
|
||||||
|
// 为什么保留结果而不是“读到即删”:完成通知会写进持久上下文
|
||||||
|
// (formatMergedTimeline 每轮都重新注入),模型之后还会再查。若读到即删,
|
||||||
|
// 第二次查询就得到“不存在或已过期”这个**永久失败信号**——模型据此认为
|
||||||
|
// 任务未完成,会无限重试/汇报(实测单轮 35 次工具调用、持续 514 秒)。
|
||||||
|
childTasks map[string]*childTaskState
|
||||||
|
// childSeq 给完成的任务排个序,用于有界淘汰。
|
||||||
|
childSeq int64
|
||||||
|
|
||||||
// 高优先级打断通道:interceptLoop 注入,process() 在工具循环轮次间非阻塞读取
|
// 高优先级打断通道:interceptLoop 注入,process() 在工具循环轮次间非阻塞读取
|
||||||
interceptCh chan *agentIO.InputEvent
|
interceptCh chan *agentIO.InputEvent
|
||||||
@ -106,6 +120,13 @@ type Agent struct {
|
|||||||
// 当前轮次的非文本媒体数据(图片/音频),供 describe_image 等工具访问
|
// 当前轮次的非文本媒体数据(图片/音频),供 describe_image 等工具访问
|
||||||
pendingMedia map[string]interface{}
|
pendingMedia map[string]interface{}
|
||||||
|
|
||||||
|
// pendingMediaDigests 累积本轮已落进 CAS 的媒体 digest。
|
||||||
|
//
|
||||||
|
// 需要缓存而不是当场挂到事件上:媒体在 process() 执行期间被捕获,
|
||||||
|
// 而承载它的 ContextEvent 要等 process() 返回后才 Append——此刻还没有 owner_id。
|
||||||
|
// 与 pendingMedia 同受 a.mu 保护。
|
||||||
|
pendingMediaDigests []string
|
||||||
|
|
||||||
// 当前输入是否为工具提醒/中断(以 system 角色注入,避免被当成用户消息)
|
// 当前输入是否为工具提醒/中断(以 system 角色注入,避免被当成用户消息)
|
||||||
interruptInput bool
|
interruptInput bool
|
||||||
|
|
||||||
@ -119,12 +140,19 @@ type Agent struct {
|
|||||||
|
|
||||||
// 输入去重:防 webui/GUI 断线重连导致的消息重放
|
// 输入去重:防 webui/GUI 断线重连导致的消息重放
|
||||||
// key=source+"|"+content, value=上次接收时间;短窗口内同内容丢弃
|
// key=source+"|"+content, value=上次接收时间;短窗口内同内容丢弃
|
||||||
lastInput map[string]time.Time
|
lastInput map[string]time.Time
|
||||||
lastInputMu sync.Mutex
|
lastInputMu sync.Mutex
|
||||||
|
|
||||||
// 词嵌入模型,用于实体语义相似度计算
|
// 词嵌入模型,用于实体语义相似度计算
|
||||||
embedder *memory.StaticEmbedder
|
embedder *memory.StaticEmbedder
|
||||||
|
|
||||||
|
// multimodalSpace 是统一多模态向量空间(可选)。实现可以是内嵌 ONNX,
|
||||||
|
// 也可以是外部 API 客户端;两者共享同一套 L0/L2/L3 向量缓存与检索基础设施。
|
||||||
|
multimodalSpace vector.MultimodalEmbedder
|
||||||
|
|
||||||
|
// fusionCfg 控制文本路与视觉路的跨模态融合权重,可按模型实测结果配置。
|
||||||
|
fusionCfg CrossModalFusionConfig
|
||||||
|
|
||||||
// 技能索引提供者:由 skillmgr 插件实现,向 system prompt 注入轻量技能索引
|
// 技能索引提供者:由 skillmgr 插件实现,向 system prompt 注入轻量技能索引
|
||||||
skillIndex SkillIndexProvider
|
skillIndex SkillIndexProvider
|
||||||
}
|
}
|
||||||
@ -151,16 +179,19 @@ type AgentConfig struct {
|
|||||||
Knowledge *knowledge.Store
|
Knowledge *knowledge.Store
|
||||||
SocialStore *social.SocialStore
|
SocialStore *social.SocialStore
|
||||||
TextMemory *text.Memory
|
TextMemory *text.Memory
|
||||||
|
MediaStore *media.Store
|
||||||
|
MultimodalSpace vector.MultimodalEmbedder
|
||||||
|
FusionCfg CrossModalFusionConfig // 跨模态融合权重;零值用默认
|
||||||
Personality *agentPkg.Personality
|
Personality *agentPkg.Personality
|
||||||
PluginReg *plugin.Registry
|
PluginReg *plugin.Registry
|
||||||
PluginDir string
|
PluginDir string
|
||||||
DistillInterval time.Duration
|
DistillInterval time.Duration
|
||||||
ArchiveInterval time.Duration // 冷文档归档间隔(L2→L3),0 则使用 DistillInterval
|
ArchiveInterval time.Duration // 冷文档归档间隔(L2→L3),0 则使用 DistillInterval
|
||||||
ReviewInterval time.Duration // 关系复审间隔,0 则使用 DistillInterval
|
ReviewInterval time.Duration // 关系复审间隔,0 则使用 DistillInterval
|
||||||
MergeInterval time.Duration // 实体合并检测间隔,0 则使用 DistillInterval
|
MergeInterval time.Duration // 实体合并检测间隔,0 则使用 DistillInterval
|
||||||
MaxContextSize int // 活跃上下文最大条数,超出按相关性裁剪
|
MaxContextSize int // 活跃上下文最大条数,超出按相关性裁剪
|
||||||
ContextSavePath string // 上下文持久化路径,空则不持久化
|
ContextSavePath string // 上下文持久化路径,空则不持久化
|
||||||
EmbeddingModelPath string // 预训练词嵌入模型路径(word2vec 文本格式),空则不使用
|
EmbeddingModelPath string // 预训练词嵌入模型路径(word2vec 文本格式),空则不使用
|
||||||
Embedder *memory.StaticEmbedder // 共享词嵌入实例;nil 时按 EmbeddingModelPath 自建
|
Embedder *memory.StaticEmbedder // 共享词嵌入实例;nil 时按 EmbeddingModelPath 自建
|
||||||
StageHost *StageHost
|
StageHost *StageHost
|
||||||
EventBus *events.Bus
|
EventBus *events.Bus
|
||||||
@ -194,8 +225,7 @@ func New(cfg AgentConfig) *Agent {
|
|||||||
embedder = memory.NewStaticEmbedder(strings.Split(cfg.EmbeddingModelPath, ",")...)
|
embedder = memory.NewStaticEmbedder(strings.Split(cfg.EmbeddingModelPath, ",")...)
|
||||||
}
|
}
|
||||||
if cfg.DocStore != nil {
|
if cfg.DocStore != nil {
|
||||||
cfg.DocStore.SetVectorizer(embedder)
|
// TF-IDF 内置为 fallback,无需外部注入
|
||||||
cfg.DocStore.ReindexWithVectorizer(embedder)
|
|
||||||
}
|
}
|
||||||
if cfg.Knowledge != nil {
|
if cfg.Knowledge != nil {
|
||||||
cfg.Knowledge.SetVectorizer(embedder)
|
cfg.Knowledge.SetVectorizer(embedder)
|
||||||
@ -209,6 +239,16 @@ func New(cfg AgentConfig) *Agent {
|
|||||||
if cfg.IO != nil {
|
if cfg.IO != nil {
|
||||||
rc.SetChannelDefLookup(cfg.IO.GetInputChannelDef)
|
rc.SetChannelDefLookup(cfg.IO.GetInputChannelDef)
|
||||||
}
|
}
|
||||||
|
// 注入稠密多模态向量空间(可选):配置后文档检索、L0 相关性裁剪、
|
||||||
|
// 跨模态检索全部共享同一向量空间,取代稀疏 fastText 语义路。
|
||||||
|
// 未配置时退化到 TF-IDF/fastText 稀疏检索,保持既有行为。
|
||||||
|
if cfg.MultimodalSpace != nil && cfg.MultimodalSpace.Loaded() {
|
||||||
|
rc.SetDenseSpace(cfg.MultimodalSpace)
|
||||||
|
if cfg.DocStore != nil {
|
||||||
|
cfg.DocStore.SetDenseSpace(cfg.MultimodalSpace)
|
||||||
|
cfg.DocStore.BuildDenseIndex(cfg.MultimodalSpace)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
return &Agent{
|
return &Agent{
|
||||||
id: cfg.ID,
|
id: cfg.ID,
|
||||||
@ -227,6 +267,7 @@ func New(cfg AgentConfig) *Agent {
|
|||||||
knowledge: cfg.Knowledge,
|
knowledge: cfg.Knowledge,
|
||||||
social: cfg.SocialStore,
|
social: cfg.SocialStore,
|
||||||
textMem: cfg.TextMemory,
|
textMem: cfg.TextMemory,
|
||||||
|
mediaStore: cfg.MediaStore,
|
||||||
personality: cfg.Personality,
|
personality: cfg.Personality,
|
||||||
pluginReg: cfg.PluginReg,
|
pluginReg: cfg.PluginReg,
|
||||||
pluginDir: cfg.PluginDir,
|
pluginDir: cfg.PluginDir,
|
||||||
@ -239,13 +280,14 @@ func New(cfg AgentConfig) *Agent {
|
|||||||
skillIndex: cfg.SkillIndexProvider,
|
skillIndex: cfg.SkillIndexProvider,
|
||||||
eventBus: cfg.EventBus,
|
eventBus: cfg.EventBus,
|
||||||
selfInputCh: make(chan selfInputMsg, 64),
|
selfInputCh: make(chan selfInputMsg, 64),
|
||||||
childResults: make(map[string]string),
|
childTasks: make(map[string]*childTaskState),
|
||||||
childRunning: make(map[string]bool),
|
|
||||||
interceptCh: make(chan *agentIO.InputEvent, 64),
|
interceptCh: make(chan *agentIO.InputEvent, 64),
|
||||||
pluginHealth: newPluginHealthTracker(),
|
pluginHealth: newPluginHealthTracker(),
|
||||||
thinkingEnabled: cfg.ThinkingEnabled,
|
thinkingEnabled: cfg.ThinkingEnabled,
|
||||||
inputCfg: cfg.InputProcessing,
|
inputCfg: cfg.InputProcessing,
|
||||||
embedder: embedder,
|
embedder: embedder,
|
||||||
|
multimodalSpace: cfg.MultimodalSpace,
|
||||||
|
fusionCfg: cfg.FusionCfg,
|
||||||
noMergeMarkers: make(map[string]int),
|
noMergeMarkers: make(map[string]int),
|
||||||
lastInput: make(map[string]time.Time),
|
lastInput: make(map[string]time.Time),
|
||||||
}
|
}
|
||||||
@ -261,6 +303,8 @@ func (a *Agent) Start() {
|
|||||||
go a.archiveLoop()
|
go a.archiveLoop()
|
||||||
go a.mergeLoop()
|
go a.mergeLoop()
|
||||||
go a.reviewLoop()
|
go a.reviewLoop()
|
||||||
|
a.reembedStaleMedia()
|
||||||
|
a.migrateLegacyGraphMedia()
|
||||||
log.Printf("[agent] %s started, waiting for IO interrupts", a.id)
|
log.Printf("[agent] %s started, waiting for IO interrupts", a.id)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -15,14 +15,14 @@ type mockOutputDevice struct {
|
|||||||
toolFn func(string, map[string]interface{}) (interface{}, error)
|
toolFn func(string, map[string]interface{}) (interface{}, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (d *mockOutputDevice) Name() string { return d.name }
|
func (d *mockOutputDevice) Name() string { return d.name }
|
||||||
func (d *mockOutputDevice) Type() agentIO.DeviceType { return agentIO.DeviceOutput }
|
func (d *mockOutputDevice) Type() agentIO.DeviceType { return agentIO.DeviceOutput }
|
||||||
func (d *mockOutputDevice) Description() string { return "mock " + d.name }
|
func (d *mockOutputDevice) Description() string { return "mock " + d.name }
|
||||||
func (d *mockOutputDevice) Tools() []agentIO.ToolDef { return d.tools }
|
func (d *mockOutputDevice) Tools() []agentIO.ToolDef { return d.tools }
|
||||||
func (d *mockOutputDevice) Start() error { return nil }
|
func (d *mockOutputDevice) Start() error { return nil }
|
||||||
func (d *mockOutputDevice) Stop() error { return nil }
|
func (d *mockOutputDevice) Stop() error { return nil }
|
||||||
func (d *mockOutputDevice) OutputCapabilities() agentIO.OutputCapability { return d.caps }
|
func (d *mockOutputDevice) OutputCapabilities() agentIO.OutputCapability { return d.caps }
|
||||||
func (d *mockOutputDevice) ChannelDef() agentIO.ChannelDef { return agentIO.ChannelDef{} }
|
func (d *mockOutputDevice) ChannelDef() agentIO.ChannelDef { return agentIO.ChannelDef{} }
|
||||||
func (d *mockOutputDevice) Execute(tool string, args map[string]interface{}) (interface{}, error) {
|
func (d *mockOutputDevice) Execute(tool string, args map[string]interface{}) (interface{}, error) {
|
||||||
if d.toolFn != nil {
|
if d.toolFn != nil {
|
||||||
return d.toolFn(tool, args)
|
return d.toolFn(tool, args)
|
||||||
@ -67,8 +67,8 @@ func TestExecuteOutputSendTool(t *testing.T) {
|
|||||||
"type": "text",
|
"type": "text",
|
||||||
}}
|
}}
|
||||||
result := a.executeOutputSendTool(tc)
|
result := a.executeOutputSendTool(tc)
|
||||||
if !strings.Contains(result, "screen") {
|
if result != "ok" {
|
||||||
t.Errorf("unexpected result: %s", result)
|
t.Errorf("expected ok, got: %s", result)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -22,13 +22,23 @@ type ToolResultItem struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type ContextEvent struct {
|
type ContextEvent struct {
|
||||||
Timestamp time.Time `json:"timestamp"`
|
// ID 是事件的稳定标识。惰性生成:只有真的要挂媒体块时才赋值。
|
||||||
Source string `json:"source"`
|
//
|
||||||
Input string `json:"input"`
|
// 全量生成会让每条事件都多一个字段进 context.json,而绝大多数对话没有媒体。
|
||||||
Response string `json:"response,omitempty"`
|
// omitempty 保证存量 context.json 读回来时该字段为空,不影响任何既有行为。
|
||||||
ToolsUsed []string `json:"tools_used,omitempty"`
|
ID string `json:"id,omitempty"`
|
||||||
ToolResults []ToolResultItem `json:"tool_results,omitempty"`
|
Timestamp time.Time `json:"timestamp"`
|
||||||
Vector vector.Vector `json:"-"`
|
Source string `json:"source"`
|
||||||
|
Input string `json:"input"`
|
||||||
|
Response string `json:"response,omitempty"`
|
||||||
|
ToolsUsed []string `json:"tools_used,omitempty"`
|
||||||
|
ToolResults []ToolResultItem `json:"tool_results,omitempty"`
|
||||||
|
// --- 原生多模态记忆 ---
|
||||||
|
// 一等记忆块:块本身随事件在层间迁移,身份不变,不建引用计数。
|
||||||
|
Blocks []memory.MemoryBlock `json:"blocks,omitempty"` // 一等记忆块(text/image/video/audio)
|
||||||
|
Vector vector.Vector `json:"-"` // 稀疏词向量(TF-IDF/fastText 空间)
|
||||||
|
DenseVec []float64 `json:"-"` // 稠密多模态向量(与媒体/文档共享空间)
|
||||||
|
DenseFP string `json:"-"` // DenseVec 所属统一空间指纹(缓存字段,不持久化)
|
||||||
}
|
}
|
||||||
|
|
||||||
const contextFlushInterval = 5 * time.Second
|
const contextFlushInterval = 5 * time.Second
|
||||||
@ -37,6 +47,7 @@ type RelevanceContext struct {
|
|||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
events []*ContextEvent
|
events []*ContextEvent
|
||||||
embedder *memory.StaticEmbedder
|
embedder *memory.StaticEmbedder
|
||||||
|
denseSpace vector.MultimodalEmbedder
|
||||||
savePath string
|
savePath string
|
||||||
saveTimer *time.Timer
|
saveTimer *time.Timer
|
||||||
dirty bool
|
dirty bool
|
||||||
@ -55,6 +66,14 @@ func NewRelevanceContext(savePath string, embedder *memory.StaticEmbedder) *Rele
|
|||||||
return rc
|
return rc
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SetDenseSpace 注入稠密多模态向量空间。配置后 L0 相关性裁剪可用稠密向量
|
||||||
|
// 余弦(与媒体检索、文档检索共享同一空间),未配置时退化到稀疏词向量。
|
||||||
|
func (c *RelevanceContext) SetDenseSpace(ds vector.MultimodalEmbedder) {
|
||||||
|
c.mu.Lock()
|
||||||
|
defer c.mu.Unlock()
|
||||||
|
c.denseSpace = ds
|
||||||
|
}
|
||||||
|
|
||||||
func (c *RelevanceContext) SetToolDefLookup(fn func(name string) *sdk.ToolDef) {
|
func (c *RelevanceContext) SetToolDefLookup(fn func(name string) *sdk.ToolDef) {
|
||||||
c.mu.Lock()
|
c.mu.Lock()
|
||||||
defer c.mu.Unlock()
|
defer c.mu.Unlock()
|
||||||
@ -77,7 +96,7 @@ func (c *RelevanceContext) load() {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
for _, evt := range events {
|
for _, evt := range events {
|
||||||
evt.Vector = c.computeVector(evt)
|
c.computeVector(evt)
|
||||||
}
|
}
|
||||||
c.events = events
|
c.events = events
|
||||||
}
|
}
|
||||||
@ -176,12 +195,32 @@ func (c *RelevanceContext) channelCleanerForDoc() document.ChannelCleaner {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *RelevanceContext) computeVector(evt *ContextEvent) vector.Vector {
|
func (c *RelevanceContext) computeVector(evt *ContextEvent) {
|
||||||
text := textForVector(evt, c.toolDefLookup, c.channelDefLookup)
|
text := textForVector(evt, c.toolDefLookup, c.channelDefLookup)
|
||||||
if text == "" {
|
// 稀疏向量始终计算(TF-IDF/fastText,退化时仍可用)
|
||||||
return nil
|
if text != "" {
|
||||||
|
evt.Vector = c.embedder.Vectorize(text)
|
||||||
|
}
|
||||||
|
// 稠密向量:文本向量 ⊕ 本事件持有的一等记忆块媒体向量(同一统一空间)。
|
||||||
|
// 只有媒体的输入(无文本)也要有可比较的坐标,因此不再按 text=="" 提前返回。
|
||||||
|
if c.denseSpace != nil && c.denseSpace.Loaded() {
|
||||||
|
fp := c.denseSpace.Fingerprint()
|
||||||
|
var parts [][]float64
|
||||||
|
if text != "" {
|
||||||
|
if dv, err := c.denseSpace.VectorizeDense(text); err == nil && len(dv) > 0 {
|
||||||
|
parts = append(parts, dv)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, b := range evt.Blocks {
|
||||||
|
// 只融合同指纹的块向量:另一套坐标系的向量混进来会算出
|
||||||
|
// 两边都不像的方向。
|
||||||
|
if len(b.Vector) > 0 && b.Fingerprint == fp {
|
||||||
|
parts = append(parts, b.Vector)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
evt.DenseVec = vector.FuseVectors(parts...)
|
||||||
|
evt.DenseFP = fp
|
||||||
}
|
}
|
||||||
return c.embedder.Vectorize(text)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *RelevanceContext) Save() error {
|
func (c *RelevanceContext) Save() error {
|
||||||
@ -202,7 +241,7 @@ func (c *RelevanceContext) Append(evt ContextEvent) {
|
|||||||
c.mu.Lock()
|
c.mu.Lock()
|
||||||
defer c.mu.Unlock()
|
defer c.mu.Unlock()
|
||||||
|
|
||||||
evt.Vector = c.computeVector(&evt)
|
c.computeVector(&evt)
|
||||||
c.events = append(c.events, &evt)
|
c.events = append(c.events, &evt)
|
||||||
|
|
||||||
c.save()
|
c.save()
|
||||||
@ -212,7 +251,7 @@ func (c *RelevanceContext) InsertByTimestamp(evt ContextEvent) {
|
|||||||
c.mu.Lock()
|
c.mu.Lock()
|
||||||
defer c.mu.Unlock()
|
defer c.mu.Unlock()
|
||||||
|
|
||||||
evt.Vector = c.computeVector(&evt)
|
c.computeVector(&evt)
|
||||||
|
|
||||||
idx := sort.Search(len(c.events), func(i int) bool {
|
idx := sort.Search(len(c.events), func(i int) bool {
|
||||||
return c.events[i].Timestamp.After(evt.Timestamp)
|
return c.events[i].Timestamp.After(evt.Timestamp)
|
||||||
@ -256,6 +295,15 @@ func (c *RelevanceContext) flush() {
|
|||||||
c.dirty = false
|
c.dirty = false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// scoredEvent 是 Prune 里按相关度排序的事件。
|
||||||
|
//
|
||||||
|
// 提为包级类型:Prune 需要把待归档列表传给后续处理。
|
||||||
|
type scoredEvent struct {
|
||||||
|
event *ContextEvent
|
||||||
|
score float64
|
||||||
|
idx int
|
||||||
|
}
|
||||||
|
|
||||||
func (c *RelevanceContext) Prune(currentInput string, topK int, docStore *document.Store) int {
|
func (c *RelevanceContext) Prune(currentInput string, topK int, docStore *document.Store) int {
|
||||||
c.mu.Lock()
|
c.mu.Lock()
|
||||||
defer c.mu.Unlock()
|
defer c.mu.Unlock()
|
||||||
@ -275,17 +323,30 @@ func (c *RelevanceContext) Prune(currentInput string, topK int, docStore *docume
|
|||||||
return 0
|
return 0
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 优先使用稠密向量余弦(与媒体/文档共享空间);退化到稀疏词向量。
|
||||||
|
var queryDense []float64
|
||||||
|
useDense := false
|
||||||
|
queryFP := ""
|
||||||
|
if c.denseSpace != nil && c.denseSpace.Loaded() {
|
||||||
|
if dv, err := c.denseSpace.VectorizeDense(currentInput); err == nil {
|
||||||
|
queryDense = dv
|
||||||
|
queryFP = c.denseSpace.Fingerprint()
|
||||||
|
useDense = true
|
||||||
|
}
|
||||||
|
}
|
||||||
queryVec := c.embedder.VectorizeClean(currentInput)
|
queryVec := c.embedder.VectorizeClean(currentInput)
|
||||||
|
|
||||||
type scored struct {
|
scoredEvents := make([]scoredEvent, len(candidates))
|
||||||
event *ContextEvent
|
|
||||||
score float64
|
|
||||||
idx int
|
|
||||||
}
|
|
||||||
scoredEvents := make([]scored, len(candidates))
|
|
||||||
for i, evt := range candidates {
|
for i, evt := range candidates {
|
||||||
score := vector.CosineSimilarity(queryVec, evt.Vector)
|
var score float64
|
||||||
scoredEvents[i] = scored{event: evt, score: score, idx: i}
|
// 只在同一统一空间内比稠密余弦:换了模型/维度后旧事件的向量
|
||||||
|
// 属于另一个坐标系,拿来比会得到无意义的分数。
|
||||||
|
if useDense && evt.DenseFP == queryFP && len(evt.DenseVec) == len(queryDense) {
|
||||||
|
score = vector.DenseCosine(queryDense, evt.DenseVec)
|
||||||
|
} else {
|
||||||
|
score = vector.CosineSimilarity(queryVec, evt.Vector)
|
||||||
|
}
|
||||||
|
scoredEvents[i] = scoredEvent{event: evt, score: score, idx: i}
|
||||||
}
|
}
|
||||||
|
|
||||||
sort.Slice(scoredEvents, func(i, j int) bool {
|
sort.Slice(scoredEvents, func(i, j int) bool {
|
||||||
@ -322,11 +383,20 @@ func (c *RelevanceContext) Prune(currentInput string, topK int, docStore *docume
|
|||||||
Content: s.event.Input,
|
Content: s.event.Input,
|
||||||
Response: s.event.Response,
|
Response: s.event.Response,
|
||||||
ToolResults: convertToolResults(s.event.ToolResults),
|
ToolResults: convertToolResults(s.event.ToolResults),
|
||||||
|
Blocks: append([]memory.MemoryBlock(nil), s.event.Blocks...),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
doc, err := docStore.ContextToDoc("context_archived", entries, c.embedder, nil, c.toolOutputClean, c.channelCleanerForDoc())
|
doc, err := docStore.ContextToDoc("context_archived", entries, c.embedder, nil, c.toolOutputClean, c.channelCleanerForDoc())
|
||||||
if err == nil && doc != nil {
|
if err == nil && doc != nil {
|
||||||
archived = len(entries)
|
archived = len(entries)
|
||||||
|
// 一等记忆块的迁移:块随归档事件离开 L0、进入 L2。
|
||||||
|
// 迁移的是块本身(ID 不变、只换持有层),不是复制也不是保活引用;
|
||||||
|
// 因此归档后清空源事件的块,确保同一块不同时留在两层。
|
||||||
|
for _, s := range archive {
|
||||||
|
if s.event != nil {
|
||||||
|
s.event.Blocks = nil
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -369,6 +439,18 @@ func (c *RelevanceContext) Recent(n int) []ContextEvent {
|
|||||||
return result
|
return result
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Blocks 返回当前上下文持有的一等记忆块(供跨层存活判定)。
|
||||||
|
// 迁移后源事件已被清空,因此这里只会拿到真正属于 L0 的块。
|
||||||
|
func (c *RelevanceContext) Blocks() []memory.MemoryBlock {
|
||||||
|
c.mu.Lock()
|
||||||
|
defer c.mu.Unlock()
|
||||||
|
var out []memory.MemoryBlock
|
||||||
|
for _, e := range c.events {
|
||||||
|
out = append(out, e.Blocks...)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
func (c *RelevanceContext) Len() int {
|
func (c *RelevanceContext) Len() int {
|
||||||
c.mu.Lock()
|
c.mu.Lock()
|
||||||
defer c.mu.Unlock()
|
defer c.mu.Unlock()
|
||||||
@ -385,5 +467,3 @@ func convertToolResults(items []ToolResultItem) []document.ToolResultItem {
|
|||||||
}
|
}
|
||||||
return result
|
return result
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
268
internal/agent/core/crossmodal.go
Normal file
268
internal/agent/core/crossmodal.go
Normal file
@ -0,0 +1,268 @@
|
|||||||
|
package core
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"log"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitcode.com/JianFeeeee/HomeAgent/internal/memory/document"
|
||||||
|
"gitcode.com/JianFeeeee/HomeAgent/internal/memory/media"
|
||||||
|
)
|
||||||
|
|
||||||
|
// CrossModalHit 是跨模态检索融合后的一条候选。
|
||||||
|
//
|
||||||
|
// 统一的检索单元是记忆块而非 CAS 全库:媒体在 L0/L2/L3 都由层容器持有,
|
||||||
|
// 只有仍被某层记忆块持有的媒体才可召回。Doc 是 L2 文档;Media 是该块携带的
|
||||||
|
// 原生媒体坐标。两路分数尺度不同,融合前各自归一化,见 fuseCrossModal。
|
||||||
|
type CrossModalHit struct {
|
||||||
|
Doc *document.Doc // 文本路命中的文档;视觉路命中时为 nil
|
||||||
|
Media *media.Item // 视觉路命中的媒体;文本路命中时也可能带关联媒体
|
||||||
|
MediaScore float64 // 视觉路原始 cosine(无则 0)
|
||||||
|
DocScore float64 // 文本路原始 cosine(无则 0)
|
||||||
|
Fused float64 // 归一化加权融合分,供最终排序
|
||||||
|
// 该媒体同时被两路命中(文本路经文档关联、视觉路直接命中)时,
|
||||||
|
// DoubleHit=true —— 双信号确认,应排在只被一路命中的候选之前。
|
||||||
|
DoubleHit bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// CrossModalFusionConfig 控制文本路与视觉路的融合行为。
|
||||||
|
// 默认各路权重 0.5,双命中加权 0.15;不同模型/场景可按实测调整。
|
||||||
|
type CrossModalFusionConfig struct {
|
||||||
|
WeightText float64 // 文本路融合权重(默认 0.5)
|
||||||
|
WeightVisual float64 // 视觉路融合权重(默认 0.5)
|
||||||
|
DoubleHitBonus float64 // 双命中额外加分(默认 0.15)
|
||||||
|
MinMaxEps float64 // min-max 归一化除零保护(默认 1e-12)
|
||||||
|
}
|
||||||
|
|
||||||
|
var defaultFusionConfig = CrossModalFusionConfig{
|
||||||
|
WeightText: 0.5,
|
||||||
|
WeightVisual: 0.5,
|
||||||
|
DoubleHitBonus: 0.15,
|
||||||
|
MinMaxEps: 1e-12,
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c CrossModalFusionConfig) textWeight() float64 {
|
||||||
|
if c.WeightText <= 0 {
|
||||||
|
return defaultFusionConfig.WeightText
|
||||||
|
}
|
||||||
|
return c.WeightText
|
||||||
|
}
|
||||||
|
func (c CrossModalFusionConfig) visualWeight() float64 {
|
||||||
|
if c.WeightVisual <= 0 {
|
||||||
|
return defaultFusionConfig.WeightVisual
|
||||||
|
}
|
||||||
|
return c.WeightVisual
|
||||||
|
}
|
||||||
|
func (c CrossModalFusionConfig) doubleHitBonus() float64 {
|
||||||
|
return c.DoubleHitBonus
|
||||||
|
}
|
||||||
|
func (c CrossModalFusionConfig) minMaxEps() float64 {
|
||||||
|
if c.MinMaxEps <= 0 {
|
||||||
|
return defaultFusionConfig.MinMaxEps
|
||||||
|
}
|
||||||
|
return c.MinMaxEps
|
||||||
|
}
|
||||||
|
|
||||||
|
// retrieveCrossModal 是跨模态并行检索的统一入口。
|
||||||
|
//
|
||||||
|
// 策略(两路并行,召回真正最相似的):
|
||||||
|
// 1. 文本路:query 整段文本编码后查文档层(Doc.DenseVec 已融合其块的媒体向量),
|
||||||
|
// 命中文档若持有媒体块,直接带上该块。
|
||||||
|
// 2. 视觉路:query 经多模态模型文本编码 → 与媒体块向量比余弦
|
||||||
|
// (QueryMediaScored),覆盖文本向量没写到的视觉内容。
|
||||||
|
// 3. 融合:两条路候选各自 min-max 归一化到 [0,1],加权求和后降序,取 topK。
|
||||||
|
// 同一媒体被两路同时命中视为双信号确认,额外加权。
|
||||||
|
//
|
||||||
|
// 多模态空间未配置时视觉路为空,退化为纯文本路(等价旧 docStore.Query)。
|
||||||
|
func (a *Agent) retrieveCrossModal(query string, topK int, cfg CrossModalFusionConfig) []CrossModalHit {
|
||||||
|
if topK <= 0 {
|
||||||
|
topK = 5
|
||||||
|
}
|
||||||
|
// 融合前各取 2× 余量,保证融合排序后 topK 仍有足够候选。
|
||||||
|
per := topK * 2
|
||||||
|
if per < 8 {
|
||||||
|
per = 8
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- 文本路 ----
|
||||||
|
var textHits []CrossModalHit
|
||||||
|
if a.docStore != nil {
|
||||||
|
for _, dh := range a.docStore.QueryScored(query, per) {
|
||||||
|
hit := CrossModalHit{Doc: dh.Doc, DocScore: dh.Score}
|
||||||
|
// 命中文档若持有一等记忆块,把首个媒体块一并带上。
|
||||||
|
if a.mediaStore != nil && len(dh.Doc.Blocks) > 0 {
|
||||||
|
if it, err := a.mediaStore.Stat(dh.Doc.Blocks[0].PayloadDigest); err == nil {
|
||||||
|
hit.Media = it
|
||||||
|
}
|
||||||
|
}
|
||||||
|
textHits = append(textHits, hit)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- 视觉路(多模态文本编码 → 当前记忆层持有的媒体块)----
|
||||||
|
var visualHits []CrossModalHit
|
||||||
|
if a.multimodalSpace != nil && a.multimodalSpace.Loaded() && a.mediaStore != nil {
|
||||||
|
qv, err := a.multimodalSpace.VectorizeDense(query)
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("[crossmodal] 多模态文本编码失败: %v", err)
|
||||||
|
} else if mh, err := a.mediaStore.QueryMediaScored(qv, a.multimodalSpace.Fingerprint(), per); err != nil {
|
||||||
|
log.Printf("[crossmodal] 媒体记忆检索失败: %v", err)
|
||||||
|
} else {
|
||||||
|
// 只有仍被某层记忆块持有的媒体才可召回:CAS 是全库字节存储,
|
||||||
|
// 直接拿它的检索结果会把已无处可归的内容也从记忆里翻出来。
|
||||||
|
held := a.heldMediaDigests()
|
||||||
|
for _, h := range mh {
|
||||||
|
if h.Item == nil || !held[h.Item.Digest] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
visualHits = append(visualHits, CrossModalHit{
|
||||||
|
Media: h.Item, MediaScore: h.Score,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return fuseCrossModal(textHits, visualHits, topK, cfg)
|
||||||
|
}
|
||||||
|
|
||||||
|
// fuseCrossModal 把文本路与视觉路候选按各自归一化分融合排序。
|
||||||
|
//
|
||||||
|
// 归一化模板:两路分数尺度不可直接相加,先各自在路内 min-max 到 [0,1]:
|
||||||
|
//
|
||||||
|
// norm(x) = (x - min) / (max - min),max==min 时置 1
|
||||||
|
//
|
||||||
|
// 再加权求和:fused = wText·normText + wVisual·normVisual。同一媒体两路都命中
|
||||||
|
// (经文档关联 + 视觉直接)时 DoubleHit,在加权分上再加双信号确认分。
|
||||||
|
// 权重通过 CrossModalFusionConfig 按场景配置,不同模型/版本可按实测调整。
|
||||||
|
func fuseCrossModal(textHits, visualHits []CrossModalHit, topK int, cfg CrossModalFusionConfig) []CrossModalHit {
|
||||||
|
norm := func(hits []CrossModalHit, pick func(CrossModalHit) float64) []float64 {
|
||||||
|
out := make([]float64, len(hits))
|
||||||
|
if len(hits) == 0 {
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
maxV, minV := pick(hits[0]), pick(hits[0])
|
||||||
|
for _, h := range hits[1:] {
|
||||||
|
v := pick(h)
|
||||||
|
if v > maxV {
|
||||||
|
maxV = v
|
||||||
|
}
|
||||||
|
if v < minV {
|
||||||
|
minV = v
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for i, h := range hits {
|
||||||
|
v := pick(h)
|
||||||
|
if maxV-minV < cfg.minMaxEps() {
|
||||||
|
out[i] = 1
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out[i] = (v - minV) / (maxV - minV)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
textN := norm(textHits, func(h CrossModalHit) float64 { return h.DocScore })
|
||||||
|
visualN := norm(visualHits, func(h CrossModalHit) float64 { return h.MediaScore })
|
||||||
|
|
||||||
|
byKey := make(map[string]*CrossModalHit)
|
||||||
|
var keys []string
|
||||||
|
key := func(h CrossModalHit) string {
|
||||||
|
if h.Doc != nil {
|
||||||
|
return "doc:" + h.Doc.ID
|
||||||
|
}
|
||||||
|
if h.Media != nil {
|
||||||
|
return "media:" + h.Media.Digest
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// 先并入视觉路(视觉媒体是独立实体)
|
||||||
|
for i, h := range visualHits {
|
||||||
|
k := key(h)
|
||||||
|
if k == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
clone := h
|
||||||
|
clone.Fused = cfg.visualWeight() * visualN[i]
|
||||||
|
byKey[k] = &clone
|
||||||
|
keys = append(keys, k)
|
||||||
|
}
|
||||||
|
// 再并入文本路:命中的文档是独立实体;带媒体的文档若其媒体 digest
|
||||||
|
// 已在视觉路(双命中),合并到同一候选并标记 DoubleHit。
|
||||||
|
for i, h := range textHits {
|
||||||
|
if h.Doc == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if h.Media != nil {
|
||||||
|
if ex, ok := byKey["media:"+h.Media.Digest]; ok {
|
||||||
|
ex.DoubleHit = true
|
||||||
|
ex.Doc = h.Doc
|
||||||
|
ex.Fused += cfg.textWeight()*textN[i] + cfg.doubleHitBonus()
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
k := "doc:" + h.Doc.ID
|
||||||
|
if ex, ok := byKey[k]; ok {
|
||||||
|
ex.Doc = h.Doc
|
||||||
|
ex.DoubleHit = false
|
||||||
|
ex.Fused += cfg.textWeight() * textN[i]
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
clone := h
|
||||||
|
clone.Fused = cfg.textWeight() * textN[i]
|
||||||
|
byKey[k] = &clone
|
||||||
|
keys = append(keys, k)
|
||||||
|
}
|
||||||
|
|
||||||
|
var merged []CrossModalHit
|
||||||
|
for _, k := range keys {
|
||||||
|
if c := byKey[k]; c != nil {
|
||||||
|
merged = append(merged, *c)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.SliceStable(merged, func(i, j int) bool {
|
||||||
|
if merged[i].DoubleHit != merged[j].DoubleHit {
|
||||||
|
return merged[i].DoubleHit
|
||||||
|
}
|
||||||
|
return merged[i].Fused > merged[j].Fused
|
||||||
|
})
|
||||||
|
if len(merged) > topK {
|
||||||
|
merged = merged[:topK]
|
||||||
|
}
|
||||||
|
return merged
|
||||||
|
}
|
||||||
|
|
||||||
|
// crossModalMarkdown 把融合候选渲染成注入上下文的文本。
|
||||||
|
// 文档行给出摘要;媒体行只给 MIME + 短 digest(不再有生成的描述)。
|
||||||
|
func (a *Agent) crossModalMarkdown(hits []CrossModalHit) string {
|
||||||
|
if len(hits) == 0 {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
var lines []string
|
||||||
|
for i, h := range hits {
|
||||||
|
marker := ""
|
||||||
|
switch {
|
||||||
|
case h.DoubleHit:
|
||||||
|
marker = "(图文双命中)"
|
||||||
|
case h.Doc != nil:
|
||||||
|
marker = "(文本命中)"
|
||||||
|
case h.Media != nil:
|
||||||
|
marker = "(视觉命中)"
|
||||||
|
}
|
||||||
|
parts := []string{fmt.Sprintf("[%d]", i+1)}
|
||||||
|
if h.Doc != nil {
|
||||||
|
parts = append(parts, h.Doc.Summary)
|
||||||
|
if h.Doc.Source != "" {
|
||||||
|
parts = append(parts, fmt.Sprintf("(来源:%s)", h.Doc.Source))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if h.Media != nil {
|
||||||
|
if line := mediaLabel(h.Media); line != "" {
|
||||||
|
parts = append(parts, line)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
parts = append(parts, fmt.Sprintf("相关度:%.2f%s", h.Fused, marker))
|
||||||
|
lines = append(lines, strings.Join(parts, " "))
|
||||||
|
}
|
||||||
|
return "【跨模态相关记忆】\n" + strings.Join(lines, "\n")
|
||||||
|
}
|
||||||
@ -180,15 +180,40 @@ func (a *Agent) archiveColdDocs() {
|
|||||||
coldDocs := a.docStore.FindColdDocs(72*time.Hour, 2)
|
coldDocs := a.docStore.FindColdDocs(72*time.Hour, 2)
|
||||||
for _, doc := range coldDocs {
|
for _, doc := range coldDocs {
|
||||||
triples := docToTriples(doc, a.embedder)
|
triples := docToTriples(doc, a.embedder)
|
||||||
if len(triples) > 0 {
|
if len(triples) == 0 {
|
||||||
ec, rc, err := a.memory.Commit(triples, string(a.id)+"_doc_archival", 0)
|
continue
|
||||||
if err != nil {
|
}
|
||||||
log.Printf("[agent] doc→graph archival error: %v", err)
|
ec, rc, blocks, err := a.commitTriplesWithMedia(triples, string(a.id)+"_doc_archival", 0, doc.Blocks)
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("[agent] doc→graph archival error: %v", err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// 归档的实质是「信息从 L2 搬到 L3」。一条实体、一条关系都没写进
|
||||||
|
// 图库时,信息并没有搬过去,此时删文档等于直接丢数据。
|
||||||
|
//
|
||||||
|
// 这不是理论情形:Commit 会静默跳过实体名不合法的三元组
|
||||||
|
//(validEntityName 要求 2–50 字符),而 LLM 生成的长描述几乎
|
||||||
|
// 提不出合规实体名——实测 456 字图片描述得到 0 entities 0
|
||||||
|
// relations,随后文档被删、媒体引用被释放、blob 被 GC 清掉,
|
||||||
|
// 图片与描述彻底消失。保留文档,下一轮再试。
|
||||||
|
if ec == 0 && rc == 0 {
|
||||||
|
log.Printf("[agent] doc→graph: %s 未写入任何实体/关系,保留文档待下轮重试"+
|
||||||
|
"(三元组 %d 条全被实体名校验拒绝)", doc.ID, len(triples))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
log.Printf("[agent] doc→graph: %s → %d entities, %d relations, %d blocks", doc.ID, ec, rc, blocks)
|
||||||
|
|
||||||
|
// 文档持有的一等块写入 L3,并以 document --contains--> block 边关联;
|
||||||
|
// 块 ID 原样保留(迁移而非重建)。块迁走后删除文档即完成迁移。
|
||||||
|
if len(doc.Blocks) > 0 {
|
||||||
|
if bound := a.linkBlocksToDocument(doc.ID, doc.Blocks); bound != len(doc.Blocks) {
|
||||||
|
log.Printf("[agent] doc→graph: %s 块迁移不完整 (%d/%d),保留文档待下轮重试",
|
||||||
|
doc.ID, bound, len(doc.Blocks))
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
log.Printf("[agent] doc→graph: %s → %d entities, %d relations", doc.ID, ec, rc)
|
|
||||||
a.docStore.Remove(doc.ID)
|
|
||||||
}
|
}
|
||||||
|
a.docStore.Remove(doc.ID)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -412,6 +437,10 @@ func docToTriples(doc *document.Doc, embedder nlp.Vectorizer) []memory.Triple {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 媒体不再参与三元组:它作为一等块由 linkBlocksToDocument
|
||||||
|
// 写入 L3 并以 document --contains--> block 边关联,
|
||||||
|
// 不经过文本描述与 NLP 提取器。
|
||||||
|
|
||||||
// NLP 通用提取
|
// NLP 通用提取
|
||||||
e := nlp.NewExtractor(nil)
|
e := nlp.NewExtractor(nil)
|
||||||
if embedder != nil {
|
if embedder != nil {
|
||||||
|
|||||||
@ -10,6 +10,7 @@ import (
|
|||||||
agentIO "gitcode.com/JianFeeeee/HomeAgent/internal/agent/io"
|
agentIO "gitcode.com/JianFeeeee/HomeAgent/internal/agent/io"
|
||||||
"gitcode.com/JianFeeeee/HomeAgent/internal/events"
|
"gitcode.com/JianFeeeee/HomeAgent/internal/events"
|
||||||
sdk "gitcode.com/JianFeeeee/HomeAgent/internal/sdk"
|
sdk "gitcode.com/JianFeeeee/HomeAgent/internal/sdk"
|
||||||
|
pubsdk "gitcode.com/JianFeeeee/homeagent-sdk/sdk"
|
||||||
)
|
)
|
||||||
|
|
||||||
func (a *Agent) eventLoop() {
|
func (a *Agent) eventLoop() {
|
||||||
@ -125,30 +126,18 @@ func (a *Agent) handleSelfInput(msg selfInputMsg) {
|
|||||||
if msg.channel == "" {
|
if msg.channel == "" {
|
||||||
msg.channel = channelConsolidation // 兼容空值:默认走整理路径
|
msg.channel = channelConsolidation // 兼容空值:默认走整理路径
|
||||||
}
|
}
|
||||||
a.processTextInput(&agentIO.InputEvent{
|
a.processInput(&agentIO.InputEvent{
|
||||||
Source: "system",
|
Source: "system",
|
||||||
Type: "text",
|
Type: "text",
|
||||||
Payload: map[string]interface{}{"content": msg.text},
|
Payload: map[string]interface{}{"content": msg.text},
|
||||||
OutputChannel: msg.channel,
|
OutputChannel: msg.channel,
|
||||||
}, msg.text)
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func (a *Agent) handleInput(evt *agentIO.InputEvent) {
|
func (a *Agent) handleInput(evt *agentIO.InputEvent) {
|
||||||
switch evt.Type {
|
switch evt.Type {
|
||||||
case "text":
|
case "text", "image", "audio":
|
||||||
input, _ := evt.Payload["content"].(string)
|
a.processInput(evt)
|
||||||
if input == "" {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
// 去重:webui/GUI 断线重连会重放未确认消息,短窗口内同来源同内容丢弃,避免轰炸
|
|
||||||
if a.isDuplicateInput(evt.Source, input) {
|
|
||||||
log.Printf("[agent] dropped duplicate input from %s: %s", evt.Source, truncateStr(input, 60))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
a.processTextInput(evt, input)
|
|
||||||
|
|
||||||
case "image", "audio":
|
|
||||||
a.processMediaInput(evt)
|
|
||||||
|
|
||||||
case "event":
|
case "event":
|
||||||
log.Printf("[agent] event from %s: %v", evt.Source, evt.Payload)
|
log.Printf("[agent] event from %s: %v", evt.Source, evt.Payload)
|
||||||
@ -162,74 +151,97 @@ func (a *Agent) handleInput(evt *agentIO.InputEvent) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (a *Agent) processMediaInput(evt *agentIO.InputEvent) {
|
// inputPayload 是一次输入在「模态」这个维度上的全部内容。
|
||||||
start := time.Now()
|
//
|
||||||
a.pendingMedia = evt.Payload
|
// 拆出这个结构,是为了让 processInput 只有一条主干:模态不再决定走哪个函数,
|
||||||
defer func() { a.pendingMedia = nil }()
|
// 只决定这里的字段填不填。此前 text 与 image/audio 各有一个 process 函数,
|
||||||
|
// 媒体那条缺了去重、no_memory、通道 Cleaner、中断语义、EventRawInput 五项——
|
||||||
|
// 不是因为媒体不需要,而是复制粘贴之后文本那条继续演进、媒体那条没跟上。
|
||||||
|
type inputPayload struct {
|
||||||
|
// text 是进 LLM 与记忆的文本。纯媒体输入时它是 mediaToBlocks 给的 alt 文案。
|
||||||
|
text string
|
||||||
|
// blocks 非空表示本轮带多模态内容,随当前轮的 message 一起发给模型。
|
||||||
|
blocks []agentAPI.ContentBlock
|
||||||
|
// mediaType 供插件在 stage 里判断本轮媒体的模态。
|
||||||
|
mediaType string
|
||||||
|
// captureTool 是媒体落进 CAS 时记录的来源标签。
|
||||||
|
captureTool string
|
||||||
|
}
|
||||||
|
|
||||||
a.currentOutputChannel = evt.OutputChannel
|
// resolveInput 把 InputEvent 归一成 inputPayload。
|
||||||
if a.currentOutputChannel == "" {
|
//
|
||||||
a.currentOutputChannel = evt.Source
|
// 三种来源在这里合流:
|
||||||
|
// 1. evt.Type 是 image/audio —— 用户直接发的媒体,payload 里是 data/url;
|
||||||
|
// 2. evt.Type 是 text 且 payload 带 media_blocks —— 插件经 IOInjector 的
|
||||||
|
// InjectInputMedia / InjectInputMediaSync / InjectInterruptMedia 注入的
|
||||||
|
// 媒体,块已经是成品;
|
||||||
|
// 3. 纯文本。
|
||||||
|
//
|
||||||
|
// 第 2 种此前无处可去:注入方把块放进 payload,而文本路径不看这个键,
|
||||||
|
// 于是插件注入的媒体到 payload 就断了,且不报错。
|
||||||
|
func (a *Agent) resolveInput(evt *agentIO.InputEvent) (inputPayload, bool) {
|
||||||
|
switch evt.Type {
|
||||||
|
case "image", "audio":
|
||||||
|
blocks, alt := a.mediaToBlocks(evt.Payload, evt.Type, evt.Source)
|
||||||
|
return inputPayload{
|
||||||
|
text: alt,
|
||||||
|
blocks: blocks,
|
||||||
|
mediaType: evt.Type,
|
||||||
|
captureTool: "input_" + evt.Type,
|
||||||
|
}, true
|
||||||
}
|
}
|
||||||
|
|
||||||
blocks, fallback := a.mediaToBlocks(evt.Payload, evt.Type, evt.Source)
|
text, _ := evt.Payload["content"].(string)
|
||||||
|
blocks, mediaType := injectedBlocks(evt.Payload)
|
||||||
stageCtx := a.stageCtxFromInput(fallback, evt.Source, "")
|
// 文本与媒体都空才算无效输入:只带图不带字是合法的(插件注入常这样)。
|
||||||
stageCtx.Extra = map[string]interface{}{
|
if text == "" && len(blocks) == 0 {
|
||||||
"media_blocks": blocks,
|
return inputPayload{}, false
|
||||||
"media_type": evt.Type,
|
|
||||||
"input_source": evt.Source,
|
|
||||||
"output_channel": evt.OutputChannel,
|
|
||||||
}
|
}
|
||||||
a.injectSourceContext(stageCtx, evt)
|
return inputPayload{
|
||||||
|
text: text,
|
||||||
|
blocks: blocks,
|
||||||
|
mediaType: mediaType,
|
||||||
|
captureTool: "inject_" + evt.Source,
|
||||||
|
}, true
|
||||||
|
}
|
||||||
|
|
||||||
if a.runStage(sdk.StageOnInput, stageCtx) {
|
// injectedBlocks 取出 payload 里插件注入的多模态块。
|
||||||
a.emitResponse(evt, *stageCtx.Response)
|
//
|
||||||
return
|
// 两种静态类型都要认:内核内部注入直接给 []agentAPI.ContentBlock,
|
||||||
|
// 而经公共 SDK 的 IOInjector 过来的是 []pubsdk.ContentBlock。两者字段完全一致,
|
||||||
|
// 但 Go 不会自动转换,只认一种的后果是另一种被静默丢弃。
|
||||||
|
func injectedBlocks(payload map[string]interface{}) ([]agentAPI.ContentBlock, string) {
|
||||||
|
var blocks []agentAPI.ContentBlock
|
||||||
|
switch v := payload["media_blocks"].(type) {
|
||||||
|
case []agentAPI.ContentBlock:
|
||||||
|
blocks = v
|
||||||
|
case []pubsdk.ContentBlock:
|
||||||
|
blocks = make([]agentAPI.ContentBlock, 0, len(v))
|
||||||
|
for _, b := range v {
|
||||||
|
nb := agentAPI.ContentBlock{Type: b.Type, Text: b.Text}
|
||||||
|
if b.ImageURL != nil {
|
||||||
|
nb.ImageURL = &agentAPI.ImageURL{URL: b.ImageURL.URL, Detail: b.ImageURL.Detail}
|
||||||
|
}
|
||||||
|
if b.AudioURL != nil {
|
||||||
|
nb.AudioURL = &agentAPI.AudioURL{URL: b.AudioURL.URL}
|
||||||
|
}
|
||||||
|
blocks = append(blocks, nb)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
if len(blocks) == 0 {
|
||||||
a.publishEvent(events.EventRawInput, map[string]interface{}{
|
return nil, ""
|
||||||
"content": evt.Payload,
|
|
||||||
"source": evt.Source,
|
|
||||||
})
|
|
||||||
|
|
||||||
archived := a.context.Prune(fallback, a.maxContextSize-1, a.docStore)
|
|
||||||
if archived > 0 {
|
|
||||||
log.Printf("[agent] pruned %d low-relevance events to document memory", archived)
|
|
||||||
}
|
}
|
||||||
|
// 模态由块自身判定,注入方不必额外声明。图优先:一次注入里图片是主体。
|
||||||
a.context.Append(ContextEvent{
|
mediaType := ""
|
||||||
Timestamp: start,
|
for _, b := range blocks {
|
||||||
Source: evt.Source,
|
if b.ImageURL != nil {
|
||||||
Input: fallback,
|
return blocks, "image"
|
||||||
})
|
}
|
||||||
|
if b.AudioURL != nil {
|
||||||
response, toolsUsed, toolResults, err := a.process(fallback, stageCtx)
|
mediaType = "audio"
|
||||||
if err != nil {
|
}
|
||||||
log.Printf("[agent] process media error: %v", err)
|
|
||||||
resp := fmt.Sprintf("处理错误: %v", err)
|
|
||||||
a.emitResponse(evt, resp)
|
|
||||||
a.context.Append(ContextEvent{Timestamp: time.Now(), Source: "agent", Input: fallback, Response: resp})
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
elapsed := time.Since(start)
|
|
||||||
log.Printf("[agent] %s from %s → response (%dms, tools=%v)", evt.Type, evt.Source, elapsed.Milliseconds(), toolsUsed)
|
|
||||||
|
|
||||||
a.context.Append(ContextEvent{
|
|
||||||
Timestamp: time.Now(),
|
|
||||||
Source: "agent",
|
|
||||||
Input: fallback,
|
|
||||||
Response: response,
|
|
||||||
ToolsUsed: toolsUsed,
|
|
||||||
ToolResults: toolResults,
|
|
||||||
})
|
|
||||||
|
|
||||||
a.emitResponse(evt, response)
|
|
||||||
|
|
||||||
if !stageCtx.NoMemory {
|
|
||||||
a.emitMemoryCandidate(evt.Source, fallback, response, toolResults, toolsUsed)
|
|
||||||
}
|
}
|
||||||
|
return blocks, mediaType
|
||||||
}
|
}
|
||||||
|
|
||||||
func (a *Agent) mediaToBlocks(payload map[string]interface{}, mediaType string, source string) ([]agentAPI.ContentBlock, string) {
|
func (a *Agent) mediaToBlocks(payload map[string]interface{}, mediaType string, source string) ([]agentAPI.ContentBlock, string) {
|
||||||
@ -271,12 +283,12 @@ func (a *Agent) mediaToBlocks(payload map[string]interface{}, mediaType string,
|
|||||||
}
|
}
|
||||||
if mediaType == "image" {
|
if mediaType == "image" {
|
||||||
blocks = append(blocks, agentAPI.ContentBlock{
|
blocks = append(blocks, agentAPI.ContentBlock{
|
||||||
Type: "image_url",
|
Type: "image_url",
|
||||||
ImageURL: &agentAPI.ImageURL{URL: imgURL, Detail: "auto"},
|
ImageURL: &agentAPI.ImageURL{URL: imgURL, Detail: "auto"},
|
||||||
})
|
})
|
||||||
} else if mediaType == "audio" {
|
} else if mediaType == "audio" {
|
||||||
blocks = append(blocks, agentAPI.ContentBlock{
|
blocks = append(blocks, agentAPI.ContentBlock{
|
||||||
Type: "audio_url",
|
Type: "audio_url",
|
||||||
AudioURL: &agentAPI.AudioURL{URL: imgURL},
|
AudioURL: &agentAPI.AudioURL{URL: imgURL},
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@ -285,19 +297,51 @@ func (a *Agent) mediaToBlocks(payload map[string]interface{}, mediaType string,
|
|||||||
return blocks, alt
|
return blocks, alt
|
||||||
}
|
}
|
||||||
|
|
||||||
func (a *Agent) processTextInput(evt *agentIO.InputEvent, input string) {
|
// processInput 是全部模态输入的唯一主干。
|
||||||
|
//
|
||||||
|
// 文本、用户上传的图/音频、插件注入的多模态块走同一条路径,因此去重、
|
||||||
|
// no_memory、通道 Cleaner、中断语义、EventRawInput、媒体入 CAS、媒体记忆绑定
|
||||||
|
// 对所有模态一致——不会再出现「文本路径加了功能、媒体路径没跟上」。
|
||||||
|
func (a *Agent) processInput(evt *agentIO.InputEvent) {
|
||||||
start := time.Now()
|
start := time.Now()
|
||||||
|
|
||||||
|
in, ok := a.resolveInput(evt)
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// 去重按文本做:webui/GUI 断线重连会重放未确认消息。
|
||||||
|
// 带媒体时跳过——媒体输入的 alt 文案("[从 qq 收到了 image]")对不同图片
|
||||||
|
// 是同一句,拿它去重会把连发的两张图误判成重复。
|
||||||
|
if len(in.blocks) == 0 && a.isDuplicateInput(evt.Source, in.text) {
|
||||||
|
log.Printf("[agent] dropped duplicate input from %s: %s", evt.Source, truncateStr(in.text, 60))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
a.currentOutputChannel = evt.OutputChannel
|
a.currentOutputChannel = evt.OutputChannel
|
||||||
if a.currentOutputChannel == "" {
|
if a.currentOutputChannel == "" {
|
||||||
a.currentOutputChannel = evt.Source
|
a.currentOutputChannel = evt.Source
|
||||||
}
|
}
|
||||||
|
|
||||||
if evt.OutputChannel == "_consolidation_" {
|
if evt.OutputChannel == "_consolidation_" {
|
||||||
a.processConsolidation(evt, input)
|
a.processConsolidation(evt, in.text)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// pendingMedia 让 describe_image / transcribe_audio / ocr_image 拿到本轮媒体的
|
||||||
|
// 原始 data/url,也是这三个工具是否出现在工具表里的开关。仅对用户直接上传成立
|
||||||
|
//(payload 里才有 data/url);插件注入的是成品 block,取不到原始数据。
|
||||||
|
if evt.Type == "image" || evt.Type == "audio" {
|
||||||
|
a.pendingMedia = evt.Payload
|
||||||
|
defer func() { a.pendingMedia = nil }()
|
||||||
|
}
|
||||||
|
|
||||||
|
// 媒体先落进 CAS。不存的后果是 ContextEvent.Input 只剩一句 alt 文本,
|
||||||
|
// base64 随 message 数组发给模型后就丢了。
|
||||||
|
if len(in.blocks) > 0 {
|
||||||
|
a.stageMediaDigests(a.captureBlockMedia(in.blocks, in.captureTool)...)
|
||||||
|
}
|
||||||
|
|
||||||
noMemory := false
|
noMemory := false
|
||||||
if v, ok := evt.Payload["no_memory"].(bool); ok {
|
if v, ok := evt.Payload["no_memory"].(bool); ok {
|
||||||
noMemory = v
|
noMemory = v
|
||||||
@ -318,9 +362,13 @@ func (a *Agent) processTextInput(evt *agentIO.InputEvent, input string) {
|
|||||||
noMemory = true
|
noMemory = true
|
||||||
}
|
}
|
||||||
|
|
||||||
stageCtx := a.stageCtxFromInput(input, evt.Source, "")
|
stageCtx := a.stageCtxFromInput(in.text, evt.Source, "")
|
||||||
stageCtx.Extra["input_source"] = evt.Source
|
stageCtx.Extra["input_source"] = evt.Source
|
||||||
stageCtx.Extra["output_channel"] = evt.OutputChannel
|
stageCtx.Extra["output_channel"] = evt.OutputChannel
|
||||||
|
if len(in.blocks) > 0 {
|
||||||
|
stageCtx.Extra["media_blocks"] = in.blocks
|
||||||
|
stageCtx.Extra["media_type"] = in.mediaType
|
||||||
|
}
|
||||||
if noMemory {
|
if noMemory {
|
||||||
stageCtx.NoMemory = true
|
stageCtx.NoMemory = true
|
||||||
}
|
}
|
||||||
@ -331,7 +379,7 @@ func (a *Agent) processTextInput(evt *agentIO.InputEvent, input string) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
input = stageCtx.RawMessage
|
input := stageCtx.RawMessage
|
||||||
|
|
||||||
// 计算层用的清洗文本(不改原文):通道 Cleaner 提取语义内容后用于向量化/提关键词
|
// 计算层用的清洗文本(不改原文):通道 Cleaner 提取语义内容后用于向量化/提关键词
|
||||||
cleanInput := input
|
cleanInput := input
|
||||||
@ -341,12 +389,18 @@ func (a *Agent) processTextInput(evt *agentIO.InputEvent, input string) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
a.publishEvent(events.EventRawInput, map[string]interface{}{
|
// upload_* 字段一并转发:webui 的 EventRawInput 订阅方靠它们还原附件卡片。
|
||||||
"content": input,
|
// 媒体路径此前把整个 payload 塞进 content(一个 map),订阅方按 string 断言
|
||||||
"source": evt.Source,
|
// 直接失败 → 用户发的图从不出现在聊天记录里。
|
||||||
})
|
rawPayload := map[string]interface{}{"content": input, "source": evt.Source}
|
||||||
|
for _, k := range []string{"upload_url", "upload_type", "upload_size", "upload_name"} {
|
||||||
|
if v, ok := evt.Payload[k]; ok {
|
||||||
|
rawPayload[k] = v
|
||||||
|
}
|
||||||
|
}
|
||||||
|
a.publishEvent(events.EventRawInput, rawPayload)
|
||||||
|
|
||||||
archived := a.context.Prune(cleanInput, a.maxContextSize-1, a.docStore)
|
archived := a.pruneOnInput(evt, cleanInput)
|
||||||
if archived > 0 {
|
if archived > 0 {
|
||||||
log.Printf("[agent] pruned %d low-relevance events to document memory", archived)
|
log.Printf("[agent] pruned %d low-relevance events to document memory", archived)
|
||||||
}
|
}
|
||||||
@ -361,7 +415,7 @@ func (a *Agent) processTextInput(evt *agentIO.InputEvent, input string) {
|
|||||||
|
|
||||||
response, toolsUsed, toolResults, err := a.process(input, stageCtx)
|
response, toolsUsed, toolResults, err := a.process(input, stageCtx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Printf("[agent] process error: %v", err)
|
log.Printf("[agent] process %s error: %v", evt.Type, err)
|
||||||
resp := fmt.Sprintf("处理错误: %v", err)
|
resp := fmt.Sprintf("处理错误: %v", err)
|
||||||
a.emitResponse(evt, resp)
|
a.emitResponse(evt, resp)
|
||||||
a.context.Append(ContextEvent{Timestamp: time.Now(), Source: "agent", Input: input, Response: resp})
|
a.context.Append(ContextEvent{Timestamp: time.Now(), Source: "agent", Input: input, Response: resp})
|
||||||
@ -369,16 +423,21 @@ func (a *Agent) processTextInput(evt *agentIO.InputEvent, input string) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
elapsed := time.Since(start)
|
elapsed := time.Since(start)
|
||||||
log.Printf("[agent] input from %s → response (%dms, tools=%v)", evt.Source, elapsed.Milliseconds(), toolsUsed)
|
log.Printf("[agent] %s from %s → response (%dms, tools=%v)", evt.Type, evt.Source, elapsed.Milliseconds(), toolsUsed)
|
||||||
|
|
||||||
a.context.Append(ContextEvent{
|
// 本轮捕获的媒体一起挂到这条事件上:用户上传的、插件注入的,以及模型调
|
||||||
|
// multimodal_see_picture / see_video 时经 SetToolBlocks 注入的(后者在
|
||||||
|
// process() 里被捕获,纯文本输入也会有)。
|
||||||
|
turnEvt := ContextEvent{
|
||||||
Timestamp: time.Now(),
|
Timestamp: time.Now(),
|
||||||
Source: "agent",
|
Source: "agent",
|
||||||
Input: cleanInput,
|
Input: cleanInput,
|
||||||
Response: response,
|
Response: response,
|
||||||
ToolsUsed: toolsUsed,
|
ToolsUsed: toolsUsed,
|
||||||
ToolResults: toolResults,
|
ToolResults: toolResults,
|
||||||
})
|
}
|
||||||
|
a.bindEventMedia(&turnEvt, a.drainMediaDigests())
|
||||||
|
a.context.Append(turnEvt)
|
||||||
|
|
||||||
a.emitResponse(evt, response)
|
a.emitResponse(evt, response)
|
||||||
|
|
||||||
@ -463,3 +522,66 @@ func (a *Agent) drainInterrupts() []string {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// pruneOnInput 按声明的上下文策略裁剪上下文,返回归档的事件数。
|
||||||
|
//
|
||||||
|
// 默认**不裁剪**:ContextPolicy 必须在注入点(payload 的 context_policy)
|
||||||
|
// 或通道定义(ChannelDef.ContextPolicy)上显式声明为 prune 才会裁剪。
|
||||||
|
//
|
||||||
|
// 为什么把无条件裁剪改成需声明:裁剪会把低相关事件归档到文档记忆并从上下文里
|
||||||
|
// 移走,是破坏性的。此前每条输入都裁一次,于是「谁把上下文裁了」在排查时无从
|
||||||
|
// 得知;而插件注入的内容也会被不相关的内容挤掉。按来源/注入点声明后,触发条件
|
||||||
|
// 是可枚举、可审计的。
|
||||||
|
//
|
||||||
|
// 查询向量取**清洗后**的输入(通道 Cleaner 的输出),与工具侧同一套语义:
|
||||||
|
// 原始输入里的 ANSI/base64/JSON 包装会把相关性打分带偏,裁掉本该保留的事件。
|
||||||
|
func (a *Agent) pruneOnInput(evt *agentIO.InputEvent, cleanInput string) int {
|
||||||
|
if a.context == nil || !a.pruneDeclared(evt) {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
topK := a.maxContextSize - 1
|
||||||
|
if topK < 1 {
|
||||||
|
topK = 1
|
||||||
|
}
|
||||||
|
return a.context.Prune(cleanInput, topK, a.docStore)
|
||||||
|
}
|
||||||
|
|
||||||
|
// pruneDeclared 判定这次输入是否显式声明了裁剪。
|
||||||
|
//
|
||||||
|
// 优先级:注入点声明的(payload)> 通道声明的(ChannelDef)> 默认不裁剪。
|
||||||
|
// 注入点是更窄的声明面,同一通道下的不同注入可以有不同意图。
|
||||||
|
func (a *Agent) pruneDeclared(evt *agentIO.InputEvent) bool {
|
||||||
|
if p, ok := evt.Payload["context_policy"].(string); ok && p != "" {
|
||||||
|
return p == pubsdk.ContextPolicyPrune
|
||||||
|
}
|
||||||
|
if a.io != nil {
|
||||||
|
if chDef, ok := a.io.GetInputChannelDef(evt.Source); ok {
|
||||||
|
return chDef.ContextPolicy == pubsdk.ContextPolicyPrune
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// cleanInputFor 解析这条输入在计算层应当使用的清洗文本。
|
||||||
|
//
|
||||||
|
// 优先级:注入点声明的 cleaner(payload.cleaner_name,引用某个已注册的通道
|
||||||
|
// cleaner)> 按 source 查到的通道 cleaner > 原文。
|
||||||
|
//
|
||||||
|
// 声明的 cleaner 名字查不到时**记日志并回退**,而不是静默当没声明:
|
||||||
|
// 注入是 fire-and-forget 的,插件那边看不到错误;至少要在内核日志里留下
|
||||||
|
// 「你声明的清洗没生效」的痕迹,否则排查时只能看到「记忆里的内容很脏」。
|
||||||
|
func (a *Agent) cleanInputFor(evt *agentIO.InputEvent, input string) string {
|
||||||
|
if a.io == nil {
|
||||||
|
return input
|
||||||
|
}
|
||||||
|
if name, ok := evt.Payload["cleaner_name"].(string); ok && name != "" {
|
||||||
|
if chDef, ok := a.io.GetInputChannelDef(name); ok && chDef.Cleaner != nil {
|
||||||
|
return chDef.Cleaner(input)
|
||||||
|
}
|
||||||
|
log.Printf("[agent] 注入声明了 cleaner_name=%q 但没有注册过该通道的 Cleaner,已回退", name)
|
||||||
|
}
|
||||||
|
if chDef, ok := a.io.GetInputChannelDef(evt.Source); ok && chDef.Cleaner != nil {
|
||||||
|
return chDef.Cleaner(input)
|
||||||
|
}
|
||||||
|
return input
|
||||||
|
}
|
||||||
|
|||||||
266
internal/agent/core/graphmedia.go
Normal file
266
internal/agent/core/graphmedia.go
Normal file
@ -0,0 +1,266 @@
|
|||||||
|
package core
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"log"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitcode.com/JianFeeeee/HomeAgent/internal/memory"
|
||||||
|
"gitcode.com/JianFeeeee/HomeAgent/internal/memory/document"
|
||||||
|
)
|
||||||
|
|
||||||
|
// L3 图库的媒体绑定。
|
||||||
|
//
|
||||||
|
// 媒体在 L3 是一等记忆块(memory_blocks),以结构边与承载它的节点相连:
|
||||||
|
// sentence --contains--> block(对话/三元组产生的记忆)
|
||||||
|
// document --contains--> block(L2 文档归档进 L3)
|
||||||
|
//
|
||||||
|
// 这里不再有任何 marker 文本、正则反解或"描述文本当索引"的路径:
|
||||||
|
// 媒体只按自己的统一空间向量被检索,图库/文档只记录它的结构归属。
|
||||||
|
|
||||||
|
// migrateLegacyGraphMedia 把 marker 反解出来的旧媒体实体迁移成原生一等块。
|
||||||
|
//
|
||||||
|
// 旧数据里媒体是 type=Media 的普通实体(「图片 a1b2c3d4e5f6」),
|
||||||
|
// 靠生成的描述文本当索引。迁移后它变成真正的记忆块,以
|
||||||
|
// sentence --contains--> block 结构边挂回原句子,旧实体与描述关系删除。
|
||||||
|
// 迁移幂等(实体处理完即删除),因此在每个 Agent 启动时跑一次是安全的。
|
||||||
|
func (a *Agent) migrateLegacyGraphMedia() {
|
||||||
|
if a.memory == nil || a.mediaStore == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
blocks, entities, err := a.memory.MigrateLegacyMediaEntities(func(short string) (memory.MemoryBlock, bool) {
|
||||||
|
full, err := a.mediaStore.ResolvePrefix(short)
|
||||||
|
if err != nil {
|
||||||
|
return memory.MemoryBlock{}, false
|
||||||
|
}
|
||||||
|
return a.blockFromDigest(full)
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("[media] 旧媒体实体迁移失败(下轮重试): %v", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if blocks > 0 || entities > 0 {
|
||||||
|
log.Printf("[media] 旧媒体实体迁移完成: 新建 %d 个原生块,删除 %d 个描述式实体", blocks, entities)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// attachBlocksToSentence 把一组 digest 变成 L3 一等块并挂到句子上。
|
||||||
|
// seed 允许复用已持有块的 ID(L2→L3 迁移保持块身份不变)。
|
||||||
|
func (a *Agent) attachBlocksToSentence(sentenceID int64, digests []string, seed map[string]memory.MemoryBlock) int {
|
||||||
|
if a.mediaStore == nil || a.memory == nil || sentenceID == 0 {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
bound := 0
|
||||||
|
for _, d := range digests {
|
||||||
|
full, err := a.mediaStore.ResolvePrefix(d)
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("[media] digest %s 无法解析: %v", d, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
b, ok := seed[full]
|
||||||
|
if !ok {
|
||||||
|
if b, ok = a.blockFromDigest(full); !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := a.memory.PutMemoryBlocks([]memory.MemoryBlock{b}); err != nil {
|
||||||
|
log.Printf("[media] L3 块写入失败 (%s): %v", shortDigest(full), err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err := a.memory.AddMemoryBlockEdge("sentence", strconv.FormatInt(sentenceID, 10), "block", b.ID, "contains"); err != nil {
|
||||||
|
log.Printf("[media] 句子→块边建立失败 (%s): %v", shortDigest(full), err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
bound++
|
||||||
|
}
|
||||||
|
return bound
|
||||||
|
}
|
||||||
|
|
||||||
|
// linkBlocksToDocument 把文档持有的块写入 L3,并建立
|
||||||
|
// document --contains--> block 边。块的 ID 原样保留(迁移而非重建)。
|
||||||
|
func (a *Agent) linkBlocksToDocument(docID string, blocks []memory.MemoryBlock) int {
|
||||||
|
if a.memory == nil || docID == "" || len(blocks) == 0 {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
if err := a.memory.PutDocumentNode(docID, ""); err != nil {
|
||||||
|
log.Printf("[media] 写入 L3 文档节点失败 (%s): %v", docID, err)
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
if err := a.memory.PutMemoryBlocks(blocks); err != nil {
|
||||||
|
log.Printf("[media] 写入 L3 记忆块失败 (doc %s): %v", docID, err)
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
bound := 0
|
||||||
|
for _, b := range blocks {
|
||||||
|
if err := a.memory.AddMemoryBlockEdge("document", docID, "block", b.ID, "contains"); err != nil {
|
||||||
|
log.Printf("[media] 文档→块边建立失败 (%s): %v", shortDigest(b.PayloadDigest), err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
bound++
|
||||||
|
}
|
||||||
|
return bound
|
||||||
|
}
|
||||||
|
|
||||||
|
// commitTriplesWithMedia 提交三元组并把三元组显式携带的媒体变成 L3 一等块。
|
||||||
|
//
|
||||||
|
// seed 是调用方已持有的一等块(如 L2 文档的 Blocks),用于保持块身份;
|
||||||
|
// 普通对话路径传 nil。blocks 是本次写入 L3 的块数。
|
||||||
|
func (a *Agent) commitTriplesWithMedia(triples []memory.Triple, sessionID string, turnID int, seed []memory.MemoryBlock) (entities, relations, blocks int, err error) {
|
||||||
|
if a.memory == nil {
|
||||||
|
return 0, 0, 0, fmt.Errorf("graph memory 未启用")
|
||||||
|
}
|
||||||
|
if a.mediaStore == nil {
|
||||||
|
ec, rc, cErr := a.memory.Commit(triples, sessionID, turnID)
|
||||||
|
return ec, rc, 0, cErr
|
||||||
|
}
|
||||||
|
sentenceIDs, ec, rc, err := a.memory.CommitWithMedia(triples, sessionID, turnID)
|
||||||
|
if err != nil {
|
||||||
|
return ec, rc, 0, err
|
||||||
|
}
|
||||||
|
byDigest := make(map[string]memory.MemoryBlock, len(seed))
|
||||||
|
for _, b := range seed {
|
||||||
|
if b.PayloadDigest != "" {
|
||||||
|
byDigest[b.PayloadDigest] = b
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, t := range triples {
|
||||||
|
if len(t.MediaDigests) == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
sid := sentenceIDs[t.SentenceText]
|
||||||
|
if sid == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
blocks += a.attachBlocksToSentence(sid, t.MediaDigests, byDigest)
|
||||||
|
}
|
||||||
|
return ec, rc, blocks, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// RecallBlocksForSentence 反查某条图库句子持有的一等记忆块。
|
||||||
|
func (a *Agent) RecallBlocksForSentence(sentenceID int64) ([]memory.MemoryBlock, error) {
|
||||||
|
if a.memory == nil {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
return a.memory.BlocksForNode("sentence", strconv.FormatInt(sentenceID, 10))
|
||||||
|
}
|
||||||
|
|
||||||
|
// resolveMediaDigests 把模型给的(多为短)digest 补全成完整 digest。
|
||||||
|
//
|
||||||
|
// 补不上就丢弃那一条并记日志:模型可能凭印象编了个 digest,也可能内容已被删除。
|
||||||
|
func (a *Agent) resolveMediaDigests(digests []string) []string {
|
||||||
|
if a.mediaStore == nil || len(digests) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
seen := make(map[string]bool, len(digests))
|
||||||
|
var out []string
|
||||||
|
for _, d := range digests {
|
||||||
|
full, err := a.mediaStore.ResolvePrefix(d)
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("[media] 模型给的 digest %s 无法解析: %v", d, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if seen[full] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seen[full] = true
|
||||||
|
out = append(out, full)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// sentenceIDsFromRelations 收集一批关系引用的句子 id(去重、去零)。
|
||||||
|
//
|
||||||
|
// 关系行本身不持有媒体,媒体作为一等块以 sentence --contains--> block
|
||||||
|
// 结构边与句子相连;因此"这次召回涉及哪些媒体"必须经由关系 → 句子这一跳。
|
||||||
|
func sentenceIDsFromRelations(relations []memory.Relation) []int64 {
|
||||||
|
if len(relations) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
seen := make(map[int64]bool, len(relations))
|
||||||
|
var out []int64
|
||||||
|
for _, r := range relations {
|
||||||
|
if r.SentenceID == 0 || seen[r.SentenceID] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seen[r.SentenceID] = true
|
||||||
|
out = append(out, r.SentenceID)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// mediaContextForRelations 是 mediaContextForSentences 的关系入口。
|
||||||
|
func (a *Agent) mediaContextForRelations(relations []memory.Relation) string {
|
||||||
|
return a.mediaContextForSentences(sentenceIDsFromRelations(relations))
|
||||||
|
}
|
||||||
|
|
||||||
|
// mediaContextForInjectedEntities 为自动注入路径产出媒体说明。
|
||||||
|
//
|
||||||
|
// Indexer.BuildContext 刻意不返回关系(只给实体索引以省 token),
|
||||||
|
// 因此这里用命中的实体名再查一次关系,只为拿到 sentence_id。
|
||||||
|
func (a *Agent) mediaContextForInjectedEntities(injected *memory.InjectedContext) string {
|
||||||
|
if a.mediaStore == nil || a.memory == nil || injected == nil || len(injected.Entities) == 0 {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
names := make([]string, 0, len(injected.Entities))
|
||||||
|
for _, e := range injected.Entities {
|
||||||
|
names = append(names, e.Name)
|
||||||
|
}
|
||||||
|
res, err := a.memory.Recall(nil, names, 1, "")
|
||||||
|
if err != nil || res == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return a.mediaContextForRelations(res.Relations)
|
||||||
|
}
|
||||||
|
|
||||||
|
// blockLabelsForDoc 渲染文档持有块的标签(MIME + 短 digest),供 doc_query 展示。
|
||||||
|
func (a *Agent) blockLabelsForDoc(d *document.Doc) string {
|
||||||
|
if a.mediaStore == nil || d == nil || len(d.Blocks) == 0 {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
var parts []string
|
||||||
|
for _, b := range d.Blocks {
|
||||||
|
it, err := a.mediaStore.Stat(b.PayloadDigest)
|
||||||
|
if err != nil || it == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if line := mediaLabel(it); line != "" {
|
||||||
|
parts = append(parts, line)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return strings.Join(parts, ";")
|
||||||
|
}
|
||||||
|
|
||||||
|
// mediaContextForSentences 给一组句子附上其持有的一等块标签。
|
||||||
|
//
|
||||||
|
// 标签只含 MIME 与短 digest:图片按向量检索,标签的作用是告诉模型
|
||||||
|
// "这条记忆当时带着哪份媒体、可用该 digest 取回字节"。
|
||||||
|
func (a *Agent) mediaContextForSentences(sentenceIDs []int64) string {
|
||||||
|
if a.mediaStore == nil || a.memory == nil || len(sentenceIDs) == 0 {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
var lines []string
|
||||||
|
for _, sid := range sentenceIDs {
|
||||||
|
blocks, err := a.memory.BlocksForNode("sentence", strconv.FormatInt(sid, 10))
|
||||||
|
if err != nil || len(blocks) == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
var parts []string
|
||||||
|
for _, b := range blocks {
|
||||||
|
it, err := a.mediaStore.Stat(b.PayloadDigest)
|
||||||
|
if err != nil || it == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if line := mediaLabel(it); line != "" {
|
||||||
|
parts = append(parts, line)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(parts) > 0 {
|
||||||
|
lines = append(lines, fmt.Sprintf("句子 #%d 关联媒体:%s", sid, strings.Join(parts, ";")))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(lines) == 0 {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return strings.Join(lines, "\n")
|
||||||
|
}
|
||||||
761
internal/agent/core/graphmedia_test.go
Normal file
761
internal/agent/core/graphmedia_test.go
Normal file
@ -0,0 +1,761 @@
|
|||||||
|
package core
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"path/filepath"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitcode.com/JianFeeeee/HomeAgent/internal/memory"
|
||||||
|
"gitcode.com/JianFeeeee/HomeAgent/internal/memory/document"
|
||||||
|
"gitcode.com/JianFeeeee/HomeAgent/internal/memory/media"
|
||||||
|
)
|
||||||
|
|
||||||
|
// L3 图库媒体绑定测试。
|
||||||
|
//
|
||||||
|
// 这一层的目的只有一个:几个月后从图谱走到一条句子,要能取回当时那份媒体。
|
||||||
|
// 媒体作为一等块进入 L3,以结构边与承载节点相连:
|
||||||
|
//
|
||||||
|
// sentence --contains--> block(对话/三元组产生的记忆)
|
||||||
|
// document --contains--> block(L2 文档归档进 L3)
|
||||||
|
//
|
||||||
|
// 描述文本、marker 反解、由 marker 反推出的「媒体实体」全部已废弃,
|
||||||
|
// 因此这些测试也不存在任何按描述检索的断言。
|
||||||
|
|
||||||
|
// attachBlockToSentence 提交一条句子并把媒体变成 L3 一等块。
|
||||||
|
// 必须走真实提交:边要求两端都是真实图节点。
|
||||||
|
func attachBlockToSentence(t *testing.T, g *memory.GraphDB, ms *media.Store, sentenceText, digest string) (int64, memory.MemoryBlock) {
|
||||||
|
t.Helper()
|
||||||
|
ids, _, _, err := g.CommitWithMedia([]memory.Triple{{
|
||||||
|
Subject: "媒体载体", Relation: "包含", Object: "内容", SentenceText: sentenceText,
|
||||||
|
}}, "test", 0)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("CommitWithMedia: %v", err)
|
||||||
|
}
|
||||||
|
sid := ids[sentenceText]
|
||||||
|
if sid == 0 {
|
||||||
|
t.Fatalf("拿不到句子 id: %q", sentenceText)
|
||||||
|
}
|
||||||
|
it, err := ms.Stat(digest)
|
||||||
|
if err != nil || it == nil {
|
||||||
|
t.Fatalf("Stat(%s): %v", shortDigest(digest), err)
|
||||||
|
}
|
||||||
|
b := memory.MemoryBlock{
|
||||||
|
ID: fmt.Sprintf("blk_test_%d_%s", sid, shortDigest(digest)),
|
||||||
|
Modality: memory.BlockImage,
|
||||||
|
PayloadDigest: it.Digest,
|
||||||
|
MIME: it.MIME,
|
||||||
|
Size: it.Size,
|
||||||
|
Width: it.Width,
|
||||||
|
Height: it.Height,
|
||||||
|
Vector: it.Vec,
|
||||||
|
Fingerprint: it.VecModel,
|
||||||
|
}
|
||||||
|
if err := g.PutMemoryBlocks([]memory.MemoryBlock{b}); err != nil {
|
||||||
|
t.Fatalf("PutMemoryBlocks: %v", err)
|
||||||
|
}
|
||||||
|
if err := g.AddMemoryBlockEdge("sentence", strconv.FormatInt(sid, 10), "block", b.ID, "contains"); err != nil {
|
||||||
|
t.Fatalf("AddMemoryBlockEdge: %v", err)
|
||||||
|
}
|
||||||
|
return sid, b
|
||||||
|
}
|
||||||
|
|
||||||
|
func newGraphMediaAgent(t *testing.T) (*Agent, *memory.GraphDB, *media.Store) {
|
||||||
|
t.Helper()
|
||||||
|
dir := t.TempDir()
|
||||||
|
|
||||||
|
g, err := memory.NewGraphDB(filepath.Join(dir, "graph.db"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewGraphDB: %v", err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { g.Close() })
|
||||||
|
|
||||||
|
ms, err := media.New(filepath.Join(dir, "media"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("media.New: %v", err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { ms.Close() })
|
||||||
|
|
||||||
|
return &Agent{memory: g, mediaStore: ms}, g, ms
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCommitWithMedia_ReturnsSentenceIDs(t *testing.T) {
|
||||||
|
_, g, _ := newGraphMediaAgent(t)
|
||||||
|
|
||||||
|
sentence := "这张图是紫蓝红三色带。"
|
||||||
|
triples := []memory.Triple{{
|
||||||
|
Subject: "图片", Relation: "内容", Object: "三色带",
|
||||||
|
SentenceText: sentence,
|
||||||
|
}}
|
||||||
|
|
||||||
|
ids, ec, rc, err := g.CommitWithMedia(triples, "s1", 0)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if ec == 0 || rc == 0 {
|
||||||
|
t.Fatalf("应写入实体与关系,实际 ec=%d rc=%d", ec, rc)
|
||||||
|
}
|
||||||
|
if ids[sentence] == 0 {
|
||||||
|
t.Fatalf("应返回句子 id,实际 %v", ids)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCommit_StillWorksAfterRefactor(t *testing.T) {
|
||||||
|
// Commit 有三十多个调用点,内部转调后行为必须完全不变
|
||||||
|
_, g, _ := newGraphMediaAgent(t)
|
||||||
|
|
||||||
|
triples := []memory.Triple{
|
||||||
|
{Subject: "张三", Relation: "喜欢", Object: "咖啡", SentenceText: "张三喜欢咖啡"},
|
||||||
|
{Subject: "李四", Relation: "住在", Object: "北京"},
|
||||||
|
}
|
||||||
|
ec, rc, err := g.Commit(triples, "s1", 0)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if ec != 4 || rc != 2 {
|
||||||
|
t.Fatalf("期望 4 实体 2 关系,实际 ec=%d rc=%d", ec, rc)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 重复提交同一批:关系被唯一约束去重。
|
||||||
|
//
|
||||||
|
// 实体计数**不**归零——这是 upsertEntity 的既有行为:SQLite 的
|
||||||
|
// ON CONFLICT DO UPDATE 也算一行 affected,于是 RowsAffected() > 0
|
||||||
|
// 被当成"新建了"。用 main 分支的 graph.go 单独验证过基线同样是
|
||||||
|
// 首次 ec=2 / 重复 ec=2,与 CommitWithMedia 重构无关。
|
||||||
|
// entitiesCreated 只用于日志,故此处记录现状而不改行为。
|
||||||
|
ec2, rc2, err := g.Commit(triples, "s1", 0)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if rc2 != 0 {
|
||||||
|
t.Fatalf("重复提交不该新建关系,实际 rc=%d", rc2)
|
||||||
|
}
|
||||||
|
if ec2 != 4 {
|
||||||
|
t.Fatalf("实体计数应与首次一致(既有 upsert 计数行为),实际 ec=%d", ec2)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCommitTriplesWithMedia_RoundTrip(t *testing.T) {
|
||||||
|
// 整层的核心断言:写入 → 提交 → 反查取回原始字节
|
||||||
|
a, _, ms := newGraphMediaAgent(t)
|
||||||
|
|
||||||
|
content := []byte("\x89PNG\r\n\x1a\n fake image bytes")
|
||||||
|
digest, err := ms.Put(content, media.Item{MIME: "image/png", Kind: media.KindImage})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
sentence := "用户发来一张紫蓝红三色带图。"
|
||||||
|
triples := []memory.Triple{{
|
||||||
|
Subject: "图片", Relation: "内容", Object: "三色带",
|
||||||
|
SentenceText: sentence,
|
||||||
|
MediaDigests: []string{digest[:12]}, // 模型手里通常只有短 digest
|
||||||
|
}}
|
||||||
|
|
||||||
|
if _, _, bound, err := a.commitTriplesWithMedia(triples, "s1", 0, nil); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
} else if bound != 1 {
|
||||||
|
t.Fatalf("应绑定 1 个块,实际 %d", bound)
|
||||||
|
}
|
||||||
|
|
||||||
|
ids, _, _, err := a.memory.CommitWithMedia(triples, "s1", 0)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
sid := ids[sentence]
|
||||||
|
if sid == 0 {
|
||||||
|
t.Fatal("拿不到句子 id")
|
||||||
|
}
|
||||||
|
|
||||||
|
// 反查:从句子取回一等块,再取回字节
|
||||||
|
blocks, err := a.RecallBlocksForSentence(sid)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(blocks) != 1 || blocks[0].PayloadDigest != digest {
|
||||||
|
t.Fatalf("反查应得完整 digest %s,实际 %+v", shortDigest(digest), blocks)
|
||||||
|
}
|
||||||
|
got, err := ms.Get(blocks[0].PayloadDigest)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("取回内容失败: %v", err)
|
||||||
|
}
|
||||||
|
if string(got) != string(content) {
|
||||||
|
t.Fatal("取回的内容与写入不一致")
|
||||||
|
}
|
||||||
|
|
||||||
|
// 块仍被 L3 持有 → 内容应仍可读
|
||||||
|
if _, err := ms.Get(digest); err != nil {
|
||||||
|
t.Fatalf("被 L3 记忆块持有的内容不该被清除: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAttachBlocksToSentence_SkipsUnresolvable(t *testing.T) {
|
||||||
|
// digest 在库里不存在时必须跳过,不能建一条指向虚无的块边。
|
||||||
|
a, g, _ := newGraphMediaAgent(t)
|
||||||
|
if n := a.attachBlocksToSentence(42, []string{"deadbeefdead"}, nil); n != 0 {
|
||||||
|
t.Fatalf("无法补全的 digest 不该建块,实际绑定 %d", n)
|
||||||
|
}
|
||||||
|
blocks, err := g.BlocksForNode("sentence", "42")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(blocks) != 0 {
|
||||||
|
t.Fatalf("不该有块,实际 %+v", blocks)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAttachBlocksToSentence_NilStoreNoop(t *testing.T) {
|
||||||
|
a := &Agent{}
|
||||||
|
if n := a.attachBlocksToSentence(1, []string{"aaaaaaaaaaaa"}, nil); n != 0 {
|
||||||
|
t.Fatalf("媒体关闭时应静默无操作,实际 %d", n)
|
||||||
|
}
|
||||||
|
if got, err := a.RecallBlocksForSentence(1); err != nil || got != nil {
|
||||||
|
t.Fatalf("媒体关闭时应静默无操作,实际 %v / %v", got, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAttachBlocksToSentence_ReusesSeedIdentity(t *testing.T) {
|
||||||
|
// L2→L3 迁移必须保持块身份:同一个块换层,而不是另建一个同内容的新块。
|
||||||
|
a, g, ms := newGraphMediaAgent(t)
|
||||||
|
digest, _ := ms.Put([]byte("seed-img"), media.Item{MIME: "image/png"})
|
||||||
|
seedBlock, ok := a.blockFromDigest(digest)
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("blockFromDigest 失败")
|
||||||
|
}
|
||||||
|
|
||||||
|
ids, _, _, err := g.CommitWithMedia([]memory.Triple{{
|
||||||
|
Subject: "迁移", Relation: "包含", Object: "媒体", SentenceText: "迁移测试句。",
|
||||||
|
}}, "seed", 0)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
sid := ids["迁移测试句。"]
|
||||||
|
|
||||||
|
byDigest := map[string]memory.MemoryBlock{digest: seedBlock}
|
||||||
|
if n := a.attachBlocksToSentence(sid, []string{digest}, byDigest); n != 1 {
|
||||||
|
t.Fatalf("应绑定 1 个块,实际 %d", n)
|
||||||
|
}
|
||||||
|
blocks, err := g.BlocksForNode("sentence", strconv.FormatInt(sid, 10))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(blocks) != 1 || blocks[0].ID != seedBlock.ID {
|
||||||
|
t.Fatalf("块身份应保持为 %s,实际 %+v", seedBlock.ID, blocks)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLinkBlocksToDocument_CreatesDocumentNodeEdge(t *testing.T) {
|
||||||
|
// 文档归档进 L3:块原样迁入,document --contains--> block 边建立。
|
||||||
|
a, g, ms := newGraphMediaAgent(t)
|
||||||
|
|
||||||
|
digest, _ := ms.Put([]byte("doc-img"), media.Item{MIME: "image/png"})
|
||||||
|
b, ok := a.blockFromDigest(digest)
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("blockFromDigest 失败")
|
||||||
|
}
|
||||||
|
|
||||||
|
if n := a.linkBlocksToDocument("doc_42", []memory.MemoryBlock{b}); n != 1 {
|
||||||
|
t.Fatalf("应建立 1 条文档→块边,实际 %d", n)
|
||||||
|
}
|
||||||
|
blocks, err := g.BlocksForNode("document", "doc_42")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(blocks) != 1 || blocks[0].ID != b.ID {
|
||||||
|
t.Fatalf("文档应持有块 %s,实际 %+v", b.ID, blocks)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCommitTriplesWithMedia_FallsBackWithoutStore(t *testing.T) {
|
||||||
|
// 媒体关闭时退回普通 Commit,行为与直接调 Commit 完全一致
|
||||||
|
dir := t.TempDir()
|
||||||
|
g, err := memory.NewGraphDB(filepath.Join(dir, "g.db"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer g.Close()
|
||||||
|
|
||||||
|
a := &Agent{memory: g}
|
||||||
|
ec, rc, _, err := a.commitTriplesWithMedia([]memory.Triple{
|
||||||
|
{Subject: "张三", Relation: "喜欢", Object: "咖啡"},
|
||||||
|
}, "s1", 0, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if ec != 2 || rc != 1 {
|
||||||
|
t.Fatalf("期望 2 实体 1 关系,实际 ec=%d rc=%d", ec, rc)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMediaContextForSentences(t *testing.T) {
|
||||||
|
a, g, ms := newGraphMediaAgent(t)
|
||||||
|
|
||||||
|
digest, _ := ms.Put([]byte("img"), media.Item{MIME: "image/png"})
|
||||||
|
sid, _ := attachBlockToSentence(t, g, ms, "一张紫蓝红三色带图。", digest)
|
||||||
|
|
||||||
|
out := a.mediaContextForSentences([]int64{sid, sid + 100})
|
||||||
|
if out == "" {
|
||||||
|
t.Fatal("应产出媒体说明")
|
||||||
|
}
|
||||||
|
if !contains(out, fmt.Sprintf("句子 #%d", sid)) || !contains(out, shortDigest(digest)) {
|
||||||
|
t.Fatalf("说明内容不对: %q", out)
|
||||||
|
}
|
||||||
|
// 说明只含 MIME 与短 digest,不含任何生成的描述
|
||||||
|
if contains(out, "紫蓝红") {
|
||||||
|
t.Fatalf("说明里不该有描述文本(描述式索引已废弃): %q", out)
|
||||||
|
}
|
||||||
|
// 无引用的句子不该出现
|
||||||
|
if contains(out, fmt.Sprintf("句子 #%d", sid+100)) {
|
||||||
|
t.Fatalf("无引用的句子不该出现: %q", out)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMediaContextForRelations_SurfacesMediaToAgent(t *testing.T) {
|
||||||
|
// L3 检索接线回归:媒体作为一等块进了图库,agent 必须拿得出来。
|
||||||
|
a, g, ms := newGraphMediaAgent(t)
|
||||||
|
|
||||||
|
digest, err := ms.Put([]byte("img bytes"), media.Item{MIME: "image/png"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
sid, _ := attachBlockToSentence(t, g, ms, "一张紫蓝红三色带图。", digest)
|
||||||
|
|
||||||
|
// 命中的关系挂着该句子 → 应产出媒体说明
|
||||||
|
out := a.mediaContextForRelations([]memory.Relation{{ID: 1, SentenceID: sid}})
|
||||||
|
if out == "" {
|
||||||
|
t.Fatal("关系挂着有媒体的句子,却没产出媒体说明——L3 检索接线断了")
|
||||||
|
}
|
||||||
|
if !contains(out, shortDigest(digest)) {
|
||||||
|
t.Errorf("媒体说明里应含短 digest 供反查: %q", out)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 没挂媒体的关系不该产出噪声
|
||||||
|
if out := a.mediaContextForRelations([]memory.Relation{{ID: 2, SentenceID: 99}}); out != "" {
|
||||||
|
t.Errorf("无媒体的句子不该产出说明: %q", out)
|
||||||
|
}
|
||||||
|
if out := a.mediaContextForRelations(nil); out != "" {
|
||||||
|
t.Errorf("空关系不该产出说明: %q", out)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBuildMemoryContext_IncludesMediaSection(t *testing.T) {
|
||||||
|
// buildMemoryContext 是自动注入路径(每次 LLM 调用都走)。
|
||||||
|
// 媒体说明必须出现在这里,否则 agent 只有显式调 memory_recall 才知道有图。
|
||||||
|
a, graph, ms := newGraphMediaAgent(t)
|
||||||
|
|
||||||
|
digest, err := ms.Put([]byte("auto inject"), media.Item{MIME: "image/png"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
sentence := "用户发来的图片。"
|
||||||
|
sids, _, _, err := graph.CommitWithMedia([]memory.Triple{{
|
||||||
|
Subject: "测试图片", Relation: "包含", Object: "三色带", SentenceText: sentence,
|
||||||
|
}}, "auto", 0)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
sid := sids[sentence]
|
||||||
|
if sid == 0 {
|
||||||
|
t.Fatal("拿不到句子 id")
|
||||||
|
}
|
||||||
|
if err := graph.PutMemoryBlocks([]memory.MemoryBlock{{
|
||||||
|
ID: "blk_auto_1", Modality: memory.BlockImage,
|
||||||
|
PayloadDigest: digest, MIME: "image/png",
|
||||||
|
}}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := graph.AddMemoryBlockEdge("sentence", strconv.FormatInt(sid, 10), "block", "blk_auto_1", "contains"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
a.indexer = memory.NewIndexer(graph)
|
||||||
|
if err := a.indexer.Sync(); err != nil {
|
||||||
|
t.Fatalf("indexer sync: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
out := a.buildMemoryContext("测试图片", 0)
|
||||||
|
if out == "" {
|
||||||
|
t.Skip("图库召回未命中(indexer 检索策略所致),无法验证媒体段注入")
|
||||||
|
}
|
||||||
|
if !contains(out, "【关联媒体】") {
|
||||||
|
t.Errorf("自动注入的记忆上下文缺少媒体段: %q", out)
|
||||||
|
}
|
||||||
|
if !contains(out, shortDigest(digest)) {
|
||||||
|
t.Errorf("媒体段里应含短 digest: %q", out)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolvePrefix(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
ms, err := media.New(filepath.Join(dir, "m"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer ms.Close()
|
||||||
|
|
||||||
|
digest, _ := ms.Put([]byte("content"), media.Item{MIME: "image/png"})
|
||||||
|
|
||||||
|
// 短前缀补全
|
||||||
|
full, err := ms.ResolvePrefix(digest[:12])
|
||||||
|
if err != nil || full != digest {
|
||||||
|
t.Fatalf("短前缀补全失败: %v / %v", full, err)
|
||||||
|
}
|
||||||
|
// 完整 digest 原样返回
|
||||||
|
full, err = ms.ResolvePrefix(digest)
|
||||||
|
if err != nil || full != digest {
|
||||||
|
t.Fatalf("完整 digest 应原样返回: %v / %v", full, err)
|
||||||
|
}
|
||||||
|
// 过短拒绝
|
||||||
|
if _, err := ms.ResolvePrefix("abc"); err == nil {
|
||||||
|
t.Fatal("过短前缀应报错")
|
||||||
|
}
|
||||||
|
// 不存在
|
||||||
|
if _, err := ms.ResolvePrefix("deadbeefdead"); err == nil {
|
||||||
|
t.Fatal("不存在的前缀应报错")
|
||||||
|
}
|
||||||
|
// 完整但不存在的 digest 也要报错,否则调用方会挂一条孤儿块
|
||||||
|
fake := strings.Repeat("0", 64)
|
||||||
|
if _, err := ms.ResolvePrefix(fake); err == nil {
|
||||||
|
t.Fatal("不存在的完整 digest 应报错")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolvePrefix_AmbiguityIsError(t *testing.T) {
|
||||||
|
// 前缀歧义视为错误而非"取第一个":挂错块会让内容被误删。
|
||||||
|
// 构造歧义需要两个同前缀 digest——sha256 无法人为构造,
|
||||||
|
// 因此这里退而验证「12 位前缀在大量样本下的行为是确定的」:
|
||||||
|
// 要么唯一命中,要么明确报歧义,绝不静默取第一个。
|
||||||
|
dir := t.TempDir()
|
||||||
|
ms, err := media.New(filepath.Join(dir, "m"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer ms.Close()
|
||||||
|
|
||||||
|
digests := make([]string, 0, 200)
|
||||||
|
for i := 0; i < 200; i++ {
|
||||||
|
d, err := ms.Put([]byte("content-"+strconv.Itoa(i)), media.Item{MIME: "image/png"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
digests = append(digests, d)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, d := range digests {
|
||||||
|
got, err := ms.ResolvePrefix(d[:12])
|
||||||
|
if err != nil {
|
||||||
|
if !contains(err.Error(), "歧义") {
|
||||||
|
t.Fatalf("非歧义错误: %v", err)
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if got != d {
|
||||||
|
t.Fatalf("补全结果错误: 前缀 %s 得到 %s", d[:12], got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestArchiveColdDocs_KeepsDocWhenGraphWriteEmpty(t *testing.T) {
|
||||||
|
// 数据丢失回归:三元组全被实体名校验拒绝时(Commit 无错但 0 entities
|
||||||
|
// 0 relations),文档不能删、其持有的块不能丢。
|
||||||
|
a, _, ms := newGraphMediaAgent(t)
|
||||||
|
|
||||||
|
dir := t.TempDir()
|
||||||
|
ds := document.NewStore(filepath.Join(dir, "docs"), memory.TokenizeWords)
|
||||||
|
if err := ds.Start(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer ds.Stop()
|
||||||
|
a.docStore = ds
|
||||||
|
a.embedder = memory.NewStaticEmbedder()
|
||||||
|
|
||||||
|
content := []byte("image bytes")
|
||||||
|
digest, err := ms.Put(content, media.Item{MIME: "image/png"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 精确构造「三元组非空 + Commit 全部拒绝」这个状态:
|
||||||
|
// Source/Summary 都超过 validEntityName 的 50 字符上限,
|
||||||
|
// 于是 docToTriples 产出的两条元数据三元组都被跳过。
|
||||||
|
longSource := strings.Repeat("超长来源名", 20) // 100 字
|
||||||
|
longSummary := strings.Repeat("超长摘要文本", 20) // >80 字触发长度门槛被跳过
|
||||||
|
it, _ := ms.Stat(digest)
|
||||||
|
doc := &document.Doc{
|
||||||
|
ID: "doc_keep",
|
||||||
|
Summary: longSummary,
|
||||||
|
Content: "一段没有媒体标记的正文",
|
||||||
|
Source: longSource,
|
||||||
|
CreatedAt: time.Now().Add(-200 * time.Hour),
|
||||||
|
LastAccess: time.Now().Add(-200 * time.Hour),
|
||||||
|
AccessCount: 0,
|
||||||
|
Blocks: []memory.MemoryBlock{{ID: "blk_keep_1", Modality: memory.BlockImage,
|
||||||
|
PayloadDigest: it.Digest, MIME: it.MIME, Size: it.Size}},
|
||||||
|
}
|
||||||
|
if err := ds.Insert(doc); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// Insert 会把 LastAccess 覆写成 now、AccessCount 置 1,
|
||||||
|
// 于是 FindColdDocs(72h, 2) 一篇都找不到。插入后再改回来,
|
||||||
|
// 让文档真正满足"冷"的条件——这是触发归档路径的前提。
|
||||||
|
for _, d := range ds.RecentDocs(10) {
|
||||||
|
if d.ID == doc.ID {
|
||||||
|
d.LastAccess = time.Now().Add(-200 * time.Hour)
|
||||||
|
d.AccessCount = 0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
a.archiveColdDocs()
|
||||||
|
|
||||||
|
// 关键断言:内容在、块在、文档在
|
||||||
|
if _, err := ms.Get(digest); err != nil {
|
||||||
|
t.Fatalf("图库未写入任何实体/关系,内容却丢了: %v", err)
|
||||||
|
}
|
||||||
|
held := false
|
||||||
|
for _, d := range ds.RecentDocs(10) {
|
||||||
|
if d.ID == doc.ID && len(d.Blocks) > 0 {
|
||||||
|
held = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !held {
|
||||||
|
t.Error("文档或块被释放了——图库没有句子承载它,内容会被删除")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestArchiveColdDocs_MigratesBlocksToGraph(t *testing.T) {
|
||||||
|
// 归档成功时块必须迁进 L3 并以 document --contains--> block 关联,
|
||||||
|
// 然后文档才被删除(迁移而非复制/引用保活)。
|
||||||
|
a, g, ms := newGraphMediaAgent(t)
|
||||||
|
|
||||||
|
dir := t.TempDir()
|
||||||
|
ds := document.NewStore(filepath.Join(dir, "docs"), memory.TokenizeWords)
|
||||||
|
if err := ds.Start(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer ds.Stop()
|
||||||
|
a.docStore = ds
|
||||||
|
a.embedder = memory.NewStaticEmbedder()
|
||||||
|
|
||||||
|
digest, _ := ms.Put([]byte("archived-image"), media.Item{MIME: "image/png"})
|
||||||
|
it, _ := ms.Stat(digest)
|
||||||
|
doc := &document.Doc{
|
||||||
|
ID: "doc_arch",
|
||||||
|
Summary: "带图的冷文档",
|
||||||
|
Content: "张三把三色带图交给了李四。",
|
||||||
|
Source: "manual",
|
||||||
|
Blocks: []memory.MemoryBlock{{ID: "blk_arch_1", Modality: memory.BlockImage,
|
||||||
|
PayloadDigest: it.Digest, MIME: it.MIME, Size: it.Size}},
|
||||||
|
}
|
||||||
|
if err := ds.Insert(doc); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, d := range ds.RecentDocs(10) {
|
||||||
|
if d.ID == doc.ID {
|
||||||
|
d.LastAccess = time.Now().Add(-200 * time.Hour)
|
||||||
|
d.AccessCount = 0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
a.archiveColdDocs()
|
||||||
|
|
||||||
|
if d := ds.Get("doc_arch"); d != nil {
|
||||||
|
t.Fatal("块已迁入 L3,文档应被删除")
|
||||||
|
}
|
||||||
|
blocks, err := g.BlocksForNode("document", "doc_arch")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(blocks) != 1 || blocks[0].ID != "blk_arch_1" {
|
||||||
|
t.Fatalf("L3 文档节点应持有原块(身份不变),实际 %+v", blocks)
|
||||||
|
}
|
||||||
|
if _, err := ms.Get(digest); err != nil {
|
||||||
|
t.Fatalf("块被 L3 持有,内容应仍可读: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMigrateLegacyMediaEntities(t *testing.T) {
|
||||||
|
// 旧数据:媒体被伪装成 type=Media 的实体,靠描述文本当索引。
|
||||||
|
// 迁移必须把它还原成原生块(挂回原句子)并删掉旧实体与描述关系。
|
||||||
|
_, g, ms := newGraphMediaAgent(t)
|
||||||
|
|
||||||
|
digest, _ := ms.Put([]byte("legacy-img"), media.Item{MIME: "image/png"})
|
||||||
|
sentence := "老数据里的三色带图 [image/png " + digest[:12] + "]"
|
||||||
|
// 直接构造旧的实体/关系形态(不走已删除的 marker 代码)。
|
||||||
|
ids, _, _, err := g.CommitWithMedia([]memory.Triple{{
|
||||||
|
Subject: "图片 " + digest[:12],
|
||||||
|
SubjectType: "Media",
|
||||||
|
Relation: "内容",
|
||||||
|
Object: "三色带的描述文本",
|
||||||
|
ObjectType: "Description",
|
||||||
|
SentenceText: sentence,
|
||||||
|
}}, "legacy", 0)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
sid := ids[sentence]
|
||||||
|
if sid == 0 {
|
||||||
|
t.Fatal("拿不到句子 id")
|
||||||
|
}
|
||||||
|
|
||||||
|
blocks, entities, err := g.MigrateLegacyMediaEntities(func(short string) (memory.MemoryBlock, bool) {
|
||||||
|
full, err := ms.ResolvePrefix(short)
|
||||||
|
if err != nil {
|
||||||
|
return memory.MemoryBlock{}, false
|
||||||
|
}
|
||||||
|
it, err := ms.Stat(full)
|
||||||
|
if err != nil {
|
||||||
|
return memory.MemoryBlock{}, false
|
||||||
|
}
|
||||||
|
return memory.MemoryBlock{
|
||||||
|
ID: "blk_legacy_" + short, Modality: memory.BlockImage,
|
||||||
|
PayloadDigest: it.Digest, MIME: it.MIME, Size: it.Size,
|
||||||
|
}, true
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if blocks != 1 || entities != 1 {
|
||||||
|
t.Fatalf("应迁移 1 块 / 删 1 实体,实际 %d / %d", blocks, entities)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 旧媒体实体与描述关系必须消失
|
||||||
|
res, err := g.Recall([]string{"图片 " + digest[:12]}, nil, 2, "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, e := range res.Entities {
|
||||||
|
if e.Type == "Media" {
|
||||||
|
t.Fatalf("旧媒体实体仍存在: %+v", e)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// 块必须挂回原句子
|
||||||
|
got, err := g.BlocksForNode("sentence", strconv.FormatInt(sid, 10))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(got) != 1 || got[0].PayloadDigest != digest {
|
||||||
|
t.Fatalf("句子应持有原生块,实际 %+v", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 幂等:再跑一遍不应重复建块
|
||||||
|
blocks2, entities2, err := g.MigrateLegacyMediaEntities(nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if blocks2 != 0 || entities2 != 0 {
|
||||||
|
t.Fatalf("无 resolver 时应空操作,实际 %d / %d", blocks2, entities2)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCleanupOrphanedSentences_KeepsBlockBackedSentences(t *testing.T) {
|
||||||
|
// 旧媒体实体被删除后,承载它的句子可能再无关系引用,
|
||||||
|
// 但它还挂着媒体块——清理孤儿句子时不能把它删掉。
|
||||||
|
a, g, ms := newGraphMediaAgent(t)
|
||||||
|
|
||||||
|
digest, _ := ms.Put([]byte("orphan-img"), media.Item{MIME: "image/png"})
|
||||||
|
sentence := "只靠媒体块存活的句子。"
|
||||||
|
ids, _, _, err := g.CommitWithMedia([]memory.Triple{{
|
||||||
|
Subject: "媒体载体", Relation: "包含", Object: "内容", SentenceText: sentence,
|
||||||
|
}}, "orphan", 0)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
sid := ids[sentence]
|
||||||
|
|
||||||
|
b, ok := a.blockFromDigest(digest)
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("blockFromDigest 失败")
|
||||||
|
}
|
||||||
|
if err := g.PutMemoryBlocks([]memory.MemoryBlock{b}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := g.AddMemoryBlockEdge("sentence", strconv.FormatInt(sid, 10), "block", b.ID, "contains"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 解除关系引用,句子只剩块边
|
||||||
|
res, err := g.Recall([]string{"媒体载体"}, nil, 2, "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, r := range res.Relations {
|
||||||
|
if err := g.ClearSentenceID(r.ID); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, err := g.CleanupOrphanedSentences(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
blocks, err := g.BlocksForNode("sentence", strconv.FormatInt(sid, 10))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(blocks) != 1 {
|
||||||
|
t.Fatalf("承载媒体块的句子被误删,块反查失败: %+v", blocks)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSentenceIDsFromRelations(t *testing.T) {
|
||||||
|
// 关系行不持有媒体,媒体挂在句子上。这个函数负责"关系→句子"这一跳,
|
||||||
|
// 去重与去零都不能少:sentence_id=0 表示该关系没有关联句子。
|
||||||
|
rels := []memory.Relation{
|
||||||
|
{ID: 1, SentenceID: 5},
|
||||||
|
{ID: 2, SentenceID: 0}, // 无句子
|
||||||
|
{ID: 3, SentenceID: 5}, // 重复
|
||||||
|
{ID: 4, SentenceID: 7},
|
||||||
|
}
|
||||||
|
got := sentenceIDsFromRelations(rels)
|
||||||
|
if len(got) != 2 {
|
||||||
|
t.Fatalf("应得 2 个去重后的句子 id,实际 %v", got)
|
||||||
|
}
|
||||||
|
if got[0] != 5 || got[1] != 7 {
|
||||||
|
t.Fatalf("句子 id 或顺序不对: %v", got)
|
||||||
|
}
|
||||||
|
if n := sentenceIDsFromRelations(nil); n != nil {
|
||||||
|
t.Fatalf("空输入应返回 nil,实际 %v", n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMediaBlocksHeldByDocumentSurviveDeletion(t *testing.T) {
|
||||||
|
// 文档持有的一等块把内容钉住;文档被删后块随之消失,内容才可回收。
|
||||||
|
_, _, ms := newGraphMediaAgent(t)
|
||||||
|
|
||||||
|
digest, err := ms.Put([]byte("doc image"), media.Item{MIME: "image/png"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
dir := t.TempDir()
|
||||||
|
ds := document.NewStore(filepath.Join(dir, "docs"), memory.TokenizeWords)
|
||||||
|
if err := ds.Start(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer ds.Stop()
|
||||||
|
|
||||||
|
it, _ := ms.Stat(digest)
|
||||||
|
doc := &document.Doc{
|
||||||
|
ID: "doc_1", Summary: "带图的文档", Content: "正文",
|
||||||
|
Blocks: []memory.MemoryBlock{{ID: "blk_doc_1", Modality: memory.BlockImage,
|
||||||
|
PayloadDigest: it.Digest, MIME: it.MIME, Size: it.Size}},
|
||||||
|
}
|
||||||
|
if err := ds.Insert(doc); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 文档仍持有块 → 内容在
|
||||||
|
if _, err := ms.Stat(digest); err != nil {
|
||||||
|
t.Fatal("有文档块持有内容时不该被清")
|
||||||
|
}
|
||||||
|
|
||||||
|
// 删除文档 → 一并删除其内容(与文本块一致:删块即删内容)
|
||||||
|
ds.Remove(doc.ID)
|
||||||
|
if blocks := ds.Blocks(); len(blocks) != 0 {
|
||||||
|
t.Fatalf("删除文档后不该还有块,实际 %+v", blocks)
|
||||||
|
}
|
||||||
|
if err := ms.Delete(digest); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := ms.Stat(digest); err == nil {
|
||||||
|
t.Fatal("删除后内容应已移除")
|
||||||
|
}
|
||||||
|
}
|
||||||
615
internal/agent/core/inputunify_test.go
Normal file
615
internal/agent/core/inputunify_test.go
Normal file
@ -0,0 +1,615 @@
|
|||||||
|
package core
|
||||||
|
|
||||||
|
import (
|
||||||
|
"path/filepath"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
agentAPI "gitcode.com/JianFeeeee/HomeAgent/internal/agent/api"
|
||||||
|
agentIO "gitcode.com/JianFeeeee/HomeAgent/internal/agent/io"
|
||||||
|
"gitcode.com/JianFeeeee/HomeAgent/internal/memory"
|
||||||
|
"gitcode.com/JianFeeeee/HomeAgent/internal/memory/document"
|
||||||
|
"gitcode.com/JianFeeeee/HomeAgent/internal/memory/media"
|
||||||
|
pubsdk "gitcode.com/JianFeeeee/homeagent-sdk/sdk"
|
||||||
|
)
|
||||||
|
|
||||||
|
// 统一输入主干(processInput / resolveInput / injectedBlocks)与
|
||||||
|
// 模型可调用工具的媒体接线测试。
|
||||||
|
//
|
||||||
|
// 这一层此前的结构性缺陷:text 与 image/audio 各有一个 process 函数,
|
||||||
|
// 媒体那条缺了去重、no_memory、通道 Cleaner、中断语义、EventRawInput 五项。
|
||||||
|
// 归一成一条主干后,这些行为对所有模态一致——下面的断言就是这个不变量。
|
||||||
|
|
||||||
|
func newInputTestAgent(t *testing.T) (*Agent, *media.Store) {
|
||||||
|
t.Helper()
|
||||||
|
dir := t.TempDir()
|
||||||
|
|
||||||
|
ms, err := media.New(filepath.Join(dir, "media"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("media.New: %v", err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { ms.Close() })
|
||||||
|
|
||||||
|
return &Agent{mediaStore: ms}, ms
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------- injectedBlocks ----------
|
||||||
|
|
||||||
|
// 内核内部注入直接给 []agentAPI.ContentBlock;经公共 SDK 的 IOInjector 过来的是
|
||||||
|
// []pubsdk.ContentBlock。两者字段一致但 Go 不会自动转换,只认一种的后果是
|
||||||
|
// 另一种被静默丢弃——插件注入的图到 payload 就断了,且不报错。
|
||||||
|
func TestInjectedBlocks_AcceptsBothStaticTypes(t *testing.T) {
|
||||||
|
t.Run("内核类型", func(t *testing.T) {
|
||||||
|
blocks, kind := injectedBlocks(map[string]interface{}{
|
||||||
|
"media_blocks": []agentAPI.ContentBlock{
|
||||||
|
{Type: "text", Text: "看图"},
|
||||||
|
{Type: "image_url", ImageURL: &agentAPI.ImageURL{URL: "data:image/png;base64,AAA"}},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if len(blocks) != 2 {
|
||||||
|
t.Fatalf("blocks = %d,期望 2", len(blocks))
|
||||||
|
}
|
||||||
|
if kind != "image" {
|
||||||
|
t.Errorf("mediaType = %q,期望 image", kind)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("公共SDK类型", func(t *testing.T) {
|
||||||
|
blocks, kind := injectedBlocks(map[string]interface{}{
|
||||||
|
"media_blocks": []pubsdk.ContentBlock{
|
||||||
|
{Type: "text", Text: "听音频"},
|
||||||
|
{Type: "audio_url", AudioURL: &pubsdk.AudioURL{URL: "data:audio/wav;base64,BBB"}},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if len(blocks) != 2 {
|
||||||
|
t.Fatalf("blocks = %d,期望 2(公共 SDK 类型被静默丢弃)", len(blocks))
|
||||||
|
}
|
||||||
|
if kind != "audio" {
|
||||||
|
t.Errorf("mediaType = %q,期望 audio", kind)
|
||||||
|
}
|
||||||
|
// 转换必须保留 URL,否则块到了模型手上是空的
|
||||||
|
if blocks[1].AudioURL == nil || blocks[1].AudioURL.URL != "data:audio/wav;base64,BBB" {
|
||||||
|
t.Errorf("AudioURL 转换丢失: %+v", blocks[1].AudioURL)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("图优先于音频", func(t *testing.T) {
|
||||||
|
_, kind := injectedBlocks(map[string]interface{}{
|
||||||
|
"media_blocks": []agentAPI.ContentBlock{
|
||||||
|
{Type: "audio_url", AudioURL: &agentAPI.AudioURL{URL: "a"}},
|
||||||
|
{Type: "image_url", ImageURL: &agentAPI.ImageURL{URL: "b"}},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if kind != "image" {
|
||||||
|
t.Errorf("mediaType = %q,期望 image", kind)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("无媒体块", func(t *testing.T) {
|
||||||
|
blocks, kind := injectedBlocks(map[string]interface{}{"content": "纯文本"})
|
||||||
|
if blocks != nil || kind != "" {
|
||||||
|
t.Errorf("无 media_blocks 时应返回 (nil,\"\"),实际 (%v,%q)", blocks, kind)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("ImageURL 的 Detail 透传", func(t *testing.T) {
|
||||||
|
blocks, _ := injectedBlocks(map[string]interface{}{
|
||||||
|
"media_blocks": []pubsdk.ContentBlock{
|
||||||
|
{Type: "image_url", ImageURL: &pubsdk.ImageURL{URL: "u", Detail: "high"}},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if len(blocks) != 1 || blocks[0].ImageURL.Detail != "high" {
|
||||||
|
t.Errorf("Detail 未透传: %+v", blocks)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------- resolveInput ----------
|
||||||
|
|
||||||
|
func TestResolveInput_UnifiesAllModalities(t *testing.T) {
|
||||||
|
a, _ := newInputTestAgent(t)
|
||||||
|
|
||||||
|
t.Run("用户上传图片", func(t *testing.T) {
|
||||||
|
in, ok := a.resolveInput(&agentIO.InputEvent{
|
||||||
|
Source: "qq",
|
||||||
|
Type: "image",
|
||||||
|
Payload: map[string]interface{}{"data": "AAAA", "mime": "image/png"},
|
||||||
|
})
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("图片输入被判为无效")
|
||||||
|
}
|
||||||
|
if in.mediaType != "image" || in.captureTool != "input_image" {
|
||||||
|
t.Errorf("mediaType=%q captureTool=%q", in.mediaType, in.captureTool)
|
||||||
|
}
|
||||||
|
if in.text == "" {
|
||||||
|
t.Error("纯媒体输入应有 alt 文案作为文本落点")
|
||||||
|
}
|
||||||
|
if len(in.blocks) == 0 {
|
||||||
|
t.Error("图片应转成内容块")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("插件注入的媒体", func(t *testing.T) {
|
||||||
|
in, ok := a.resolveInput(&agentIO.InputEvent{
|
||||||
|
Source: "myplugin",
|
||||||
|
Type: "text",
|
||||||
|
Payload: map[string]interface{}{
|
||||||
|
"content": "帮我看看这张图",
|
||||||
|
"media_blocks": []pubsdk.ContentBlock{
|
||||||
|
{Type: "image_url", ImageURL: &pubsdk.ImageURL{URL: "data:image/png;base64,AAA"}},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("带媒体的文本输入被判为无效")
|
||||||
|
}
|
||||||
|
if in.text != "帮我看看这张图" {
|
||||||
|
t.Errorf("text = %q", in.text)
|
||||||
|
}
|
||||||
|
if len(in.blocks) != 1 || in.mediaType != "image" {
|
||||||
|
t.Errorf("blocks=%d mediaType=%q —— 插件注入的媒体到 payload 就断了", len(in.blocks), in.mediaType)
|
||||||
|
}
|
||||||
|
if in.captureTool != "inject_myplugin" {
|
||||||
|
t.Errorf("captureTool = %q,期望带来源便于溯源", in.captureTool)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("只带图不带字也合法", func(t *testing.T) {
|
||||||
|
_, ok := a.resolveInput(&agentIO.InputEvent{
|
||||||
|
Source: "myplugin",
|
||||||
|
Type: "text",
|
||||||
|
Payload: map[string]interface{}{
|
||||||
|
"media_blocks": []agentAPI.ContentBlock{
|
||||||
|
{Type: "image_url", ImageURL: &agentAPI.ImageURL{URL: "u"}},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if !ok {
|
||||||
|
t.Error("只带媒体不带文本应视为有效输入(插件注入常这样)")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("文本与媒体都空才无效", func(t *testing.T) {
|
||||||
|
if _, ok := a.resolveInput(&agentIO.InputEvent{
|
||||||
|
Source: "cli",
|
||||||
|
Type: "text",
|
||||||
|
Payload: map[string]interface{}{"content": ""},
|
||||||
|
}); ok {
|
||||||
|
t.Error("空输入应被拒")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("纯文本", func(t *testing.T) {
|
||||||
|
in, ok := a.resolveInput(&agentIO.InputEvent{
|
||||||
|
Source: "cli",
|
||||||
|
Type: "text",
|
||||||
|
Payload: map[string]interface{}{"content": "你好"},
|
||||||
|
})
|
||||||
|
if !ok || in.text != "你好" || len(in.blocks) != 0 || in.mediaType != "" {
|
||||||
|
t.Errorf("纯文本路径异常: ok=%v in=%+v", ok, in)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------- 模型工具侧:memory_digests 结构化传递 ----------
|
||||||
|
|
||||||
|
// 模型只知道 digest(从对话或 memory_recall 的「关联媒体」读到)。
|
||||||
|
// 它不再需要自己拼任何标记:digest 作为结构化字段随三元组提交。
|
||||||
|
func TestResolveMediaDigestsAndNoMarkerText(t *testing.T) {
|
||||||
|
a, ms := newInputTestAgent(t)
|
||||||
|
digest, err := ms.Put([]byte("marker-bytes"), media.Item{MIME: "image/png"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Put: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Run("短digest补全", func(t *testing.T) {
|
||||||
|
got := a.resolveMediaDigests([]string{digest[:12]})
|
||||||
|
if len(got) != 1 || got[0] != digest {
|
||||||
|
t.Fatalf("短 digest 应补全为完整 digest,得到 %v", got)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("无法解析的digest被丢弃", func(t *testing.T) {
|
||||||
|
if got := a.resolveMediaDigests([]string{"ffffffffffff"}); len(got) != 0 {
|
||||||
|
t.Errorf("不存在的 digest 不该保留: %v", got)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("无媒体存储时返回nil", func(t *testing.T) {
|
||||||
|
bare := &Agent{}
|
||||||
|
if got := bare.resolveMediaDigests([]string{digest}); got != nil {
|
||||||
|
t.Errorf("无媒体存储时应返回 nil: %v", got)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// 句子文本必须保持原样:媒体归属走结构化块边,不往文本里贴 marker。
|
||||||
|
func TestMemoryCommit_DoesNotPolluteSentenceText(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
g, err := memory.NewGraphDB(filepath.Join(dir, "graph.db"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer g.Close()
|
||||||
|
ms, err := media.New(filepath.Join(dir, "media"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer ms.Close()
|
||||||
|
a := &Agent{memory: g, mediaStore: ms}
|
||||||
|
|
||||||
|
digest, _ := ms.Put([]byte("clean-sentence"), media.Item{MIME: "image/png"})
|
||||||
|
|
||||||
|
sentence := "用户发来一张图。"
|
||||||
|
triples := []memory.Triple{{
|
||||||
|
Subject: "用户", Relation: "发来", Object: "图片",
|
||||||
|
SentenceText: sentence,
|
||||||
|
MediaDigests: a.resolveMediaDigests([]string{digest[:12]}),
|
||||||
|
}}
|
||||||
|
if _, _, _, err := a.commitTriplesWithMedia(triples, "s1", 0, nil); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
res, err := a.memory.Recall([]string{"用户"}, nil, 2, "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(res.Relations) == 0 {
|
||||||
|
t.Fatal("召回为空")
|
||||||
|
}
|
||||||
|
if res.Relations[0].SentenceText != sentence {
|
||||||
|
t.Errorf("句子文本被污染: %q", res.Relations[0].SentenceText)
|
||||||
|
}
|
||||||
|
blocks, err := a.memory.BlocksForNode("sentence", strconv.FormatInt(res.Relations[0].SentenceID, 10))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(blocks) != 1 || blocks[0].PayloadDigest != digest {
|
||||||
|
t.Errorf("块应挂到句子,实际 %+v", blocks)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------- resolveMediaDigests ----------
|
||||||
|
|
||||||
|
func TestResolveMediaDigests(t *testing.T) {
|
||||||
|
a, ms := newInputTestAgent(t)
|
||||||
|
d1, _ := ms.Put([]byte("one"), media.Item{MIME: "image/png"})
|
||||||
|
d2, _ := ms.Put([]byte("two"), media.Item{MIME: "image/png"})
|
||||||
|
|
||||||
|
got := a.resolveMediaDigests([]string{d1[:10], d2, d1, "ffffffffffff"})
|
||||||
|
if len(got) != 2 {
|
||||||
|
t.Fatalf("got = %v,期望 2 条(去重 + 丢弃无法解析的)", got)
|
||||||
|
}
|
||||||
|
for _, d := range got {
|
||||||
|
if len(d) != 64 {
|
||||||
|
t.Errorf("应返回完整 digest,实际 %q", d)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if a.resolveMediaDigests(nil) != nil {
|
||||||
|
t.Error("空输入应返回 nil")
|
||||||
|
}
|
||||||
|
bare := &Agent{}
|
||||||
|
if bare.resolveMediaDigests([]string{d1}) != nil {
|
||||||
|
t.Error("无媒体存储时应返回 nil")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------- 文档持有的一等记忆块 ----------
|
||||||
|
|
||||||
|
func TestDocCommit_StoresBlocks(t *testing.T) {
|
||||||
|
// doc_commit 带 media_digests 时,媒体应作为一等块直接存在文档上,
|
||||||
|
// 并随 doc 一起持久化(不再靠 media_refs 保活)。
|
||||||
|
dir := t.TempDir()
|
||||||
|
ds := document.NewStore(filepath.Join(dir, "docs"), memory.TokenizeWords)
|
||||||
|
if err := ds.Start(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer ds.Stop()
|
||||||
|
|
||||||
|
ms, err := media.New(filepath.Join(dir, "media"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer ms.Close()
|
||||||
|
|
||||||
|
d1, _ := ms.Put([]byte("doc-one"), media.Item{MIME: "image/png"})
|
||||||
|
d2, _ := ms.Put([]byte("doc-two"), media.Item{MIME: "image/png"})
|
||||||
|
|
||||||
|
doc := &document.Doc{ID: "doc_x", Summary: "s", Content: "c"}
|
||||||
|
for _, d := range []string{d1, d2} {
|
||||||
|
if b, ok := (&Agent{mediaStore: ms}).blockFromDigest(d); ok {
|
||||||
|
doc.Blocks = append(doc.Blocks, b)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := ds.Insert(doc); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
blocks := ds.Blocks()
|
||||||
|
if len(blocks) != 2 {
|
||||||
|
t.Fatalf("文档应持有 2 个块,实际 %d", len(blocks))
|
||||||
|
}
|
||||||
|
seen := map[string]bool{}
|
||||||
|
for _, b := range blocks {
|
||||||
|
seen[b.PayloadDigest] = true
|
||||||
|
}
|
||||||
|
if !seen[d1] || !seen[d2] {
|
||||||
|
t.Errorf("块 digest 不对: %+v", blocks)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------- 文档持有块标签(doc_query 展示用) ----------
|
||||||
|
|
||||||
|
func TestBlockLabelsForDoc(t *testing.T) {
|
||||||
|
a, ms := newInputTestAgent(t)
|
||||||
|
digest, _ := ms.Put([]byte("ctx-bytes"), media.Item{MIME: "image/png"})
|
||||||
|
b, ok := a.blockFromDigest(digest)
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("blockFromDigest 失败")
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Run("从文档持有的一等块渲染", func(t *testing.T) {
|
||||||
|
got := a.blockLabelsForDoc(&document.Doc{ID: "doc_1", Blocks: []memory.MemoryBlock{b}})
|
||||||
|
if !strings.Contains(got, shortDigest(digest)) {
|
||||||
|
t.Errorf("标签应含短 digest: %q", got)
|
||||||
|
}
|
||||||
|
if !strings.Contains(got, "image/png") {
|
||||||
|
t.Errorf("标签应含 MIME: %q", got)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("无块时为空", func(t *testing.T) {
|
||||||
|
if got := a.blockLabelsForDoc(&document.Doc{ID: "doc_x", Content: "普通正文"}); got != "" {
|
||||||
|
t.Errorf("应返回空串,实际 %q", got)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("无媒体存储", func(t *testing.T) {
|
||||||
|
bare := &Agent{}
|
||||||
|
if got := bare.blockLabelsForDoc(&document.Doc{ID: "doc_x"}); got != "" {
|
||||||
|
t.Errorf("无媒体存储时应返回空串,实际 %q", got)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------- mediaLabel ----------
|
||||||
|
|
||||||
|
// 媒体标签的唯一生成处:只含 MIME 与短 digest,不含任何生成的描述。
|
||||||
|
func TestMediaLabel(t *testing.T) {
|
||||||
|
a, ms := newInputTestAgent(t)
|
||||||
|
_ = a
|
||||||
|
|
||||||
|
digest, _ := ms.Put([]byte("labelled"), media.Item{MIME: "image/png"})
|
||||||
|
it, err := ms.Stat(digest)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
got := mediaLabel(it)
|
||||||
|
if !strings.Contains(got, "image/png") {
|
||||||
|
t.Errorf("标签应含 MIME: %q", got)
|
||||||
|
}
|
||||||
|
if !strings.Contains(got, shortDigest(digest)) {
|
||||||
|
t.Errorf("必须带短 digest 供反查: %q", got)
|
||||||
|
}
|
||||||
|
if got := mediaLabel(nil); got != "" {
|
||||||
|
t.Errorf("nil 应返回空串,实际 %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------- 模型工具端到端:memory_commit / doc_commit / doc_query ----------
|
||||||
|
|
||||||
|
func newToolTestAgent(t *testing.T) (*Agent, *media.Store) {
|
||||||
|
t.Helper()
|
||||||
|
dir := t.TempDir()
|
||||||
|
|
||||||
|
g, err := memory.NewGraphDB(filepath.Join(dir, "graph.db"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewGraphDB: %v", err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { g.Close() })
|
||||||
|
|
||||||
|
ds := document.NewStore(filepath.Join(dir, "documents"), memory.TokenizeWords)
|
||||||
|
if err := ds.Start(); err != nil {
|
||||||
|
t.Fatalf("doc store: %v", err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { ds.Stop() })
|
||||||
|
|
||||||
|
ms, err := media.New(filepath.Join(dir, "media"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("media.New: %v", err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { ms.Close() })
|
||||||
|
|
||||||
|
emb := memory.NewStaticEmbedder("")
|
||||||
|
a := &Agent{
|
||||||
|
id: "tester",
|
||||||
|
memory: g,
|
||||||
|
docStore: ds,
|
||||||
|
mediaStore: ms,
|
||||||
|
context: NewRelevanceContext("", emb),
|
||||||
|
}
|
||||||
|
return a, ms
|
||||||
|
}
|
||||||
|
|
||||||
|
// memory_commit 带 media_digests:三元组入库后必须能从句子反查回那份字节。
|
||||||
|
func TestToolMemoryCommit_BindsMedia(t *testing.T) {
|
||||||
|
a, ms := newToolTestAgent(t)
|
||||||
|
digest, _ := ms.Put([]byte("commit-bytes"), media.Item{MIME: "image/png"})
|
||||||
|
|
||||||
|
out := a.executeMemoryTool(agentAPI.ToolCall{
|
||||||
|
Name: "memory_commit",
|
||||||
|
Arguments: map[string]interface{}{
|
||||||
|
"triples": []interface{}{
|
||||||
|
map[string]interface{}{
|
||||||
|
"subject": "配色方案",
|
||||||
|
"relation": "参考",
|
||||||
|
"object": "三色带图",
|
||||||
|
"media_digests": []interface{}{digest[:12]},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if !strings.Contains(out, "关联") {
|
||||||
|
t.Errorf("返回值应告知模型媒体已关联: %q", out)
|
||||||
|
}
|
||||||
|
|
||||||
|
res, err := a.memory.Recall([]string{"配色方案"}, nil, 2, "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Recall: %v", err)
|
||||||
|
}
|
||||||
|
if len(res.Relations) == 0 || res.Relations[0].SentenceID == 0 {
|
||||||
|
t.Fatal("没有句子落点 —— 媒体引用无从挂起")
|
||||||
|
}
|
||||||
|
blocks, err := a.memory.BlocksForNode("sentence", strconv.FormatInt(res.Relations[0].SentenceID, 10))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("BlocksForNode: %v", err)
|
||||||
|
}
|
||||||
|
if len(blocks) != 1 || blocks[0].PayloadDigest != digest {
|
||||||
|
t.Errorf("句子块 = %+v,期望 [%s]", blocks, digest)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 不带 media_digests 时行为与本特性上线前一致(不多写句子、不报错)。
|
||||||
|
func TestToolMemoryCommit_WithoutMedia(t *testing.T) {
|
||||||
|
a, _ := newToolTestAgent(t)
|
||||||
|
out := a.executeMemoryTool(agentAPI.ToolCall{
|
||||||
|
Name: "memory_commit",
|
||||||
|
Arguments: map[string]interface{}{
|
||||||
|
"triples": []interface{}{
|
||||||
|
map[string]interface{}{"subject": "甲方", "relation": "签署", "object": "合同"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if strings.Contains(out, "失败") {
|
||||||
|
t.Errorf("普通提交不该失败: %q", out)
|
||||||
|
}
|
||||||
|
if strings.Contains(out, "关联") {
|
||||||
|
t.Errorf("无媒体时不该提媒体: %q", out)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// sentence_text 必须透传:丢了它,图谱就回不到原文。
|
||||||
|
func TestToolMemoryCommit_CarriesSentenceText(t *testing.T) {
|
||||||
|
a, _ := newToolTestAgent(t)
|
||||||
|
a.executeMemoryTool(agentAPI.ToolCall{
|
||||||
|
Name: "memory_commit",
|
||||||
|
Arguments: map[string]interface{}{
|
||||||
|
"triples": []interface{}{
|
||||||
|
map[string]interface{}{
|
||||||
|
"subject": "李四",
|
||||||
|
"relation": "住在",
|
||||||
|
"object": "杭州",
|
||||||
|
"sentence_text": "李四搬到杭州已经三年了。",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
res, _ := a.memory.Recall([]string{"李四"}, nil, 2, "")
|
||||||
|
if len(res.Relations) == 0 {
|
||||||
|
t.Fatal("召回为空")
|
||||||
|
}
|
||||||
|
if res.Relations[0].SentenceText != "李四搬到杭州已经三年了。" {
|
||||||
|
t.Errorf("SentenceText = %q", res.Relations[0].SentenceText)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// doc_commit 带 media_digests:媒体成为文档直接持有的一等块;正文保持原样。
|
||||||
|
func TestToolDocCommit_BindsMedia(t *testing.T) {
|
||||||
|
a, ms := newToolTestAgent(t)
|
||||||
|
digest, _ := ms.Put([]byte("doc-commit-bytes"), media.Item{MIME: "image/png"})
|
||||||
|
|
||||||
|
out := a.executeDocTool(agentAPI.ToolCall{
|
||||||
|
Name: "doc_commit",
|
||||||
|
Arguments: map[string]interface{}{
|
||||||
|
"content": "这是一篇带图的笔记正文。",
|
||||||
|
"summary": "带图笔记",
|
||||||
|
"media_digests": []interface{}{digest[:12]},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if !strings.Contains(out, "关联") {
|
||||||
|
t.Errorf("返回值应告知模型媒体已关联: %q", out)
|
||||||
|
}
|
||||||
|
|
||||||
|
docs := a.docStore.RecentDocs(5)
|
||||||
|
if len(docs) == 0 {
|
||||||
|
t.Fatal("文档未写入")
|
||||||
|
}
|
||||||
|
d := docs[0]
|
||||||
|
if strings.Contains(d.Content, "image/png") {
|
||||||
|
t.Errorf("正文不该被媒体标记污染: %q", d.Content)
|
||||||
|
}
|
||||||
|
var held bool
|
||||||
|
for _, b := range d.Blocks {
|
||||||
|
if b.PayloadDigest == digest {
|
||||||
|
held = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !held {
|
||||||
|
t.Errorf("文档应持有一等记忆块 [%s],实际 %+v", digest, d.Blocks)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// doc_query 必须把媒体说明附在返回值里,否则模型检索到带图文档也不知道有图。
|
||||||
|
func TestToolDocQuery_ShowsMedia(t *testing.T) {
|
||||||
|
a, ms := newToolTestAgent(t)
|
||||||
|
digest, _ := ms.Put([]byte("query-bytes"), media.Item{MIME: "image/png"})
|
||||||
|
|
||||||
|
a.executeDocTool(agentAPI.ToolCall{
|
||||||
|
Name: "doc_commit",
|
||||||
|
Arguments: map[string]interface{}{
|
||||||
|
"content": "紫蓝红三色带配色说明正文",
|
||||||
|
"summary": "紫蓝红三色带",
|
||||||
|
"media_digests": []interface{}{digest},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
a.executeDocTool(agentAPI.ToolCall{
|
||||||
|
Name: "doc_query",
|
||||||
|
Arguments: map[string]interface{}{"query": "紫蓝红三色带 配色说明", "top_k": float64(3)},
|
||||||
|
})
|
||||||
|
|
||||||
|
// 正文进的是 cold_storage 事件(工具返回值只给引用编号),媒体说明也在那里。
|
||||||
|
var found bool
|
||||||
|
for _, e := range a.context.Recent(10) {
|
||||||
|
if strings.Contains(e.Response, shortDigest(digest)) {
|
||||||
|
found = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
t.Error("doc_query 未把媒体说明带进上下文 —— 模型不知道这篇文档带过图")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 无媒体存储时三个工具的行为与本特性上线前完全一致。
|
||||||
|
func TestTools_NilMediaStoreDegrades(t *testing.T) {
|
||||||
|
a, _ := newToolTestAgent(t)
|
||||||
|
a.mediaStore = nil
|
||||||
|
|
||||||
|
out := a.executeMemoryTool(agentAPI.ToolCall{
|
||||||
|
Name: "memory_commit",
|
||||||
|
Arguments: map[string]interface{}{
|
||||||
|
"triples": []interface{}{
|
||||||
|
map[string]interface{}{
|
||||||
|
"subject": "无存储", "relation": "仍可", "object": "提交",
|
||||||
|
"media_digests": []interface{}{"aabbccddeeff"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if strings.Contains(out, "失败") {
|
||||||
|
t.Errorf("无媒体存储时提交不该失败: %q", out)
|
||||||
|
}
|
||||||
|
|
||||||
|
out = a.executeDocTool(agentAPI.ToolCall{
|
||||||
|
Name: "doc_commit",
|
||||||
|
Arguments: map[string]interface{}{
|
||||||
|
"content": "无媒体存储的文档",
|
||||||
|
"media_digests": []interface{}{"aabbccddeeff"},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if strings.Contains(out, "失败") {
|
||||||
|
t.Errorf("无媒体存储时文档写入不该失败: %q", out)
|
||||||
|
}
|
||||||
|
}
|
||||||
545
internal/agent/core/medialive_test.go
Normal file
545
internal/agent/core/medialive_test.go
Normal file
@ -0,0 +1,545 @@
|
|||||||
|
//go:build medialive
|
||||||
|
|
||||||
|
// 媒体记忆自动触发链的集成测试。
|
||||||
|
//
|
||||||
|
// 与其他媒体测试的区别:**不手工调用任何一步**。这里只做两件事——
|
||||||
|
// 往 IOManager 注入一个 image 事件,然后等。之后全部由生产代码自己走:
|
||||||
|
//
|
||||||
|
// processMediaInput → captureBlockMedia(入 CAS)
|
||||||
|
// → Prune(L0→L2 块迁移)
|
||||||
|
// → describePendingMedia(真实视觉模型生成描述)
|
||||||
|
// → archiveColdDocs → commitTriplesWithMedia → bindSentenceBlocks(L2→L3)
|
||||||
|
// → 第二轮提问,验证 agent 真能召回
|
||||||
|
//
|
||||||
|
// 为什么必须这样测:单测能证明每个函数正确,却证明不了它**被接上了**——
|
||||||
|
// 手工注入 store 的单测全绿而生产链路断开,是本文件要拦的典型缺陷。
|
||||||
|
//
|
||||||
|
// 需要真实 LLM,因此加 medialive build tag,默认 go test 不跑:
|
||||||
|
//
|
||||||
|
// MEDIALIVE_BASE_URL=http://127.0.0.1:8081/v1 \
|
||||||
|
// MEDIALIVE_API_KEY=sk-xxx \
|
||||||
|
// MEDIALIVE_MODEL=claude-opus-5 \
|
||||||
|
// MEDIALIVE_ADAPTER=openai \
|
||||||
|
// go test -tags medialive ./internal/agent/core/ -run TestMediaLive -v -timeout 20m
|
||||||
|
//
|
||||||
|
// 源、模型、密钥全部由调用方显式指定,测试自己不猜任何默认值——
|
||||||
|
// 猜一个默认端点会让测试在别人机器上打到意料之外的服务。
|
||||||
|
package core
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"compress/zlib"
|
||||||
|
"encoding/binary"
|
||||||
|
"fmt"
|
||||||
|
"hash/crc32"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
agentAPI "gitcode.com/JianFeeeee/HomeAgent/internal/agent/api"
|
||||||
|
agentIO "gitcode.com/JianFeeeee/HomeAgent/internal/agent/io"
|
||||||
|
luaVM "gitcode.com/JianFeeeee/HomeAgent/internal/lua"
|
||||||
|
"gitcode.com/JianFeeeee/HomeAgent/internal/memory"
|
||||||
|
"gitcode.com/JianFeeeee/HomeAgent/internal/memory/document"
|
||||||
|
"gitcode.com/JianFeeeee/HomeAgent/internal/memory/media"
|
||||||
|
"gitcode.com/JianFeeeee/HomeAgent/pkg/types"
|
||||||
|
)
|
||||||
|
|
||||||
|
// liveCfg 是调用方通过环境变量显式提供的 LLM 源配置。
|
||||||
|
type liveCfg struct {
|
||||||
|
baseURL string
|
||||||
|
apiKey string
|
||||||
|
model string
|
||||||
|
adapter string
|
||||||
|
}
|
||||||
|
|
||||||
|
// requireLiveCfg 读取环境变量;缺任何一项就 Skip 而非猜默认值。
|
||||||
|
//
|
||||||
|
// 刻意不提供 fallback:一个猜出来的 base_url 可能打到调用者机器上
|
||||||
|
// 完全不相干的服务,而测试会把那次调用的失败报成"媒体记忆有问题"。
|
||||||
|
func requireLiveCfg(t *testing.T) liveCfg {
|
||||||
|
t.Helper()
|
||||||
|
c := liveCfg{
|
||||||
|
baseURL: os.Getenv("MEDIALIVE_BASE_URL"),
|
||||||
|
apiKey: os.Getenv("MEDIALIVE_API_KEY"),
|
||||||
|
model: os.Getenv("MEDIALIVE_MODEL"),
|
||||||
|
adapter: os.Getenv("MEDIALIVE_ADAPTER"),
|
||||||
|
}
|
||||||
|
var missing []string
|
||||||
|
if c.baseURL == "" {
|
||||||
|
missing = append(missing, "MEDIALIVE_BASE_URL")
|
||||||
|
}
|
||||||
|
if c.apiKey == "" {
|
||||||
|
missing = append(missing, "MEDIALIVE_API_KEY")
|
||||||
|
}
|
||||||
|
if c.model == "" {
|
||||||
|
missing = append(missing, "MEDIALIVE_MODEL")
|
||||||
|
}
|
||||||
|
if c.adapter == "" {
|
||||||
|
missing = append(missing, "MEDIALIVE_ADAPTER")
|
||||||
|
}
|
||||||
|
if len(missing) > 0 {
|
||||||
|
t.Skipf("缺少环境变量 %s——本测试要求调用方显式指定源/模型/密钥,不使用任何默认值",
|
||||||
|
strings.Join(missing, ", "))
|
||||||
|
}
|
||||||
|
return c
|
||||||
|
}
|
||||||
|
|
||||||
|
// livePNG 造一张横向三色带真 PNG(手工拼 IHDR/IDAT/IEND)。
|
||||||
|
//
|
||||||
|
// 用可辨认的纯色而非随机字节:断言要能检查"模型是否真的看到了内容",
|
||||||
|
// 随机噪声无法产生可验证的描述。
|
||||||
|
func livePNG(t *testing.T, w, h int, colors [][3]byte) []byte {
|
||||||
|
t.Helper()
|
||||||
|
chunk := func(typ string, data []byte) []byte {
|
||||||
|
var b bytes.Buffer
|
||||||
|
if err := binary.Write(&b, binary.BigEndian, uint32(len(data))); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
body := append([]byte(typ), data...)
|
||||||
|
b.Write(body)
|
||||||
|
if err := binary.Write(&b, binary.BigEndian, crc32.ChecksumIEEE(body)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return b.Bytes()
|
||||||
|
}
|
||||||
|
var raw bytes.Buffer
|
||||||
|
for y := 0; y < h; y++ {
|
||||||
|
raw.WriteByte(0) // filter type: none
|
||||||
|
c := colors[y*len(colors)/h]
|
||||||
|
for x := 0; x < w; x++ {
|
||||||
|
raw.Write(c[:])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var comp bytes.Buffer
|
||||||
|
zw := zlib.NewWriter(&comp)
|
||||||
|
if _, err := zw.Write(raw.Bytes()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
zw.Close()
|
||||||
|
|
||||||
|
var ihdr bytes.Buffer
|
||||||
|
binary.Write(&ihdr, binary.BigEndian, uint32(w))
|
||||||
|
binary.Write(&ihdr, binary.BigEndian, uint32(h))
|
||||||
|
ihdr.Write([]byte{8, 2, 0, 0, 0}) // 8-bit truecolor
|
||||||
|
|
||||||
|
var out bytes.Buffer
|
||||||
|
out.Write([]byte{0x89, 'P', 'N', 'G', '\r', '\n', 0x1a, '\n'})
|
||||||
|
out.Write(chunk("IHDR", ihdr.Bytes()))
|
||||||
|
out.Write(chunk("IDAT", comp.Bytes()))
|
||||||
|
out.Write(chunk("IEND", nil))
|
||||||
|
return out.Bytes()
|
||||||
|
}
|
||||||
|
|
||||||
|
// liveEnv 是一套完整但完全独立的 agent 运行环境。
|
||||||
|
type liveEnv struct {
|
||||||
|
agent *Agent
|
||||||
|
io *agentIO.IOManager
|
||||||
|
mediaSt *media.Store
|
||||||
|
docStore *document.Store
|
||||||
|
graph *memory.GraphDB
|
||||||
|
dir string
|
||||||
|
}
|
||||||
|
|
||||||
|
// newLiveEnv 构造真 Agent:真 provider、真 CAS、真图库、真文档库。
|
||||||
|
//
|
||||||
|
// 不注册任何插件:本测试关心记忆链路,插件会引入无关的外部副作用
|
||||||
|
// (网络轮询、写文件),而且生产插件目录里的进程不该被测试碰到。
|
||||||
|
func newLiveEnv(t *testing.T, c liveCfg) *liveEnv {
|
||||||
|
t.Helper()
|
||||||
|
dir := t.TempDir()
|
||||||
|
|
||||||
|
vm := luaVM.NewVM(filepath.Join(dir, "adapters"))
|
||||||
|
if err := vm.Start(); err != nil {
|
||||||
|
t.Fatalf("lua vm: %v", err)
|
||||||
|
}
|
||||||
|
t.Cleanup(vm.Stop)
|
||||||
|
|
||||||
|
// Vision: true —— 能力是声明的,不是探测的。网关可能静默剥离
|
||||||
|
// image_url 后仍返回 200,从响应无法推断它到底看见了没有。
|
||||||
|
prov := agentAPI.NewLuaAdaptedProvider(agentAPI.BaseConfig{
|
||||||
|
Model: c.model, BaseURL: c.baseURL, APIKey: c.apiKey,
|
||||||
|
MaxTokens: 1200, Temperature: 0.3, Vision: true,
|
||||||
|
}, vm, "medialive", c.adapter)
|
||||||
|
|
||||||
|
pm := agentAPI.NewProviderManager()
|
||||||
|
pm.Register("medialive", prov)
|
||||||
|
if err := pm.SetDefault("medialive"); err != nil {
|
||||||
|
t.Fatalf("set default provider: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
ms, err := media.New(filepath.Join(dir, "media"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("media store: %v", err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { ms.Close() })
|
||||||
|
|
||||||
|
graph, err := memory.NewGraphDB(filepath.Join(dir, "graph.db"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("graph: %v", err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { graph.Close() })
|
||||||
|
|
||||||
|
docStore := document.NewStore(filepath.Join(dir, "docs"), memory.TokenizeWords)
|
||||||
|
if err := docStore.Start(); err != nil {
|
||||||
|
t.Fatalf("doc store: %v", err)
|
||||||
|
}
|
||||||
|
t.Cleanup(docStore.Stop)
|
||||||
|
|
||||||
|
io := agentIO.NewIOManager()
|
||||||
|
|
||||||
|
a := New(AgentConfig{
|
||||||
|
ID: types.AgentID("medialive"),
|
||||||
|
SystemPrompt: "你是一个有长期记忆的助手。回答简洁准确。",
|
||||||
|
Provider: prov,
|
||||||
|
ProviderManager: pm,
|
||||||
|
IO: io,
|
||||||
|
Memory: graph,
|
||||||
|
DocStore: docStore,
|
||||||
|
MediaStore: ms,
|
||||||
|
StageHost: NewStageHost(),
|
||||||
|
MaxContextSize: 3, // 故意压低:第二轮就能触发 Prune 归档
|
||||||
|
InputProcessing: types.InputProcessingConfig{},
|
||||||
|
})
|
||||||
|
|
||||||
|
// 排空 outputCh:容量 256,但长跑不消费会堵住 emitResponse。
|
||||||
|
go func() {
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-io.OutputChan():
|
||||||
|
case <-a.ctx.Done():
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
return &liveEnv{agent: a, io: io, mediaSt: ms, docStore: docStore, graph: graph, dir: dir}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestMediaLive_AutoTriggerChain 全自动触发链:只注入事件,不手工调任何一步。
|
||||||
|
func TestMediaLive_AutoTriggerChain(t *testing.T) {
|
||||||
|
c := requireLiveCfg(t)
|
||||||
|
env := newLiveEnv(t, c)
|
||||||
|
a := env.agent
|
||||||
|
defer a.Stop()
|
||||||
|
|
||||||
|
img := livePNG(t, 96, 96, [][3]byte{{128, 0, 255}, {0, 64, 255}, {255, 0, 0}})
|
||||||
|
t.Logf("测试图片: %d 字节(紫/蓝/红三色带)", len(img))
|
||||||
|
|
||||||
|
// ── 阶段 1:注入 image 事件,验证 CAS 自动落盘 ──
|
||||||
|
//
|
||||||
|
// 直接调 handleInput 而不启 eventLoop:eventLoop 是纯转发(select →
|
||||||
|
// handleInput),走同一条代码路径,但同步调用让断言不必猜时序。
|
||||||
|
evt := &agentIO.InputEvent{
|
||||||
|
RequestID: "live-1",
|
||||||
|
Source: "test_channel",
|
||||||
|
Type: "image",
|
||||||
|
OutputChannel: "test_channel",
|
||||||
|
Payload: map[string]interface{}{
|
||||||
|
"data": mediaB64(img),
|
||||||
|
"mime": "image/png",
|
||||||
|
"alt": "一张测试图片",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
t0 := time.Now()
|
||||||
|
a.handleInput(evt)
|
||||||
|
t.Logf("第一轮(含真实 LLM 往返)耗时 %.1fs", time.Since(t0).Seconds())
|
||||||
|
|
||||||
|
// 媒体不再有文字描述:CAS 里只有字节、元数据与向量。
|
||||||
|
// 这里直接按 digest 定位刚落的图(不再有 Pending 队列)。
|
||||||
|
st := env.mediaSt.Stats()
|
||||||
|
if st["count"].(int) != 1 {
|
||||||
|
t.Fatalf("CAS 应自动收到 1 张图,实际 %v 张(captureBlockMedia 未被触发?)", st["count"])
|
||||||
|
}
|
||||||
|
var digest string
|
||||||
|
var found bool
|
||||||
|
for _, e := range a.context.Recent(0) {
|
||||||
|
for _, b := range e.Blocks {
|
||||||
|
digest, found = b.PayloadDigest, true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
t.Fatal("无法从上下文块定位刚落盘的图")
|
||||||
|
}
|
||||||
|
it0, err := env.mediaSt.Stat(digest)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Logf("✓ 阶段1 CAS 自动落盘: digest=%s size=%d tool=%s",
|
||||||
|
digest[:12], it0.Size, it0.Tool)
|
||||||
|
|
||||||
|
stored, err := env.mediaSt.Get(digest)
|
||||||
|
if err != nil || !bytes.Equal(stored, img) {
|
||||||
|
t.Fatalf("落盘内容与原图不一致 (err=%v)", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── 阶段 2:一等记忆块自动挂到 ContextEvent 上 ──
|
||||||
|
//
|
||||||
|
// 这一步验证 bindEventMedia:事件必须拿到 ID 并直接持有块;
|
||||||
|
// 事件文本必须保持原样(不再往正文里贴媒体标记)。
|
||||||
|
var evtID string
|
||||||
|
for _, e := range a.context.Recent(0) {
|
||||||
|
if len(e.Blocks) > 0 {
|
||||||
|
evtID = e.ID
|
||||||
|
if strings.Contains(e.Input, digest[:12]) {
|
||||||
|
t.Error("事件 Input 里被写入了媒体标记——描述式索引链应该已经拆除")
|
||||||
|
}
|
||||||
|
if e.Blocks[0].PayloadDigest != digest {
|
||||||
|
t.Fatalf("事件持有的块 digest 不对: %+v", e.Blocks)
|
||||||
|
}
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if evtID == "" {
|
||||||
|
t.Fatal("没有任何 ContextEvent 挂上媒体(bindEventMedia 未被触发)")
|
||||||
|
}
|
||||||
|
t.Logf("✓ 阶段2 块自动绑定: event=%s", evtID)
|
||||||
|
|
||||||
|
// ── 阶段 3:媒体只按自己的向量被索引,不再生成任何描述 ──
|
||||||
|
if it, err := env.mediaSt.Stat(digest); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
} else if len(it.Vec) == 0 {
|
||||||
|
// 未配置多模态空间时就没有向量——这是合法的降级状态,
|
||||||
|
// 但要明确报出来,而不是靠描述文本假装能检索。
|
||||||
|
t.Log("未配置多模态空间:本图无向量,之后只能靠块结构召回 digest")
|
||||||
|
} else {
|
||||||
|
t.Logf("✓ 阶段3 已写入原生向量: dim=%d", len(it.Vec))
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── 阶段 4:Prune 自动把块从 L0 迁移到 L2 ──
|
||||||
|
//
|
||||||
|
// MaxContextSize=3,多注入几轮文本把带图事件挤出活跃上下文。
|
||||||
|
// 迁移的是块本身(同一身份换层);L0 中不该再留下它。
|
||||||
|
// 填充数量必须 > Prune 内部固定的 10 条保护窗口。
|
||||||
|
//
|
||||||
|
// Prune 无条件保护最后 10 条事件(protected := events[len-10:]),
|
||||||
|
// 只在更早的部分里挑归档对象。填 4 条时总数才 5,全落进保护窗口、
|
||||||
|
// candidates 为空、直接返回 0——这不是缺陷,是"最近的对话不该被归档"
|
||||||
|
// 的设计。带图事件必须被推到第 11 条之前才可能被归档。
|
||||||
|
const fillerCount = 14
|
||||||
|
for i := 0; i < fillerCount; i++ {
|
||||||
|
a.context.Append(ContextEvent{
|
||||||
|
Timestamp: time.Now(),
|
||||||
|
Source: "filler",
|
||||||
|
Input: fmt.Sprintf("无关的填充对话 %d,用来把带图事件挤出活跃窗口", i),
|
||||||
|
Response: "好的。",
|
||||||
|
})
|
||||||
|
}
|
||||||
|
archived := a.context.Prune("当前输入", a.maxContextSize-1, env.docStore)
|
||||||
|
t.Logf("Prune 归档 %d 条事件", archived)
|
||||||
|
if archived == 0 {
|
||||||
|
t.Fatal("Prune 未归档任何事件,无法验证引用转移")
|
||||||
|
}
|
||||||
|
|
||||||
|
docRefsFound := ""
|
||||||
|
for _, d := range env.docStore.RecentDocs(20) {
|
||||||
|
for _, b := range d.Blocks {
|
||||||
|
if b.PayloadDigest == digest {
|
||||||
|
docRefsFound = d.ID
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if docRefsFound == "" {
|
||||||
|
t.Fatal("块未随归档事件迁移到 L2 文档")
|
||||||
|
}
|
||||||
|
// 同一块不能同时留在 L0。
|
||||||
|
for _, e := range a.context.Recent(0) {
|
||||||
|
for _, b := range e.Blocks {
|
||||||
|
if b.PayloadDigest == digest {
|
||||||
|
t.Errorf("块仍留在 L0(evt %s),违反单层不变量", e.ID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.Logf("✓ 阶段4 块自动迁移: context/%s → document/%s", evtID, docRefsFound)
|
||||||
|
|
||||||
|
// 迁移全程内容必须可读:块虽换了层,字节仍在。
|
||||||
|
if _, err := env.mediaSt.Get(digest); err != nil {
|
||||||
|
t.Fatalf("迁移后内容不可读: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── 阶段 5:archiveColdDocs 自动把块连到 L3 文档节点 ──
|
||||||
|
//
|
||||||
|
// FindColdDocs(72h, 2) 要求文档足够"冷",测试里新建的文档不满足,
|
||||||
|
// 因此把 LastAccess 往前推——这是为了触发生产代码路径,
|
||||||
|
// 而不是替代它(commitTriplesWithMedia/linkBlocksToDocument 全由它自己调)。
|
||||||
|
for _, d := range env.docStore.RecentDocs(20) {
|
||||||
|
if d.ID == docRefsFound {
|
||||||
|
d.LastAccess = time.Now().Add(-100 * time.Hour)
|
||||||
|
d.AccessCount = 0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
a.archiveColdDocs()
|
||||||
|
|
||||||
|
// 块可能以 document --contains--> block(文档归档)或
|
||||||
|
// sentence --contains--> block(对话三元组)两种边存在。
|
||||||
|
sentRefs := 0
|
||||||
|
var boundSentence int64
|
||||||
|
docBound := 0
|
||||||
|
rows, err := env.graph.Recall(nil, nil, 1, "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("graph recall: %v", err)
|
||||||
|
}
|
||||||
|
t.Logf("图库实体数 %d", len(rows.Entities))
|
||||||
|
docBlocks, err := env.graph.BlocksForNode("document", docRefsFound)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
docBound = len(docBlocks)
|
||||||
|
// 句子 id 是自增整数,扫前若干个足够覆盖本测试写入的量
|
||||||
|
for sid := int64(1); sid <= 40; sid++ {
|
||||||
|
blocks, err := env.graph.BlocksForNode("sentence", strconv.FormatInt(sid, 10))
|
||||||
|
if err == nil && len(blocks) > 0 {
|
||||||
|
sentRefs += len(blocks)
|
||||||
|
if boundSentence == 0 {
|
||||||
|
boundSentence = sid
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if sentRefs == 0 && docBound == 0 {
|
||||||
|
t.Error("L2→L3 未写入任何块边——linkBlocksToDocument 未被 archiveColdDocs 触发")
|
||||||
|
} else if docBound > 0 {
|
||||||
|
t.Logf("✓ 阶段5 L3 自动写入: 文档 %s 持有 %d 个块", docRefsFound, docBound)
|
||||||
|
got := docBlocks
|
||||||
|
if got[0].PayloadDigest != digest {
|
||||||
|
t.Errorf("文档节点持有的块 digest 不对: %+v", got)
|
||||||
|
} else if raw, err := env.mediaSt.Get(got[0].PayloadDigest); err != nil || !bytes.Equal(raw, img) {
|
||||||
|
t.Errorf("从文档块取回的字节与原图不一致 (err=%v)", err)
|
||||||
|
} else {
|
||||||
|
t.Logf("✓ 阶段5 反查取回 %d 字节,与原图逐字节一致", len(raw))
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
t.Logf("✓ 阶段5 L3 自动写入: %d 个句子块,首个 sentences.id=%d", sentRefs, boundSentence)
|
||||||
|
|
||||||
|
got, err := env.agent.RecallBlocksForSentence(boundSentence)
|
||||||
|
if err != nil || len(got) == 0 || got[0].PayloadDigest != digest {
|
||||||
|
t.Errorf("从句子反查块失败: got=%+v err=%v", got, err)
|
||||||
|
} else if raw, err := env.mediaSt.Get(got[0].PayloadDigest); err != nil || !bytes.Equal(raw, img) {
|
||||||
|
t.Errorf("从句子取回的字节与原图不一致 (err=%v)", err)
|
||||||
|
} else {
|
||||||
|
t.Logf("✓ 阶段5 反查取回 %d 字节,与原图逐字节一致", len(raw))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── 阶段 6:内容随块存在,不被单独清理 ──
|
||||||
|
if _, err := env.mediaSt.Stat(digest); err != nil {
|
||||||
|
t.Fatalf("被记忆块持有的内容不存在了: %v", err)
|
||||||
|
}
|
||||||
|
t.Logf("✓ 阶段6 被持有内容仍在")
|
||||||
|
|
||||||
|
// ── 阶段 7:E2E — 第二轮提问,验证 agent 真能召回 ──
|
||||||
|
//
|
||||||
|
// 不再提供图片,只问"还记得吗"。能答出三色说明记忆链路端到端可用。
|
||||||
|
// L2 文档此刻已被 archiveColdDocs 删除(归档的语义就是搬完删源),
|
||||||
|
// 所以这一轮只能靠 L3 图库召回——而自动注入路径依赖 indexer。
|
||||||
|
// 生产由 main.go 注入并周期 Sync;测试里手工建一个并同步一次。
|
||||||
|
a.indexer = memory.NewIndexer(env.graph)
|
||||||
|
if err := a.indexer.Sync(); err != nil {
|
||||||
|
t.Fatalf("indexer sync: %v", err)
|
||||||
|
}
|
||||||
|
if mc := a.buildMemoryContext("测试图片", 0); mc != "" {
|
||||||
|
t.Logf("注入的记忆上下文: %s", truncRunes(mc, 200))
|
||||||
|
} else {
|
||||||
|
t.Log("图库召回为空(本测试不再依赖文本描述,仅记录现状)")
|
||||||
|
}
|
||||||
|
|
||||||
|
ask := &agentIO.InputEvent{
|
||||||
|
RequestID: "live-2",
|
||||||
|
Source: "test_channel",
|
||||||
|
Type: "text",
|
||||||
|
OutputChannel: "test_channel",
|
||||||
|
Payload: map[string]interface{}{
|
||||||
|
"content": "你还记得我之前发给你的那张图片吗?它是什么样子的?请说出具体颜色。",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
respCh := make(chan *agentIO.OutputEvent, 4)
|
||||||
|
ask.ResponseCh = respCh
|
||||||
|
|
||||||
|
t2 := time.Now()
|
||||||
|
a.handleInput(ask)
|
||||||
|
t.Logf("第二轮耗时 %.1fs", time.Since(t2).Seconds())
|
||||||
|
|
||||||
|
// 第二轮仍走真实 LLM:这里只验证链路不报错、有回复。
|
||||||
|
// 不再断言"答出紫/蓝/红":图片的颜色信息只在原生向量里,
|
||||||
|
// 未配置多模态空间时模型本来就无从得知——那不属于记忆接线缺陷。
|
||||||
|
var answer string
|
||||||
|
select {
|
||||||
|
case out := <-respCh:
|
||||||
|
answer, _ = out.Payload["content"].(string)
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Fatal("第二轮没有收到回复")
|
||||||
|
}
|
||||||
|
t.Logf("agent 回答: %s", truncRunes(answer, 220))
|
||||||
|
if strings.HasPrefix(answer, "处理错误:") {
|
||||||
|
t.Skipf("上游 LLM 调用失败,端到端召回无法判定: %s", truncRunes(answer, 160))
|
||||||
|
}
|
||||||
|
t.Logf("✓ 阶段7 E2E 链路贯通(召回能力取决于是否配置多模态向量空间)")
|
||||||
|
|
||||||
|
st = env.mediaSt.Stats()
|
||||||
|
t.Logf("收尾: %v 条 / %v 字节 / 类型 %v",
|
||||||
|
st["count"], st["total_bytes"], st["by_kind"])
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestMediaLive_NegativeControl 阴性对照:没有媒体记忆时不该"记得"。
|
||||||
|
//
|
||||||
|
// 没有这条对照,任何"答出了具体内容"的结果都可能只是模型先验,
|
||||||
|
// 无法区分真召回与猜测。
|
||||||
|
func TestMediaLive_NegativeControl(t *testing.T) {
|
||||||
|
c := requireLiveCfg(t)
|
||||||
|
env := newLiveEnv(t, c)
|
||||||
|
a := env.agent
|
||||||
|
defer a.Stop()
|
||||||
|
|
||||||
|
ask := &agentIO.InputEvent{
|
||||||
|
RequestID: "neg-1",
|
||||||
|
Source: "test_channel",
|
||||||
|
Type: "text",
|
||||||
|
OutputChannel: "test_channel",
|
||||||
|
Payload: map[string]interface{}{
|
||||||
|
"content": "你还记得我之前发给你的那张图片吗?它是什么样子的?请说出具体颜色。",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
respCh := make(chan *agentIO.OutputEvent, 4)
|
||||||
|
ask.ResponseCh = respCh
|
||||||
|
|
||||||
|
a.handleInput(ask)
|
||||||
|
|
||||||
|
var answer string
|
||||||
|
select {
|
||||||
|
case out := <-respCh:
|
||||||
|
answer, _ = out.Payload["content"].(string)
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Fatal("阴性对照没有收到回复")
|
||||||
|
}
|
||||||
|
t.Logf("无记忆时的回答: %s", truncRunes(answer, 200))
|
||||||
|
|
||||||
|
// 上游不可用时这条对照没有意义:它只能证明"没答出颜色",
|
||||||
|
// 而原因是调用失败而非缺少记忆。据此判 PASS 属于假阳性。
|
||||||
|
if strings.HasPrefix(answer, "处理错误:") {
|
||||||
|
t.Skipf("上游 LLM 调用失败,阴性对照无法判定: %s", truncRunes(answer, 160))
|
||||||
|
}
|
||||||
|
|
||||||
|
guessed := strings.Contains(answer, "紫") &&
|
||||||
|
strings.Contains(answer, "蓝") &&
|
||||||
|
strings.Contains(answer, "红")
|
||||||
|
if guessed {
|
||||||
|
t.Errorf("无任何媒体记忆却猜中紫/蓝/红——"+
|
||||||
|
"说明阳性用例的通过可能只是先验偏好而非真召回: %s", truncRunes(answer, 300))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// mediaB64 返回不带 data URL 前缀的 base64(processMediaInput 自己拼前缀)。
|
||||||
|
func mediaB64(b []byte) string {
|
||||||
|
return media.DataURL("image/png", b)[len("data:image/png;base64,"):]
|
||||||
|
}
|
||||||
|
|
||||||
|
func truncRunes(s string, n int) string {
|
||||||
|
r := []rune(strings.ReplaceAll(s, "\n", " "))
|
||||||
|
if len(r) <= n {
|
||||||
|
return string(r)
|
||||||
|
}
|
||||||
|
return string(r[:n]) + "…"
|
||||||
|
}
|
||||||
199
internal/agent/core/medialoop.go
Normal file
199
internal/agent/core/medialoop.go
Normal file
@ -0,0 +1,199 @@
|
|||||||
|
package core
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"log"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
|
||||||
|
"gitcode.com/JianFeeeee/HomeAgent/internal/memory"
|
||||||
|
"gitcode.com/JianFeeeee/HomeAgent/internal/memory/vector"
|
||||||
|
)
|
||||||
|
|
||||||
|
// 媒体与记忆块的生命周期辅助。
|
||||||
|
//
|
||||||
|
// 媒体不单独做生命周期管理(没有 GC、没有引用计数):blob 是记忆块的内容,
|
||||||
|
// 块的创建/迁移/删除由记忆系统本身决定,块被永久删除时内容随之删除。
|
||||||
|
// 图片不靠文本描述索引——它只按自己的统一空间向量被检索。
|
||||||
|
|
||||||
|
// heldMediaDigests 汇总三层记忆当前持有的媒体 digest 集合。
|
||||||
|
//
|
||||||
|
// CAS 是全库字节存储,它的检索结果不等于「记忆里的媒体」——
|
||||||
|
// 召回前用它把已无处可归的内容过滤掉。
|
||||||
|
func (a *Agent) heldMediaDigests() map[string]bool {
|
||||||
|
held := map[string]bool{}
|
||||||
|
collect := func(blocks []memory.MemoryBlock) {
|
||||||
|
for _, b := range blocks {
|
||||||
|
if b.PayloadDigest != "" {
|
||||||
|
held[b.PayloadDigest] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if a.context != nil {
|
||||||
|
collect(a.context.Blocks())
|
||||||
|
}
|
||||||
|
if a.docStore != nil {
|
||||||
|
collect(a.docStore.Blocks())
|
||||||
|
}
|
||||||
|
if a.memory != nil {
|
||||||
|
if blocks, err := a.memory.MemoryBlocks(); err == nil {
|
||||||
|
collect(blocks)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return held
|
||||||
|
}
|
||||||
|
|
||||||
|
// payloadHeld 报告某个 digest 是否仍被三层记忆中的一等块持有。
|
||||||
|
// 这是删除前的一次活查询(不是持久化账本):同一份字节可能同时被多个块共享。
|
||||||
|
func (a *Agent) payloadHeld(digest string) bool {
|
||||||
|
if digest == "" {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if a.context != nil {
|
||||||
|
for _, b := range a.context.Blocks() {
|
||||||
|
if b.PayloadDigest == digest {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if a.docStore != nil {
|
||||||
|
for _, b := range a.docStore.Blocks() {
|
||||||
|
if b.PayloadDigest == digest {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if a.memory != nil {
|
||||||
|
if blocks, err := a.memory.MemoryBlocks(); err == nil {
|
||||||
|
for _, b := range blocks {
|
||||||
|
if b.PayloadDigest == digest {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// forgetPayloads 在记忆块被永久删除后删除它们的内容。
|
||||||
|
//
|
||||||
|
// 与文本块一致:删除块即删除内容。只有确认没有任何存活块仍共享该 digest
|
||||||
|
// 时才删字节(同一张图可能被多个块引用)。
|
||||||
|
func (a *Agent) forgetPayloads(digests []string) {
|
||||||
|
if a.mediaStore == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for _, d := range digests {
|
||||||
|
if d == "" || a.payloadHeld(d) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err := a.mediaStore.Delete(d); err != nil {
|
||||||
|
log.Printf("[media] 删除内容失败 %s: %v", shortDigest(d), err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// reembedStaleMedia 在启动时批量迁移历史媒体向量到当前向量空间。
|
||||||
|
//
|
||||||
|
// 触发场景(任一变化都会导致旧向量无法参与查询):
|
||||||
|
// - 切换模型(模型 A→模型 B,fp 变了)
|
||||||
|
// - 切换向量维度(ONNX→HTTP dim 512→1024)
|
||||||
|
// - 首次部署嵌入服务(历史无向量的媒体补算)
|
||||||
|
// - 嵌入服务离线后重新上线(失败条目 vec_model 仍为空)
|
||||||
|
//
|
||||||
|
// 并发策略:启动时用 worker pool 并行迁移,避免上千张图片串行耗时过长。
|
||||||
|
// 并发数在 ONNX 内嵌路径下不超 CPU 核心数(避免 ONNX 并发限流),
|
||||||
|
// 外部 API 路径下不超 8(避免打爆外部服务)。
|
||||||
|
func (a *Agent) reembedStaleMedia() {
|
||||||
|
if a.multimodalSpace == nil || a.mediaStore == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
fp := a.multimodalSpace.Fingerprint()
|
||||||
|
digests, err := a.mediaStore.StaleVecDigestsAll(fp)
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("[media] 查询需重算向量的媒体失败: %v", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if len(digests) == 0 {
|
||||||
|
log.Printf("[media] 无需迁移向量(所有媒体已与当前空间对齐 fp=%s)", shortFP(fp))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// 并发度:ONNX 内嵌不超过 4,外部 API 不超过 8(由配置或实际环境动态定)
|
||||||
|
workers := 4
|
||||||
|
if fp[:min(4, len(fp))] == "http:" {
|
||||||
|
workers = 8
|
||||||
|
}
|
||||||
|
log.Printf("[media] 启动向量迁移: %d 条 → 新空间 fp=%s dim=%d workers=%d",
|
||||||
|
len(digests), shortFP(fp), a.multimodalSpace.Dim(), workers)
|
||||||
|
|
||||||
|
jobs := make(chan string, workers*2)
|
||||||
|
var done, failed, unsupported int64
|
||||||
|
var failedMu sync.Mutex
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
|
||||||
|
for i := 0; i < workers; i++ {
|
||||||
|
wg.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
for d := range jobs {
|
||||||
|
switch err := a.reembedOne(d, fp); {
|
||||||
|
case err == nil:
|
||||||
|
atomic.AddInt64(&done, 1)
|
||||||
|
case errors.Is(err, vector.ErrModalityUnsupported):
|
||||||
|
// 该模态不在本空间内(如音频):不重试、不计失败,
|
||||||
|
// 也不拿另一个模型的向量顶替。
|
||||||
|
atomic.AddInt64(&unsupported, 1)
|
||||||
|
default:
|
||||||
|
failedMu.Lock()
|
||||||
|
failed++
|
||||||
|
failedMu.Unlock()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
|
||||||
|
for i, d := range digests {
|
||||||
|
jobs <- d
|
||||||
|
// 每迁移 20 条输出进度日志,让用户看到迁移在推进
|
||||||
|
if (i+1)%20 == 0 {
|
||||||
|
log.Printf("[media] 向量迁移进度: %d/%d (done=%d failed=%d)", i+1, len(digests), atomic.LoadInt64(&done), failed)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
close(jobs)
|
||||||
|
wg.Wait()
|
||||||
|
log.Printf("[media] 向量迁移完成: 成功=%d 失败=%d 不在本空间=%d 总计=%d fp=%s",
|
||||||
|
done, failed, unsupported, len(digests), shortFP(fp))
|
||||||
|
}
|
||||||
|
|
||||||
|
// reembedOne 为单条媒体重新计算向量并写入(stat/get 失败时跳过该条目)。
|
||||||
|
//
|
||||||
|
// 模态不在本空间覆盖范围时返回 ErrModalityUnsupported,调用方据此区分
|
||||||
|
// 「永久无向量」与「本次失败重试」。
|
||||||
|
func (a *Agent) reembedOne(digest, fp string) error {
|
||||||
|
it, err := a.mediaStore.Stat(digest)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
data, err := a.mediaStore.Get(digest)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
mime := it.MIME
|
||||||
|
if mime == "" {
|
||||||
|
mime = "image/png"
|
||||||
|
}
|
||||||
|
vec, err := a.multimodalSpace.EmbedImageDense(data, mime)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return a.mediaStore.SetVec(digest, vec, fp)
|
||||||
|
}
|
||||||
|
|
||||||
|
// shortFP 截断 fingerprint 为可读日志格式。
|
||||||
|
func shortFP(fp string) string {
|
||||||
|
if len(fp) > 12 {
|
||||||
|
return fp[:12]
|
||||||
|
}
|
||||||
|
return fp
|
||||||
|
}
|
||||||
187
internal/agent/core/medialoop_test.go
Normal file
187
internal/agent/core/medialoop_test.go
Normal file
@ -0,0 +1,187 @@
|
|||||||
|
package core
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitcode.com/JianFeeeee/HomeAgent/internal/memory"
|
||||||
|
"gitcode.com/JianFeeeee/HomeAgent/internal/memory/document"
|
||||||
|
"gitcode.com/JianFeeeee/HomeAgent/internal/memory/media"
|
||||||
|
"gitcode.com/JianFeeeee/HomeAgent/internal/memory/vector"
|
||||||
|
)
|
||||||
|
|
||||||
|
// 媒体与记忆块的生命周期测试。
|
||||||
|
//
|
||||||
|
// 媒体没有独立生命周期管理(没有 GC、没有引用计数):blob 是记忆块的内容,
|
||||||
|
// 块的创建/迁移/删除由记忆系统决定。图片也不靠文本描述索引。
|
||||||
|
|
||||||
|
func newMediaLoopAgent(t *testing.T) (*Agent, *media.Store) {
|
||||||
|
t.Helper()
|
||||||
|
dir := t.TempDir()
|
||||||
|
ms, err := media.New(filepath.Join(dir, "media"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("media.New: %v", err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { ms.Close() })
|
||||||
|
|
||||||
|
a := &Agent{mediaStore: ms}
|
||||||
|
a.ctx, a.cancel = context.WithCancel(context.Background())
|
||||||
|
t.Cleanup(a.cancel)
|
||||||
|
return a, ms
|
||||||
|
}
|
||||||
|
|
||||||
|
// heldMediaDigests 汇总三层记忆持有的媒体:只有这些才可被召回。
|
||||||
|
func TestHeldMediaDigests_CollectsAcrossLayers(t *testing.T) {
|
||||||
|
a, ms := newMediaLoopAgent(t)
|
||||||
|
d1, _ := ms.Put([]byte("ctx-layer"), media.Item{MIME: "image/png"})
|
||||||
|
d2, _ := ms.Put([]byte("doc-layer"), media.Item{MIME: "image/png"})
|
||||||
|
d3, _ := ms.Put([]byte("graph-layer"), media.Item{MIME: "image/png"})
|
||||||
|
d4, _ := ms.Put([]byte("orphan"), media.Item{MIME: "image/png"})
|
||||||
|
|
||||||
|
a.context = NewRelevanceContext("", memory.NewStaticEmbedder(""))
|
||||||
|
a.context.Append(ContextEvent{Input: "带图的一轮", Blocks: []memory.MemoryBlock{
|
||||||
|
{ID: "blk_ctx", Modality: memory.BlockImage, PayloadDigest: d1},
|
||||||
|
}})
|
||||||
|
|
||||||
|
dir := t.TempDir()
|
||||||
|
bo, ok := a.blockFromDigest(d2)
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("blockFromDigest 失败")
|
||||||
|
}
|
||||||
|
ds := document.NewStore(filepath.Join(dir, "docs"), memory.TokenizeWords)
|
||||||
|
if err := ds.Start(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer ds.Stop()
|
||||||
|
if err := ds.Insert(&document.Doc{ID: "doc_1", Summary: "s", Blocks: []memory.MemoryBlock{bo}}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
a.docStore = ds
|
||||||
|
|
||||||
|
g, err := memory.NewGraphDB(filepath.Join(dir, "graph.db"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer g.Close()
|
||||||
|
if err := g.PutMemoryBlocks([]memory.MemoryBlock{
|
||||||
|
{ID: "blk_g", Modality: memory.BlockImage, PayloadDigest: d3},
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
a.memory = g
|
||||||
|
|
||||||
|
held := a.heldMediaDigests()
|
||||||
|
for _, want := range []string{d1, d2, d3} {
|
||||||
|
if !held[want] {
|
||||||
|
t.Errorf("层次持有 %s 却不在结果里: %v", shortDigest(want), held)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if held[d4] {
|
||||||
|
t.Errorf("无人持有的 %s 不该出现在结果里", shortDigest(d4))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// fakeSpace 是一个只覆盖图像的假统一空间,用来验证「不在本空间」与
|
||||||
|
// 「本次失败」必须被区分对待。
|
||||||
|
type fakeSpace struct{}
|
||||||
|
|
||||||
|
func (fakeSpace) VectorizeDense(string) ([]float64, error) { return []float64{1, 0}, nil }
|
||||||
|
|
||||||
|
func (fakeSpace) EmbedImageDense(_ []byte, mime string) ([]float64, error) {
|
||||||
|
if strings.HasPrefix(mime, "audio/") || strings.HasPrefix(mime, "video/") {
|
||||||
|
return nil, fmt.Errorf("%w: %s", vector.ErrModalityUnsupported, mime)
|
||||||
|
}
|
||||||
|
return []float64{1, 0}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (fakeSpace) Fingerprint() string { return "fake-space" }
|
||||||
|
func (fakeSpace) Dim() int { return 2 }
|
||||||
|
func (fakeSpace) Loaded() bool { return true }
|
||||||
|
func (fakeSpace) Close() {}
|
||||||
|
|
||||||
|
// TestReembedStaleMedia_SkipsUnsupportedWithoutFaking 验证向量迁移不会:
|
||||||
|
// - 把音频当失败反复重试;
|
||||||
|
// - 更不能拿另一个模型的向量顶替音频(那会污染统一空间且静默)。
|
||||||
|
func TestReembedStaleMedia_SkipsUnsupportedWithoutFaking(t *testing.T) {
|
||||||
|
a, ms := newMediaLoopAgent(t)
|
||||||
|
img, _ := ms.Put([]byte("img-bytes"), media.Item{MIME: "image/png"})
|
||||||
|
aud, _ := ms.Put([]byte("aud-bytes"), media.Item{MIME: "audio/wav"})
|
||||||
|
|
||||||
|
a.multimodalSpace = fakeSpace{}
|
||||||
|
a.reembedStaleMedia()
|
||||||
|
|
||||||
|
it, err := ms.Stat(img)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(it.Vec) != 2 || it.VecModel != "fake-space" {
|
||||||
|
t.Fatalf("图像应拿到本空间向量,实际 vec=%v model=%q", it.Vec, it.VecModel)
|
||||||
|
}
|
||||||
|
|
||||||
|
audIt, err := ms.Stat(aud)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(audIt.Vec) != 0 || audIt.VecModel != "" {
|
||||||
|
t.Fatalf("音频不得被写入任何向量(不能用别的模型顶替),实际 vec=%v model=%q",
|
||||||
|
audIt.Vec, audIt.VecModel)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// payloadHeld 是删除前的活查询。
|
||||||
|
func TestPayloadHeld(t *testing.T) {
|
||||||
|
a, ms := newMediaLoopAgent(t)
|
||||||
|
d, _ := ms.Put([]byte("held"), media.Item{MIME: "image/png"})
|
||||||
|
if a.payloadHeld(d) {
|
||||||
|
t.Fatal("尚无块持有时不该报已持有")
|
||||||
|
}
|
||||||
|
|
||||||
|
a.context = NewRelevanceContext("", memory.NewStaticEmbedder(""))
|
||||||
|
a.context.Append(ContextEvent{Input: "x", Blocks: []memory.MemoryBlock{
|
||||||
|
{ID: "blk_1", Modality: memory.BlockImage, PayloadDigest: d},
|
||||||
|
}})
|
||||||
|
if !a.payloadHeld(d) {
|
||||||
|
t.Fatal("L0 持有却报未持有")
|
||||||
|
}
|
||||||
|
if a.payloadHeld("") {
|
||||||
|
t.Fatal("空 digest 应为 false")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestForgetPayloads_DeletesOnlyUnheldContent 验证删除语义:
|
||||||
|
// 块被删除后内容才被删;仍被其它记忆块共享的 digest 不会被误删。
|
||||||
|
func TestForgetPayloads_DeletesOnlyUnheldContent(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
ms, err := media.New(filepath.Join(dir, "media"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer ms.Close()
|
||||||
|
|
||||||
|
d1, _ := ms.Put([]byte("held-by-graph"), media.Item{MIME: "image/png"})
|
||||||
|
d2, _ := ms.Put([]byte("being-forgotten"), media.Item{MIME: "image/png"})
|
||||||
|
|
||||||
|
g, err := memory.NewGraphDB(filepath.Join(dir, "graph.db"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer g.Close()
|
||||||
|
if err := g.PutMemoryBlocks([]memory.MemoryBlock{
|
||||||
|
{ID: "blk_keep", Modality: memory.BlockImage, PayloadDigest: d1},
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
a := &Agent{mediaStore: ms, memory: g}
|
||||||
|
a.forgetPayloads([]string{d1, d2})
|
||||||
|
|
||||||
|
if _, err := ms.Stat(d1); err != nil {
|
||||||
|
t.Fatalf("仍被 L3 块持有的内容不该被删: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := ms.Stat(d2); err == nil {
|
||||||
|
t.Fatal("无人持有的内容应被删除")
|
||||||
|
}
|
||||||
|
}
|
||||||
210
internal/agent/core/mediaref.go
Normal file
210
internal/agent/core/mediaref.go
Normal file
@ -0,0 +1,210 @@
|
|||||||
|
package core
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"log"
|
||||||
|
"strings"
|
||||||
|
"sync/atomic"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
agentAPI "gitcode.com/JianFeeeee/HomeAgent/internal/agent/api"
|
||||||
|
"gitcode.com/JianFeeeee/HomeAgent/internal/memory"
|
||||||
|
"gitcode.com/JianFeeeee/HomeAgent/internal/memory/media"
|
||||||
|
)
|
||||||
|
|
||||||
|
// blockSeq 保证块 ID 全局唯一(Graph memory_blocks 以 id 为主键)。
|
||||||
|
var blockSeq int64
|
||||||
|
|
||||||
|
func newBlockID() string {
|
||||||
|
return fmt.Sprintf("blk_%d_%d", time.Now().UnixNano(), atomic.AddInt64(&blockSeq, 1))
|
||||||
|
}
|
||||||
|
|
||||||
|
// blockModalityOf 把 CAS 媒体大类映射为一等记忆块模态。
|
||||||
|
func blockModalityOf(k media.Kind) memory.BlockModality {
|
||||||
|
switch k {
|
||||||
|
case media.KindImage:
|
||||||
|
return memory.BlockImage
|
||||||
|
case media.KindVideo:
|
||||||
|
return memory.BlockVideo
|
||||||
|
case media.KindAudio:
|
||||||
|
return memory.BlockAudio
|
||||||
|
default:
|
||||||
|
return memory.BlockText
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// blockFromDigest 把一份已入库媒体变成一等记忆块。
|
||||||
|
// 块携带 digest/向量/fingerprint;CAS 只提供字节与元数据,不参与生命周期。
|
||||||
|
func (a *Agent) blockFromDigest(digest string) (memory.MemoryBlock, bool) {
|
||||||
|
if a.mediaStore == nil || digest == "" {
|
||||||
|
return memory.MemoryBlock{}, false
|
||||||
|
}
|
||||||
|
it, err := a.mediaStore.Stat(digest)
|
||||||
|
if err != nil || it == nil {
|
||||||
|
return memory.MemoryBlock{}, false
|
||||||
|
}
|
||||||
|
return memory.MemoryBlock{
|
||||||
|
ID: newBlockID(),
|
||||||
|
Modality: blockModalityOf(it.Kind),
|
||||||
|
PayloadDigest: it.Digest,
|
||||||
|
MIME: it.MIME,
|
||||||
|
Size: it.Size,
|
||||||
|
Width: it.Width,
|
||||||
|
Height: it.Height,
|
||||||
|
Vector: it.Vec,
|
||||||
|
Fingerprint: it.VecModel,
|
||||||
|
Tool: it.Tool,
|
||||||
|
CreatedAt: it.FirstSeen,
|
||||||
|
}, true
|
||||||
|
}
|
||||||
|
|
||||||
|
// 媒体记忆接线:把对话里出现的图片/音频落进内容寻址存储(CAS),
|
||||||
|
// 并让 L0 的 ContextEvent 直接持有一等记忆块。
|
||||||
|
//
|
||||||
|
// 媒体进入对话有两条路:用户直接发图(ContentBlock data URL)、插件注入
|
||||||
|
// (SetToolBlocks)。两条都在这里收口:从 data URL 取出字节存进 CAS,
|
||||||
|
// 用其向量构造一等记忆块挂到当轮 ContextEvent 上;事件被 Prune 时
|
||||||
|
// 块随之迁移到 L2 文档。
|
||||||
|
//
|
||||||
|
// 不再生成任何描述文本,也不再往正文写 media marker:图片只按自己的
|
||||||
|
// 统一空间向量被检索,描述式索引是将就方案。
|
||||||
|
|
||||||
|
// captureBlockMedia 把 blocks 里的 data URL 媒体落进 CAS,返回 digest 列表。
|
||||||
|
//
|
||||||
|
// 只处理 data URL:http(s) URL 拿不到字节就无法做内容寻址,
|
||||||
|
// 而"下载它再存"会把一次对话变成一次网络请求(超时、鉴权、SSRF 全来了),
|
||||||
|
// 不在本层解决。
|
||||||
|
func (a *Agent) captureBlockMedia(blocks []agentAPI.ContentBlock, tool string) []string {
|
||||||
|
if a.mediaStore == nil || len(blocks) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var digests []string
|
||||||
|
for _, b := range blocks {
|
||||||
|
var url string
|
||||||
|
switch {
|
||||||
|
case b.ImageURL != nil && b.ImageURL.URL != "":
|
||||||
|
url = b.ImageURL.URL
|
||||||
|
case b.AudioURL != nil && b.AudioURL.URL != "":
|
||||||
|
url = b.AudioURL.URL
|
||||||
|
default:
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
mime, data, ok := media.ParseDataURL(url)
|
||||||
|
if !ok {
|
||||||
|
continue // http(s) URL 或格式不认,跳过
|
||||||
|
}
|
||||||
|
|
||||||
|
d, err := a.mediaStore.Put(data, media.Item{
|
||||||
|
MIME: mime,
|
||||||
|
Tool: tool,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
// 媒体存不进去不该让对话失败——它是记忆增强,不是对话必需品
|
||||||
|
log.Printf("[media] 落盘失败 (tool=%s mime=%s): %v", tool, mime, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// 入库即算一次多模态坐标并缓存(多模态空间可用时)。
|
||||||
|
// 之后 doc_query / memory_recall / 内部召回直接复用 SetVec 的缓存坐标,
|
||||||
|
// 不重复跑 ONNX;模型切换由启动时的 reembedStaleMedia 补算。
|
||||||
|
a.embedMediaOnIngest(d, mime, data)
|
||||||
|
digests = append(digests, d)
|
||||||
|
}
|
||||||
|
return digests
|
||||||
|
}
|
||||||
|
|
||||||
|
// embedMediaOnIngest 给刚入库的图片立即计算多模态坐标并缓存。
|
||||||
|
// 只在 多模态空间可用且为图像时执行;音频/未配置时静默跳过(保持既有行为)。
|
||||||
|
func (a *Agent) embedMediaOnIngest(digest, mime string, data []byte) {
|
||||||
|
if a.multimodalSpace == nil || !a.multimodalSpace.Loaded() {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if !strings.HasPrefix(mime, "image/") {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
vec, err := a.multimodalSpace.EmbedImageDense(data, mime)
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("[media] 入库嵌入失败 %s: %v", shortDigest(digest), err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := a.mediaStore.SetVec(digest, vec, a.multimodalSpace.Fingerprint()); err != nil {
|
||||||
|
log.Printf("[media] 入库写向量失败 %s: %v", shortDigest(digest), err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// stageMediaDigests 累积本轮捕获的 digest,等 ContextEvent 建好后一起挂上。
|
||||||
|
//
|
||||||
|
// 为何要缓存而不是当场建块:媒体在 process() 执行期间被捕获,而承载它的
|
||||||
|
// ContextEvent 要等 process() 返回后才 Append——此刻还没有 owner_id。
|
||||||
|
// 与既有的 a.pendingMedia 同一手法(都在 a.mu 保护下)。
|
||||||
|
func (a *Agent) stageMediaDigests(digests ...string) {
|
||||||
|
if len(digests) == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
a.pendingMediaDigests = append(a.pendingMediaDigests, digests...)
|
||||||
|
}
|
||||||
|
|
||||||
|
// drainMediaDigests 取出并清空本轮累积的 digest。
|
||||||
|
func (a *Agent) drainMediaDigests() []string {
|
||||||
|
if len(a.pendingMediaDigests) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
out := a.pendingMediaDigests
|
||||||
|
a.pendingMediaDigests = nil
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// bindEventMedia 把本轮捕获的媒体变成一等记忆块,直接挂到 ContextEvent 上。
|
||||||
|
//
|
||||||
|
// 块存储在事件自身(随 context.json 持久化),不再写 media_refs:
|
||||||
|
// 存活与否由“三层记忆块是否持有这个 digest”决定,不维护引用账本。
|
||||||
|
func (a *Agent) bindEventMedia(evt *ContextEvent, digests []string) {
|
||||||
|
if a.mediaStore == nil || evt == nil || len(digests) == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if evt.ID == "" {
|
||||||
|
evt.ID = newEventID()
|
||||||
|
}
|
||||||
|
for _, d := range digests {
|
||||||
|
b, ok := a.blockFromDigest(d)
|
||||||
|
if !ok {
|
||||||
|
log.Printf("[media] 块构造失败 (%s)", shortDigest(d))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
evt.Blocks = append(evt.Blocks, b)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// mediaLabel 渲染一行媒体标签,供提示词告知"这条记忆带着哪份媒体"。
|
||||||
|
//
|
||||||
|
// 不再包含任何生成的描述文本:图片只按自己的向量被检索,标签仅提供
|
||||||
|
// MIME 与短 digest,让模型知道有这份媒体、可据 digest 取回字节。
|
||||||
|
// 查不到返回空串:内容可能已被删除,不该造出一条指向虚无的标签。
|
||||||
|
func mediaLabel(it *media.Item) string {
|
||||||
|
if it == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
label := string(it.Kind)
|
||||||
|
if it.MIME != "" {
|
||||||
|
label = it.MIME
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("[%s %s]", label, shortDigest(it.Digest))
|
||||||
|
}
|
||||||
|
|
||||||
|
// newEventID 生成 ContextEvent 的稳定标识。
|
||||||
|
//
|
||||||
|
// 沿用 document.Store 的 doc_<unixnano> 手法(同一份代码库里保持一致,
|
||||||
|
// 也避免为此引入 uuid 依赖)。纳秒精度足够:同一 Agent 的事件由
|
||||||
|
// a.mu 串行化 Append,不存在同纳秒两条。
|
||||||
|
func newEventID() string {
|
||||||
|
return fmt.Sprintf("evt_%d", time.Now().UnixNano())
|
||||||
|
}
|
||||||
|
|
||||||
|
func shortDigest(d string) string {
|
||||||
|
if len(d) > 12 {
|
||||||
|
return d[:12]
|
||||||
|
}
|
||||||
|
return d
|
||||||
|
}
|
||||||
333
internal/agent/core/mediaref_test.go
Normal file
333
internal/agent/core/mediaref_test.go
Normal file
@ -0,0 +1,333 @@
|
|||||||
|
package core
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
agentAPI "gitcode.com/JianFeeeee/HomeAgent/internal/agent/api"
|
||||||
|
"gitcode.com/JianFeeeee/HomeAgent/internal/memory"
|
||||||
|
"gitcode.com/JianFeeeee/HomeAgent/internal/memory/document"
|
||||||
|
"gitcode.com/JianFeeeee/HomeAgent/internal/memory/media"
|
||||||
|
)
|
||||||
|
|
||||||
|
// 媒体记忆接线测试:验证媒体从对话进入 CAS、挂到 L0 事件、
|
||||||
|
// 随归档转到 L2 文档的完整链路。
|
||||||
|
//
|
||||||
|
// 核心断言不是"函数被调用了",而是不变量:
|
||||||
|
// 1. 媒体存不进去时对话照常(它是记忆增强,不是对话必需品)
|
||||||
|
// 2. 引用转移期间内容始终可读(先挂后销,不留归零窗口)
|
||||||
|
// 3. mediaStore 为 nil 时全链路静默跳过,行为与本特性上线前一致
|
||||||
|
|
||||||
|
func newTestAgentWithMedia(t *testing.T) (*Agent, *media.Store) {
|
||||||
|
t.Helper()
|
||||||
|
dir := t.TempDir()
|
||||||
|
ms, err := media.New(filepath.Join(dir, "media"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("media.New: %v", err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { ms.Close() })
|
||||||
|
|
||||||
|
emb := memory.NewStaticEmbedder()
|
||||||
|
a := &Agent{
|
||||||
|
mediaStore: ms,
|
||||||
|
context: NewRelevanceContext(filepath.Join(dir, "context.json"), emb),
|
||||||
|
}
|
||||||
|
return a, ms
|
||||||
|
}
|
||||||
|
|
||||||
|
// imageBlockURL 造一个带指定 URL 的图片块。
|
||||||
|
// 名字带 URL 后缀是为了不与 modalfallback_test.go 里固定用 testPNG 的
|
||||||
|
// imageBlock() 撞名——两者用途不同:那个验回退链,这个验入库。
|
||||||
|
func imageBlockURL(dataURL string) agentAPI.ContentBlock {
|
||||||
|
return agentAPI.ContentBlock{
|
||||||
|
Type: "image_url",
|
||||||
|
ImageURL: &agentAPI.ImageURL{URL: dataURL, Detail: "auto"},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCaptureBlockMedia_StoresDataURL(t *testing.T) {
|
||||||
|
a, ms := newTestAgentWithMedia(t)
|
||||||
|
|
||||||
|
raw := []byte{0x89, 'P', 'N', 'G', 1, 2, 3}
|
||||||
|
blocks := []agentAPI.ContentBlock{
|
||||||
|
{Type: "text", Text: "看这张图"},
|
||||||
|
imageBlockURL(media.DataURL("image/png", raw)),
|
||||||
|
}
|
||||||
|
|
||||||
|
digests := a.captureBlockMedia(blocks, "multimodal_see_picture")
|
||||||
|
if len(digests) != 1 {
|
||||||
|
t.Fatalf("应捕获 1 个媒体,实际 %d", len(digests))
|
||||||
|
}
|
||||||
|
|
||||||
|
got, err := ms.Get(digests[0])
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("回读失败: %v", err)
|
||||||
|
}
|
||||||
|
if string(got) != string(raw) {
|
||||||
|
t.Fatal("内容不一致")
|
||||||
|
}
|
||||||
|
it, _ := ms.Stat(digests[0])
|
||||||
|
if it.MIME != "image/png" || it.Tool != "multimodal_see_picture" || it.Kind != media.KindImage {
|
||||||
|
t.Fatalf("元数据不对: %+v", it)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCaptureBlockMedia_SkipsHTTPURL(t *testing.T) {
|
||||||
|
// http(s) URL 拿不到字节就无法内容寻址;"下载它再存"会把一次对话
|
||||||
|
// 变成一次网络请求(超时、鉴权、SSRF 全来了),不在本层解决。
|
||||||
|
a, _ := newTestAgentWithMedia(t)
|
||||||
|
|
||||||
|
blocks := []agentAPI.ContentBlock{
|
||||||
|
imageBlockURL("https://example.com/x.png"),
|
||||||
|
}
|
||||||
|
if d := a.captureBlockMedia(blocks, "t"); len(d) != 0 {
|
||||||
|
t.Fatalf("http URL 不该被捕获,实际 %d 个", len(d))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCaptureBlockMedia_NilStoreIsNoop(t *testing.T) {
|
||||||
|
// mediaStore 未启用时全链路静默跳过,不能 panic 也不能报错——
|
||||||
|
// 行为必须与本特性上线前完全一致。
|
||||||
|
a := &Agent{}
|
||||||
|
blocks := []agentAPI.ContentBlock{imageBlockURL(media.DataURL("image/png", []byte("x")))}
|
||||||
|
if d := a.captureBlockMedia(blocks, "t"); d != nil {
|
||||||
|
t.Fatalf("nil store 应返回 nil,实际 %v", d)
|
||||||
|
}
|
||||||
|
a.stageMediaDigests("deadbeef")
|
||||||
|
if got := a.drainMediaDigests(); len(got) != 1 {
|
||||||
|
t.Fatal("stage/drain 不依赖 store,应正常工作")
|
||||||
|
}
|
||||||
|
// bindEventMedia 对 nil store 也必须安全
|
||||||
|
evt := &ContextEvent{}
|
||||||
|
a.bindEventMedia(evt, []string{"deadbeef"})
|
||||||
|
if len(evt.Blocks) != 0 || evt.ID != "" {
|
||||||
|
t.Fatalf("nil store 时不该改动事件: %+v", evt)
|
||||||
|
}
|
||||||
|
if s := mediaLabel(nil); s != "" {
|
||||||
|
t.Fatalf("nil 媒体应产出空标签,得到 %q", s)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCaptureBlockMedia_AudioAndVideo(t *testing.T) {
|
||||||
|
a, ms := newTestAgentWithMedia(t)
|
||||||
|
|
||||||
|
blocks := []agentAPI.ContentBlock{
|
||||||
|
imageBlockURL(media.DataURL("image/jpeg", []byte("frame"))),
|
||||||
|
{Type: "audio_url", AudioURL: &agentAPI.AudioURL{URL: media.DataURL("audio/wav", []byte("sound"))}},
|
||||||
|
}
|
||||||
|
digests := a.captureBlockMedia(blocks, "multimodal_see_video")
|
||||||
|
if len(digests) != 2 {
|
||||||
|
t.Fatalf("应捕获 2 个,实际 %d", len(digests))
|
||||||
|
}
|
||||||
|
|
||||||
|
kinds := map[media.Kind]int{}
|
||||||
|
for _, d := range digests {
|
||||||
|
it, err := ms.Stat(d)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
kinds[it.Kind]++
|
||||||
|
}
|
||||||
|
if kinds[media.KindImage] != 1 || kinds[media.KindAudio] != 1 {
|
||||||
|
t.Fatalf("大类归属不对: %v", kinds)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStageDrainMediaDigests(t *testing.T) {
|
||||||
|
a, _ := newTestAgentWithMedia(t)
|
||||||
|
|
||||||
|
a.stageMediaDigests("a", "b")
|
||||||
|
a.stageMediaDigests("c")
|
||||||
|
got := a.drainMediaDigests()
|
||||||
|
if len(got) != 3 {
|
||||||
|
t.Fatalf("应累积 3 个,实际 %d", len(got))
|
||||||
|
}
|
||||||
|
// drain 后必须清空——否则下一轮对话会把上一轮的媒体又挂一遍
|
||||||
|
if again := a.drainMediaDigests(); again != nil {
|
||||||
|
t.Fatalf("drain 后应为空,实际 %v", again)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBindEventMedia_CreatesBlocks(t *testing.T) {
|
||||||
|
a, ms := newTestAgentWithMedia(t)
|
||||||
|
|
||||||
|
d, err := ms.Put([]byte("img"), media.Item{MIME: "image/png"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
evt := &ContextEvent{Timestamp: time.Now(), Source: "qq", Input: "看图"}
|
||||||
|
a.bindEventMedia(evt, []string{d})
|
||||||
|
|
||||||
|
if evt.ID == "" {
|
||||||
|
t.Fatal("应懒生成事件 ID")
|
||||||
|
}
|
||||||
|
if len(evt.Blocks) != 1 || evt.Blocks[0].PayloadDigest != d {
|
||||||
|
t.Fatalf("事件应持有一等记忆块: %+v", evt.Blocks)
|
||||||
|
}
|
||||||
|
if evt.Blocks[0].Modality != memory.BlockImage || evt.Blocks[0].MIME != "image/png" {
|
||||||
|
t.Fatalf("块元数据不对: %+v", evt.Blocks[0])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBindEventMedia_LazyIDOnlyWhenNeeded(t *testing.T) {
|
||||||
|
// 绝大多数对话没有媒体,不该为它们都生成 ID 塞进 context.json
|
||||||
|
a, _ := newTestAgentWithMedia(t)
|
||||||
|
evt := &ContextEvent{Input: "纯文本"}
|
||||||
|
a.bindEventMedia(evt, nil)
|
||||||
|
if evt.ID != "" {
|
||||||
|
t.Fatalf("无媒体时不该生成 ID,得到 %q", evt.ID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMediaLabel_NoGeneratedDescription(t *testing.T) {
|
||||||
|
// 标签只用来告诉模型「这条记忆带着哪份媒体、可用该 digest 取回字节」。
|
||||||
|
// 它不包含任何生成的描述:描述式索引是把就机制,已彻底废弃。
|
||||||
|
a, ms := newTestAgentWithMedia(t)
|
||||||
|
|
||||||
|
d, _ := ms.Put([]byte("img"), media.Item{MIME: "image/png"})
|
||||||
|
it, err := ms.Stat(d)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
s := mediaLabel(it)
|
||||||
|
if s == "" {
|
||||||
|
t.Fatal("应产出标签")
|
||||||
|
}
|
||||||
|
if !strings.Contains(s, "image/png") {
|
||||||
|
t.Fatalf("标签应含 MIME 标注: %q", s)
|
||||||
|
}
|
||||||
|
if !strings.Contains(s, shortDigest(d)) {
|
||||||
|
t.Fatalf("标签应含短 digest 供反查: %q", s)
|
||||||
|
}
|
||||||
|
_ = a
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPrune_NilMediaStoreStillArchives(t *testing.T) {
|
||||||
|
// 媒体存储未启用时归档链路必须照常工作
|
||||||
|
dir := t.TempDir()
|
||||||
|
emb := memory.NewStaticEmbedder()
|
||||||
|
docStore := document.NewStore(filepath.Join(dir, "docs"), memory.TokenizeWords)
|
||||||
|
if err := docStore.Start(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
rc := NewRelevanceContext(filepath.Join(dir, "context.json"), emb)
|
||||||
|
|
||||||
|
for i := 0; i < 15; i++ {
|
||||||
|
rc.Append(ContextEvent{
|
||||||
|
Timestamp: time.Now().Add(time.Duration(i) * time.Second),
|
||||||
|
Source: "qq",
|
||||||
|
Input: "内容",
|
||||||
|
})
|
||||||
|
}
|
||||||
|
if n := rc.Prune("查询", 5, docStore); n == 0 {
|
||||||
|
t.Fatal("无媒体存储时归档也应正常")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestContextEvent_BlocksFieldRoundTrip(t *testing.T) {
|
||||||
|
// context.json 加字段必须向后兼容:存量文件读回来 Blocks 为空、ID 为空,
|
||||||
|
// 不影响任何既有行为。
|
||||||
|
dir := t.TempDir()
|
||||||
|
path := filepath.Join(dir, "context.json")
|
||||||
|
|
||||||
|
// 写一份"存量格式"(无 id / blocks 字段)
|
||||||
|
legacy := `[{"timestamp":"2026-09-04T10:00:00Z","source":"qq","input":"老数据","response":"回复"}]`
|
||||||
|
if err := os.WriteFile(path, []byte(legacy), 0644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
emb := memory.NewStaticEmbedder()
|
||||||
|
rc := NewRelevanceContext(path, emb)
|
||||||
|
if rc.Len() != 1 {
|
||||||
|
t.Fatalf("应读回 1 条,实际 %d", rc.Len())
|
||||||
|
}
|
||||||
|
|
||||||
|
// 新写入带记忆块的事件,再读回
|
||||||
|
ms, err := media.New(filepath.Join(dir, "media"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer ms.Close()
|
||||||
|
a := &Agent{mediaStore: ms, context: rc}
|
||||||
|
d, _ := ms.Put([]byte("img"), media.Item{MIME: "image/png"})
|
||||||
|
evt := ContextEvent{Timestamp: time.Now(), Source: "qq", Input: "新数据"}
|
||||||
|
a.bindEventMedia(&evt, []string{d})
|
||||||
|
rc.Append(evt)
|
||||||
|
rc.flush()
|
||||||
|
|
||||||
|
rc2 := NewRelevanceContext(path, emb)
|
||||||
|
if rc2.Len() != 2 {
|
||||||
|
t.Fatalf("应有 2 条,实际 %d", rc2.Len())
|
||||||
|
}
|
||||||
|
var persisted int
|
||||||
|
for _, e := range rc2.Recent(10) {
|
||||||
|
persisted += len(e.Blocks)
|
||||||
|
}
|
||||||
|
if persisted != 1 {
|
||||||
|
t.Fatalf("块应随 context.json 持久化,实际 %d 个", persisted)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPruneMigratesBlocksToDocument(t *testing.T) {
|
||||||
|
// 一等记忆块的 L0→L2 迁移:块随事件离开 Context、进入 Document,
|
||||||
|
// 身份(ID/模态/digest/向量)原样保留;同一块不能同时留在两层。
|
||||||
|
// 这条路径不依赖 media_refs/ref_count。
|
||||||
|
dir := t.TempDir()
|
||||||
|
emb := memory.NewStaticEmbedder()
|
||||||
|
docStore := document.NewStore(filepath.Join(dir, "docs"), memory.TokenizeWords)
|
||||||
|
if err := docStore.Start(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
rc := NewRelevanceContext(filepath.Join(dir, "context.json"), emb)
|
||||||
|
|
||||||
|
block := memory.MemoryBlock{
|
||||||
|
ID: "blk_migrate_1", Modality: memory.BlockImage,
|
||||||
|
PayloadDigest: "deadbeef", MIME: "image/png", Size: 42,
|
||||||
|
Vector: []float64{0.1, 0.2, 0.3}, Fingerprint: "qwen:test",
|
||||||
|
}
|
||||||
|
rc.Append(ContextEvent{
|
||||||
|
Timestamp: time.Now().Add(-time.Hour),
|
||||||
|
Source: "qq", Input: "很久以前的一张图",
|
||||||
|
Blocks: []memory.MemoryBlock{block},
|
||||||
|
})
|
||||||
|
for i := 0; i < 12; i++ {
|
||||||
|
rc.Append(ContextEvent{
|
||||||
|
Timestamp: time.Now().Add(time.Duration(i) * time.Second),
|
||||||
|
Source: "qq", Input: "无关内容",
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
if n := rc.Prune("完全不相关的查询", 5, docStore); n == 0 {
|
||||||
|
t.Fatal("应有事件被归档")
|
||||||
|
}
|
||||||
|
|
||||||
|
// 块应已到达 L2,且身份不变。
|
||||||
|
var found *document.Doc
|
||||||
|
for _, d := range docStore.RecentDocs(20) {
|
||||||
|
if len(d.Blocks) > 0 {
|
||||||
|
found = d
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if found == nil {
|
||||||
|
t.Fatal("归档文档应持有一等记忆块")
|
||||||
|
}
|
||||||
|
if len(found.Blocks) != 1 {
|
||||||
|
t.Fatalf("文档应有 1 个块,实际 %d", len(found.Blocks))
|
||||||
|
}
|
||||||
|
got := found.Blocks[0]
|
||||||
|
if got.ID != block.ID || got.Modality != block.Modality || got.PayloadDigest != block.PayloadDigest || got.Fingerprint != block.Fingerprint || len(got.Vector) != len(block.Vector) {
|
||||||
|
t.Fatalf("块身份应原样迁移:\n got %+v\n want %+v", got, block)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 同一块不能同时留在 L0。
|
||||||
|
for _, e := range rc.Recent(100) {
|
||||||
|
if len(e.Blocks) > 0 {
|
||||||
|
t.Fatalf("块仍留在 L0(同一块同时存在于两层): %+v", e.Blocks)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
362
internal/agent/core/modalfallback.go
Normal file
362
internal/agent/core/modalfallback.go
Normal file
@ -0,0 +1,362 @@
|
|||||||
|
package core
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"log"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
agentAPI "gitcode.com/JianFeeeee/HomeAgent/internal/agent/api"
|
||||||
|
)
|
||||||
|
|
||||||
|
// 多模态回退链:主模型看不到图/听不到音频时,改用一个声明了 vision/audio
|
||||||
|
// 能力的源把媒体转写成文字,再以 text block 注入。
|
||||||
|
//
|
||||||
|
// 为何必须有这条链:core.llm.model=AUTO 时实际落到哪个上游由网关按优先级决定,
|
||||||
|
// 而网关可能把 image_url 块静默剥离后转发给纯文本上游(llmsproxy 的
|
||||||
|
// opencode adapter 就明写着 "multimodal part not supported by zen" 并丢弃
|
||||||
|
// 非 text part)。请求依然返回 200,带图与不带图的 prompt_tokens 完全相同,
|
||||||
|
// 模型于是回答「我没有看到图片」,而内核以为注入成功。
|
||||||
|
//
|
||||||
|
// 没有这条链的话,multimodal 插件在任何非视觉主模型下都只能假成功。
|
||||||
|
|
||||||
|
const (
|
||||||
|
// modalFallbackTimeout 单次转写调用的上限。
|
||||||
|
//
|
||||||
|
// 为何是 180s:生产实测经网关转 claude-opus-5 看一张 400x400 图要 ~81s,
|
||||||
|
// 90s 阅则定时贴着上限,多帧批量请求更慢。宁可等也不要徒劳一趟。
|
||||||
|
modalFallbackTimeout = 180 * time.Second
|
||||||
|
|
||||||
|
// modalFallbackMaxTokens 转写输出上限。描述一组图/一段音频不需要长文,
|
||||||
|
// 且这段文字要塞回主模型上下文,过长会挤掉真正的对话内容。
|
||||||
|
modalFallbackMaxTokens = 1500
|
||||||
|
|
||||||
|
// modalFallbackMaxBlocks 单次最多转写多少个媒体块。
|
||||||
|
// see_video 一次能注入 10 帧;即使批量合包,图越多上游越慢也越容易超
|
||||||
|
// 单请求体积限制。超出部分如实报告未转写。
|
||||||
|
modalFallbackMaxBlocks = 6
|
||||||
|
)
|
||||||
|
|
||||||
|
// modalFallbackResult 描述一次回退转写的结果,供调用方决定注入什么。
|
||||||
|
type modalFallbackResult struct {
|
||||||
|
// Text 是转写出的文字(已含来源标注),为空表示没有可注入内容。
|
||||||
|
Text string
|
||||||
|
// Converted 实际成功转写的块数。
|
||||||
|
Converted int
|
||||||
|
// Skipped 因超出 modalFallbackMaxBlocks 而未处理的块数。
|
||||||
|
Skipped int
|
||||||
|
// Notice 给模型看的说明(能力缺失、转写失败等),始终如实。
|
||||||
|
Notice string
|
||||||
|
}
|
||||||
|
|
||||||
|
// resolveModalFallback 按配置挑一个能处理该模态的 provider。
|
||||||
|
//
|
||||||
|
// 顺序:配置指定的 fallback_provider → 任意声明了该能力的已注册源。
|
||||||
|
// 后者是刻意的兜底:用户可能只在源上声明了 vision 而忘了填 fallback_provider,
|
||||||
|
// 此时静默失败比多找一个能用的源更糟。
|
||||||
|
func (a *Agent) resolveModalFallback(kind string) (agentAPI.Provider, string) {
|
||||||
|
if a.providerManager == nil {
|
||||||
|
return nil, ""
|
||||||
|
}
|
||||||
|
|
||||||
|
var configured string
|
||||||
|
switch kind {
|
||||||
|
case "image":
|
||||||
|
configured = strings.TrimSpace(a.inputCfg.Image.FallbackProvider)
|
||||||
|
case "audio":
|
||||||
|
configured = strings.TrimSpace(a.inputCfg.Audio.FallbackProvider)
|
||||||
|
}
|
||||||
|
|
||||||
|
supports := func(p agentAPI.Provider) bool {
|
||||||
|
if p == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if kind == "audio" {
|
||||||
|
return agentAPI.ProviderSupportsAudio(p)
|
||||||
|
}
|
||||||
|
return agentAPI.ProviderSupportsVision(p)
|
||||||
|
}
|
||||||
|
|
||||||
|
if configured != "" {
|
||||||
|
p := a.providerManager.Get(configured)
|
||||||
|
if p == nil {
|
||||||
|
log.Printf("[agent] modal fallback %s: configured provider %q not registered", kind, configured)
|
||||||
|
} else if !supports(p) {
|
||||||
|
// 配置指向了一个没声明该能力的源:照用只会重演静默剥离,
|
||||||
|
// 因此拒绝并继续找,日志点明配置与声明不一致。
|
||||||
|
log.Printf("[agent] modal fallback %s: provider %q does not declare the capability, ignoring", kind, configured)
|
||||||
|
} else if !a.providerManager.IsAvailable(configured) {
|
||||||
|
log.Printf("[agent] modal fallback %s: provider %q in cooldown, trying others", kind, configured)
|
||||||
|
} else {
|
||||||
|
return p, configured
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 兜底:扫已注册源,取第一个声明了该能力且当前可用的。
|
||||||
|
for _, name := range a.providerManager.List() {
|
||||||
|
if name == configured {
|
||||||
|
continue // 上面已试过
|
||||||
|
}
|
||||||
|
p := a.providerManager.Get(name)
|
||||||
|
if supports(p) && a.providerManager.IsAvailable(name) {
|
||||||
|
return p, name
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil, ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// modalFallbackModel 返回该模态回退调用应使用的模型名(空则用源自身默认)。
|
||||||
|
func (a *Agent) modalFallbackModel(kind string) string {
|
||||||
|
switch kind {
|
||||||
|
case "image":
|
||||||
|
return strings.TrimSpace(a.inputCfg.Image.FallbackModel)
|
||||||
|
case "audio":
|
||||||
|
return strings.TrimSpace(a.inputCfg.Audio.FallbackModel)
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// modalFallbackPrompt 返回转写用的提示词,配置为空时给一个可用默认。
|
||||||
|
func (a *Agent) modalFallbackPrompt(kind string) string {
|
||||||
|
switch kind {
|
||||||
|
case "image":
|
||||||
|
if s := strings.TrimSpace(a.inputCfg.Image.DescribePrompt); s != "" {
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
return "请详细描述这张图片的内容,包括其中的文字、物体、人物、场景等信息。"
|
||||||
|
case "audio":
|
||||||
|
if s := strings.TrimSpace(a.inputCfg.Audio.DescribePrompt); s != "" {
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
return "请转写这段音频的内容。"
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// transcribeBlocksForFallback 把主模型看不懂的媒体块转写成文字。
|
||||||
|
//
|
||||||
|
// blocks 里的 text 块原样保留(它们本来就能被理解);image_url/audio_url
|
||||||
|
// **按模态批量合包,每类只发一次请求**。
|
||||||
|
//
|
||||||
|
// 为何必须批量而不是逐块:生产实测 see_video 注入 6 帧时,逐帧调用让
|
||||||
|
// 4 帧里 3 帧超时,整轮拖到 363 秒。而视觉模型本来就能在一条消息里看
|
||||||
|
// 多张图——一次调用不仅快上一个数量级,模型还能看到帧与帧的时间推进
|
||||||
|
// 关系,分析质量更好。
|
||||||
|
//
|
||||||
|
// 返回的 Text 已带来源标注,让模型知道这是转写而非自己直接看到的。这点
|
||||||
|
// 很重要:模型据此能判断细节可靠性,也不会在用户追问像素级细节时编造。
|
||||||
|
func (a *Agent) transcribeBlocksForFallback(blocks []agentAPI.ContentBlock) modalFallbackResult {
|
||||||
|
var res modalFallbackResult
|
||||||
|
var kept []string // 原样保留的 text 块
|
||||||
|
var converted []string // 转写结果
|
||||||
|
var notices []string
|
||||||
|
|
||||||
|
// 先按模态分组,同时应用块数上限。
|
||||||
|
var imgURLs, imgDetails []string
|
||||||
|
var audURLs []string
|
||||||
|
mediaSeen := 0
|
||||||
|
for _, b := range blocks {
|
||||||
|
switch b.Type {
|
||||||
|
case "text":
|
||||||
|
if b.Text != "" {
|
||||||
|
kept = append(kept, b.Text)
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
case "image_url":
|
||||||
|
mediaSeen++
|
||||||
|
if mediaSeen > modalFallbackMaxBlocks {
|
||||||
|
res.Skipped++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if b.ImageURL != nil && b.ImageURL.URL != "" {
|
||||||
|
imgURLs = append(imgURLs, b.ImageURL.URL)
|
||||||
|
imgDetails = append(imgDetails, b.ImageURL.Detail)
|
||||||
|
}
|
||||||
|
case "audio_url":
|
||||||
|
mediaSeen++
|
||||||
|
if mediaSeen > modalFallbackMaxBlocks {
|
||||||
|
res.Skipped++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if b.AudioURL != nil && b.AudioURL.URL != "" {
|
||||||
|
audURLs = append(audURLs, b.AudioURL.URL)
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
continue // 未知块类型:主模型也看不懂,丢弃
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 图片:一次请求带全部帧
|
||||||
|
if len(imgURLs) > 0 {
|
||||||
|
p, srcName := a.resolveModalFallback("image")
|
||||||
|
if p == nil {
|
||||||
|
notices = append(notices, "当前模型不支持图片,且没有可用的视觉回退源"+
|
||||||
|
"(配置 core.input_processing.image.fallback_provider,"+
|
||||||
|
"并在该源上设置 core.llm.sources.<name>.vision=true)")
|
||||||
|
} else if text, err := a.chatModalFallbackBatch(p, "image", imgURLs, imgDetails); err != nil {
|
||||||
|
// 转写失败必须说出来。静默跳过会让模型以为「图里没内容」,
|
||||||
|
// 而事实是没人看过这些图。
|
||||||
|
notices = append(notices, fmt.Sprintf("图片转写失败(源 %s): %v", srcName, err))
|
||||||
|
log.Printf("[agent] modal fallback image(%d) via %s failed: %v", len(imgURLs), srcName, err)
|
||||||
|
} else {
|
||||||
|
label := "图片内容"
|
||||||
|
if len(imgURLs) > 1 {
|
||||||
|
label = fmt.Sprintf("%d 张图片/视频帧内容", len(imgURLs))
|
||||||
|
}
|
||||||
|
converted = append(converted, fmt.Sprintf("[%s · 由 %s 转写,非当前模型直接感知]\n%s", label, srcName, text))
|
||||||
|
res.Converted += len(imgURLs)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 音频:同样一次请求
|
||||||
|
if len(audURLs) > 0 {
|
||||||
|
p, srcName := a.resolveModalFallback("audio")
|
||||||
|
if p == nil {
|
||||||
|
notices = append(notices, "当前模型不支持音频,且没有可用的音频回退源"+
|
||||||
|
"(配置 core.input_processing.audio.fallback_provider,"+
|
||||||
|
"并在该源上设置 core.llm.sources.<name>.audio=true)")
|
||||||
|
} else if text, err := a.chatModalFallbackBatch(p, "audio", audURLs, nil); err != nil {
|
||||||
|
notices = append(notices, fmt.Sprintf("音频转写失败(源 %s): %v", srcName, err))
|
||||||
|
log.Printf("[agent] modal fallback audio(%d) via %s failed: %v", len(audURLs), srcName, err)
|
||||||
|
} else {
|
||||||
|
converted = append(converted, fmt.Sprintf("[音频内容 · 由 %s 转写,非当前模型直接感知]\n%s", srcName, text))
|
||||||
|
res.Converted += len(audURLs)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if res.Skipped > 0 {
|
||||||
|
notices = append(notices, fmt.Sprintf(
|
||||||
|
"另有 %d 个媒体块未转写(单次上限 %d)",
|
||||||
|
res.Skipped, modalFallbackMaxBlocks))
|
||||||
|
}
|
||||||
|
|
||||||
|
var parts []string
|
||||||
|
parts = append(parts, kept...)
|
||||||
|
parts = append(parts, converted...)
|
||||||
|
if len(notices) > 0 {
|
||||||
|
parts = append(parts, "[注意] "+strings.Join(notices, ";"))
|
||||||
|
}
|
||||||
|
res.Text = strings.Join(parts, "\n\n")
|
||||||
|
res.Notice = strings.Join(notices, ";")
|
||||||
|
return res
|
||||||
|
}
|
||||||
|
|
||||||
|
// prepareToolBlocks 判定当前主模型能否直接消费这批媒体块。
|
||||||
|
//
|
||||||
|
// 返回 (native, ""):能直接看/听,原样作为 Blocks 注入。
|
||||||
|
// 返回 (nil, text) :不能,已经回退链转写成文字,调用方并进纯文本 content。
|
||||||
|
// 返回 (nil, "") :既不能直接看也没回退源且无话可说(理论上不发生,
|
||||||
|
// transcribeBlocksForFallback 至少会给一条 notice)。
|
||||||
|
//
|
||||||
|
// 为何逐模态判定而不是一刀切:一批块里可能图能看、音频不能听(很多视觉
|
||||||
|
// 模型就是这样)。全部走回退会白白把本可直视的图降级成二手文字描述。
|
||||||
|
func (a *Agent) prepareToolBlocks(blocks []agentAPI.ContentBlock) ([]agentAPI.ContentBlock, string) {
|
||||||
|
canVision := agentAPI.ProviderSupportsVision(a.provider)
|
||||||
|
canAudio := agentAPI.ProviderSupportsAudio(a.provider)
|
||||||
|
|
||||||
|
var native []agentAPI.ContentBlock
|
||||||
|
var needFallback []agentAPI.ContentBlock
|
||||||
|
for _, b := range blocks {
|
||||||
|
switch b.Type {
|
||||||
|
case "image_url":
|
||||||
|
if canVision {
|
||||||
|
native = append(native, b)
|
||||||
|
} else {
|
||||||
|
needFallback = append(needFallback, b)
|
||||||
|
}
|
||||||
|
case "audio_url":
|
||||||
|
if canAudio {
|
||||||
|
native = append(native, b)
|
||||||
|
} else {
|
||||||
|
needFallback = append(needFallback, b)
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
native = append(native, b) // text 等一律直通
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(needFallback) == 0 {
|
||||||
|
return native, ""
|
||||||
|
}
|
||||||
|
|
||||||
|
res := a.transcribeBlocksForFallback(needFallback)
|
||||||
|
log.Printf("[agent] modal fallback: %d block(s) transcribed, %d skipped (provider=%s vision=%v audio=%v)",
|
||||||
|
res.Converted, res.Skipped, a.provider.Name(), canVision, canAudio)
|
||||||
|
|
||||||
|
// 部分能直视、部分需转写:把转写文字作为 text 块并入 native,
|
||||||
|
// 这样两部分内容同时到达模型。
|
||||||
|
if len(native) > 0 {
|
||||||
|
if res.Text != "" {
|
||||||
|
native = append(native, agentAPI.ContentBlock{Type: "text", Text: res.Text})
|
||||||
|
}
|
||||||
|
return native, ""
|
||||||
|
}
|
||||||
|
return nil, res.Text
|
||||||
|
}
|
||||||
|
|
||||||
|
// chatModalFallbackBatch 向回退 provider 发**一次**请求,带上该模态的全部媒体。
|
||||||
|
//
|
||||||
|
// 多张图合包而非逐张调用:既为避开 N 倍往返延迟(生产实测逐帧调用使
|
||||||
|
// see_video 6 帧拖到 363 秒且 3/4 帧超时),也因为视觉模型看到成组帧时能
|
||||||
|
// 描述帧间变化,而逐帧转写只能得到 N 段互不相关的静态描述。
|
||||||
|
func (a *Agent) chatModalFallbackBatch(p agentAPI.Provider, kind string, urls, details []string) (string, error) {
|
||||||
|
if len(urls) == 0 {
|
||||||
|
return "", fmt.Errorf("no media to transcribe")
|
||||||
|
}
|
||||||
|
|
||||||
|
prompt := a.modalFallbackPrompt(kind)
|
||||||
|
if len(urls) > 1 && kind == "image" {
|
||||||
|
// 多张时补一句,否则模型容易只描述第一张。
|
||||||
|
prompt = fmt.Sprintf("%s\n\n共 %d 张(若为视频关键帧则按时间顺序),"+
|
||||||
|
"请逐张编号描述,并在最后概括帧间变化。", prompt, len(urls))
|
||||||
|
}
|
||||||
|
|
||||||
|
msg := agentAPI.Message{
|
||||||
|
Role: "user",
|
||||||
|
Blocks: []agentAPI.ContentBlock{{Type: "text", Text: prompt}},
|
||||||
|
}
|
||||||
|
for i, u := range urls {
|
||||||
|
if kind == "audio" {
|
||||||
|
msg.Blocks = append(msg.Blocks, agentAPI.ContentBlock{
|
||||||
|
Type: "audio_url",
|
||||||
|
AudioURL: &agentAPI.AudioURL{URL: u},
|
||||||
|
})
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
detail := ""
|
||||||
|
if i < len(details) {
|
||||||
|
detail = details[i]
|
||||||
|
}
|
||||||
|
if detail == "" {
|
||||||
|
// 单张时看清细节;多张(视频帧)用 low 控住体积与耗时。
|
||||||
|
if len(urls) > 1 {
|
||||||
|
detail = "low"
|
||||||
|
} else {
|
||||||
|
detail = "high"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
msg.Blocks = append(msg.Blocks, agentAPI.ContentBlock{
|
||||||
|
Type: "image_url",
|
||||||
|
ImageURL: &agentAPI.ImageURL{URL: u, Detail: detail},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx, cancel := context.WithTimeout(a.ctx, modalFallbackTimeout)
|
||||||
|
defer cancel()
|
||||||
|
resp, err := p.Chat(ctx, &agentAPI.CompletionRequest{
|
||||||
|
Model: a.modalFallbackModel(kind),
|
||||||
|
Messages: []agentAPI.Message{msg},
|
||||||
|
MaxTokens: modalFallbackMaxTokens,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
out := strings.TrimSpace(resp.Content)
|
||||||
|
if out == "" {
|
||||||
|
// 空回复不能当成功。上游剥掉媒体块后模型往往回一句「我没看到图片」
|
||||||
|
// 或干脆空串——两种都说明这条回退链也没真看到。
|
||||||
|
return "", fmt.Errorf("回退源返回空内容(该源可能同样不支持此模态)")
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
428
internal/agent/core/modalfallback_test.go
Normal file
428
internal/agent/core/modalfallback_test.go
Normal file
@ -0,0 +1,428 @@
|
|||||||
|
package core
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
agentAPI "gitcode.com/JianFeeeee/HomeAgent/internal/agent/api"
|
||||||
|
"gitcode.com/JianFeeeee/HomeAgent/pkg/types"
|
||||||
|
)
|
||||||
|
|
||||||
|
// stubProvider 是一个可声明多模态能力的假 provider。
|
||||||
|
// 记录收到的请求,供断言「回退链是否真的调了它」。
|
||||||
|
type stubProvider struct {
|
||||||
|
name string
|
||||||
|
vision bool
|
||||||
|
audio bool
|
||||||
|
reply string
|
||||||
|
err error
|
||||||
|
calls int
|
||||||
|
lastReq *agentAPI.CompletionRequest
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *stubProvider) Name() string { return s.name }
|
||||||
|
func (s *stubProvider) Chat(ctx context.Context, req *agentAPI.CompletionRequest) (*agentAPI.CompletionResponse, error) {
|
||||||
|
s.calls++
|
||||||
|
s.lastReq = req
|
||||||
|
if s.err != nil {
|
||||||
|
return nil, s.err
|
||||||
|
}
|
||||||
|
return &agentAPI.CompletionResponse{Content: s.reply}, nil
|
||||||
|
}
|
||||||
|
func (s *stubProvider) ChatStream(ctx context.Context, req *agentAPI.CompletionRequest) (<-chan agentAPI.StreamChunk, error) {
|
||||||
|
ch := make(chan agentAPI.StreamChunk)
|
||||||
|
close(ch)
|
||||||
|
return ch, nil
|
||||||
|
}
|
||||||
|
func (s *stubProvider) MaxContextTokens() int { return 8192 }
|
||||||
|
func (s *stubProvider) SupportsVision() bool { return s.vision }
|
||||||
|
func (s *stubProvider) SupportsAudio() bool { return s.audio }
|
||||||
|
|
||||||
|
// plainProvider 不实现 ModalProvider,用于验证「未声明即按不支持处理」。
|
||||||
|
type plainProvider struct{ name string }
|
||||||
|
|
||||||
|
func (p *plainProvider) Name() string { return p.name }
|
||||||
|
func (p *plainProvider) Chat(ctx context.Context, req *agentAPI.CompletionRequest) (*agentAPI.CompletionResponse, error) {
|
||||||
|
return &agentAPI.CompletionResponse{Content: "ok"}, nil
|
||||||
|
}
|
||||||
|
func (p *plainProvider) ChatStream(ctx context.Context, req *agentAPI.CompletionRequest) (<-chan agentAPI.StreamChunk, error) {
|
||||||
|
ch := make(chan agentAPI.StreamChunk)
|
||||||
|
close(ch)
|
||||||
|
return ch, nil
|
||||||
|
}
|
||||||
|
func (p *plainProvider) MaxContextTokens() int { return 8192 }
|
||||||
|
|
||||||
|
const testPNG = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUg=="
|
||||||
|
const testWAV = "data:audio/wav;base64,UklGRiQAAABXQVZF"
|
||||||
|
|
||||||
|
func imageBlock() agentAPI.ContentBlock {
|
||||||
|
return agentAPI.ContentBlock{
|
||||||
|
Type: "image_url",
|
||||||
|
ImageURL: &agentAPI.ImageURL{URL: testPNG, Detail: "auto"},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func audioBlock() agentAPI.ContentBlock {
|
||||||
|
return agentAPI.ContentBlock{
|
||||||
|
Type: "audio_url",
|
||||||
|
AudioURL: &agentAPI.AudioURL{URL: testWAV},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// newFallbackAgent 组装一个只带 provider/manager/inputCfg 的最小 Agent。
|
||||||
|
// 不走 New():那会拉起 embedder、记忆、后台循环,与本测试无关。
|
||||||
|
func newFallbackAgent(main agentAPI.Provider, mgr *agentAPI.ProviderManager, cfg types.InputProcessingConfig) *Agent {
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
return &Agent{
|
||||||
|
provider: main,
|
||||||
|
providerManager: mgr,
|
||||||
|
inputCfg: cfg,
|
||||||
|
ctx: ctx,
|
||||||
|
cancel: cancel,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPrepareToolBlocks_VisionCapableProviderPassesThrough(t *testing.T) {
|
||||||
|
main := &stubProvider{name: "vision-main", vision: true}
|
||||||
|
mgr := agentAPI.NewProviderManager()
|
||||||
|
mgr.Register("vision-main", main)
|
||||||
|
a := newFallbackAgent(main, mgr, types.InputProcessingConfig{})
|
||||||
|
|
||||||
|
native, fallbackText := a.prepareToolBlocks([]agentAPI.ContentBlock{imageBlock()})
|
||||||
|
|
||||||
|
if len(native) != 1 || native[0].Type != "image_url" {
|
||||||
|
t.Fatalf("能看图的主模型应原样透传 image_url,得到 %+v", native)
|
||||||
|
}
|
||||||
|
if fallbackText != "" {
|
||||||
|
t.Fatalf("不该触发回退,却返回了文字: %q", fallbackText)
|
||||||
|
}
|
||||||
|
if main.calls != 0 {
|
||||||
|
t.Fatalf("不该额外调用 provider,实际调了 %d 次", main.calls)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPrepareToolBlocks_TextOnlyProviderFallsBackToTranscription(t *testing.T) {
|
||||||
|
main := &stubProvider{name: "text-main"} // vision=false
|
||||||
|
vis := &stubProvider{name: "vis-src", vision: true, reply: "一只橘猫坐在窗台上"}
|
||||||
|
mgr := agentAPI.NewProviderManager()
|
||||||
|
mgr.Register("text-main", main)
|
||||||
|
mgr.Register("vis-src", vis)
|
||||||
|
|
||||||
|
a := newFallbackAgent(main, mgr, types.InputProcessingConfig{
|
||||||
|
Image: types.ImageProcessingConfig{
|
||||||
|
FallbackProvider: "vis-src",
|
||||||
|
DescribePrompt: "描述这张图",
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
native, fallbackText := a.prepareToolBlocks([]agentAPI.ContentBlock{imageBlock()})
|
||||||
|
|
||||||
|
if len(native) != 0 {
|
||||||
|
t.Fatalf("纯文本主模型不该收到原生块,得到 %+v", native)
|
||||||
|
}
|
||||||
|
if !strings.Contains(fallbackText, "一只橘猫坐在窗台上") {
|
||||||
|
t.Fatalf("回退文字应含转写内容,得到 %q", fallbackText)
|
||||||
|
}
|
||||||
|
// 关键:模型必须知道这是二手转写而非自己直接看到的
|
||||||
|
if !strings.Contains(fallbackText, "非当前模型直接感知") {
|
||||||
|
t.Fatalf("回退文字必须标注来源,得到 %q", fallbackText)
|
||||||
|
}
|
||||||
|
if vis.calls != 1 {
|
||||||
|
t.Fatalf("应调用视觉源 1 次,实际 %d", vis.calls)
|
||||||
|
}
|
||||||
|
if main.calls != 0 {
|
||||||
|
t.Fatalf("不该拿图去问纯文本主模型,实际调了 %d 次", main.calls)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPrepareToolBlocks_NoFallbackSourceReportsHonestly(t *testing.T) {
|
||||||
|
main := &stubProvider{name: "text-main"}
|
||||||
|
mgr := agentAPI.NewProviderManager()
|
||||||
|
mgr.Register("text-main", main)
|
||||||
|
a := newFallbackAgent(main, mgr, types.InputProcessingConfig{})
|
||||||
|
|
||||||
|
native, fallbackText := a.prepareToolBlocks([]agentAPI.ContentBlock{imageBlock()})
|
||||||
|
|
||||||
|
if len(native) != 0 {
|
||||||
|
t.Fatalf("不该透传,得到 %+v", native)
|
||||||
|
}
|
||||||
|
// 这是本次修复的核心:没有能力也没有回退源时必须明说,
|
||||||
|
// 而不是静默丢弃让模型以为自己看过图了。
|
||||||
|
if !strings.Contains(fallbackText, "不支持图片") {
|
||||||
|
t.Fatalf("必须如实说明看不到图,得到 %q", fallbackText)
|
||||||
|
}
|
||||||
|
if !strings.Contains(fallbackText, "fallback_provider") {
|
||||||
|
t.Fatalf("应给出可操作的配置提示,得到 %q", fallbackText)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPrepareToolBlocks_ProviderWithoutModalInterfaceTreatedAsTextOnly(t *testing.T) {
|
||||||
|
main := &plainProvider{name: "legacy"} // 未实现 ModalProvider
|
||||||
|
mgr := agentAPI.NewProviderManager()
|
||||||
|
mgr.Register("legacy", main)
|
||||||
|
a := newFallbackAgent(main, mgr, types.InputProcessingConfig{})
|
||||||
|
|
||||||
|
native, fallbackText := a.prepareToolBlocks([]agentAPI.ContentBlock{imageBlock()})
|
||||||
|
|
||||||
|
if len(native) != 0 {
|
||||||
|
t.Fatalf("未声明能力的 provider 应按不支持处理,却透传了 %+v", native)
|
||||||
|
}
|
||||||
|
if fallbackText == "" {
|
||||||
|
t.Fatal("应给出说明而非静默")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPrepareToolBlocks_MixedModalitySplitsCorrectly(t *testing.T) {
|
||||||
|
// 主模型能看图但听不到音频——很多视觉模型正是这样。
|
||||||
|
// 图应直视,只有音频走回退,不能一刀切全部降级。
|
||||||
|
main := &stubProvider{name: "vision-only", vision: true}
|
||||||
|
aud := &stubProvider{name: "aud-src", audio: true, reply: "背景有钢琴声"}
|
||||||
|
mgr := agentAPI.NewProviderManager()
|
||||||
|
mgr.Register("vision-only", main)
|
||||||
|
mgr.Register("aud-src", aud)
|
||||||
|
|
||||||
|
a := newFallbackAgent(main, mgr, types.InputProcessingConfig{
|
||||||
|
Audio: types.AudioProcessingConfig{FallbackProvider: "aud-src"},
|
||||||
|
})
|
||||||
|
|
||||||
|
native, fallbackText := a.prepareToolBlocks([]agentAPI.ContentBlock{imageBlock(), audioBlock()})
|
||||||
|
|
||||||
|
if fallbackText != "" {
|
||||||
|
t.Fatalf("有原生块时转写应并入 native,不该走 content 分支,得到 %q", fallbackText)
|
||||||
|
}
|
||||||
|
var imgCount, textCount int
|
||||||
|
for _, b := range native {
|
||||||
|
switch b.Type {
|
||||||
|
case "image_url":
|
||||||
|
imgCount++
|
||||||
|
case "text":
|
||||||
|
textCount++
|
||||||
|
if !strings.Contains(b.Text, "背景有钢琴声") {
|
||||||
|
t.Fatalf("text 块应含音频转写,得到 %q", b.Text)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if imgCount != 1 {
|
||||||
|
t.Fatalf("图应原样保留 1 个,得到 %d", imgCount)
|
||||||
|
}
|
||||||
|
if textCount != 1 {
|
||||||
|
t.Fatalf("音频转写应产出 1 个 text 块,得到 %d", textCount)
|
||||||
|
}
|
||||||
|
if aud.calls != 1 {
|
||||||
|
t.Fatalf("应调音频源 1 次,实际 %d", aud.calls)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTranscribeBlocks_EmptyReplyCountsAsFailure(t *testing.T) {
|
||||||
|
// 回退源返回空串,往往意味着它上游也剥掉了媒体块。
|
||||||
|
// 这种情况绝不能当成功——否则又是一次假成功。
|
||||||
|
main := &stubProvider{name: "text-main"}
|
||||||
|
vis := &stubProvider{name: "vis-src", vision: true, reply: " "}
|
||||||
|
mgr := agentAPI.NewProviderManager()
|
||||||
|
mgr.Register("text-main", main)
|
||||||
|
mgr.Register("vis-src", vis)
|
||||||
|
|
||||||
|
a := newFallbackAgent(main, mgr, types.InputProcessingConfig{
|
||||||
|
Image: types.ImageProcessingConfig{FallbackProvider: "vis-src"},
|
||||||
|
})
|
||||||
|
|
||||||
|
res := a.transcribeBlocksForFallback([]agentAPI.ContentBlock{imageBlock()})
|
||||||
|
|
||||||
|
if res.Converted != 0 {
|
||||||
|
t.Fatalf("空回复不应计入成功转写,得到 Converted=%d", res.Converted)
|
||||||
|
}
|
||||||
|
if !strings.Contains(res.Notice, "转写失败") {
|
||||||
|
t.Fatalf("应报告转写失败,得到 Notice=%q", res.Notice)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTranscribeBlocks_CapsBlockCount(t *testing.T) {
|
||||||
|
// see_video 能一次注入 10 帧。上限存在的理由不再是“逐帧调用慢”(现已合包),
|
||||||
|
// 而是图越多单请求体积越大、上游越慢且易超限。
|
||||||
|
main := &stubProvider{name: "text-main"}
|
||||||
|
vis := &stubProvider{name: "vis-src", vision: true, reply: "逐帧描述…"}
|
||||||
|
mgr := agentAPI.NewProviderManager()
|
||||||
|
mgr.Register("text-main", main)
|
||||||
|
mgr.Register("vis-src", vis)
|
||||||
|
|
||||||
|
a := newFallbackAgent(main, mgr, types.InputProcessingConfig{
|
||||||
|
Image: types.ImageProcessingConfig{FallbackProvider: "vis-src"},
|
||||||
|
})
|
||||||
|
|
||||||
|
blocks := make([]agentAPI.ContentBlock, 10)
|
||||||
|
for i := range blocks {
|
||||||
|
blocks[i] = imageBlock()
|
||||||
|
}
|
||||||
|
res := a.transcribeBlocksForFallback(blocks)
|
||||||
|
|
||||||
|
if res.Converted != modalFallbackMaxBlocks {
|
||||||
|
t.Fatalf("应只转写 %d 个,实际 %d", modalFallbackMaxBlocks, res.Converted)
|
||||||
|
}
|
||||||
|
if res.Skipped != 10-modalFallbackMaxBlocks {
|
||||||
|
t.Fatalf("应跳过 %d 个,实际 %d", 10-modalFallbackMaxBlocks, res.Skipped)
|
||||||
|
}
|
||||||
|
// 批量合包:上限内的帧应合成**一次**调用,而不是每帧一次。
|
||||||
|
// 生产实测逐帧调用使 see_video 6 帧拖到 363s 且 3 帧超时。
|
||||||
|
if vis.calls != 1 {
|
||||||
|
t.Fatalf("多帧应合包为 1 次调用,实际 %d 次", vis.calls)
|
||||||
|
}
|
||||||
|
// 且那一次请求里应带满上限数量的 image 块(加一个 text 提示块)
|
||||||
|
if vis.lastReq == nil || len(vis.lastReq.Messages) != 1 {
|
||||||
|
t.Fatal("应只发一条 user 消息")
|
||||||
|
}
|
||||||
|
imgBlocks := 0
|
||||||
|
for _, b := range vis.lastReq.Messages[0].Blocks {
|
||||||
|
if b.Type == "image_url" {
|
||||||
|
imgBlocks++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if imgBlocks != modalFallbackMaxBlocks {
|
||||||
|
t.Fatalf("单次请求应带 %d 个 image 块,实际 %d", modalFallbackMaxBlocks, imgBlocks)
|
||||||
|
}
|
||||||
|
// 跳过的部分也必须告知,否则模型以为自己看全了整段视频
|
||||||
|
if !strings.Contains(res.Notice, "未转写") {
|
||||||
|
t.Fatalf("应告知有块未转写,得到 %q", res.Notice)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTranscribeBlocks_MultiImageBatchedIntoOneCall(t *testing.T) {
|
||||||
|
// 上限以内的多张图(典型:see_video 4 帧)同样只能一次调用。
|
||||||
|
main := &stubProvider{name: "text-main"}
|
||||||
|
vis := &stubProvider{name: "vis-src", vision: true, reply: "1) 开场 2) 中段 3) 结尾"}
|
||||||
|
mgr := agentAPI.NewProviderManager()
|
||||||
|
mgr.Register("text-main", main)
|
||||||
|
mgr.Register("vis-src", vis)
|
||||||
|
|
||||||
|
a := newFallbackAgent(main, mgr, types.InputProcessingConfig{
|
||||||
|
Image: types.ImageProcessingConfig{FallbackProvider: "vis-src"},
|
||||||
|
})
|
||||||
|
|
||||||
|
res := a.transcribeBlocksForFallback([]agentAPI.ContentBlock{
|
||||||
|
imageBlock(), imageBlock(), imageBlock(),
|
||||||
|
})
|
||||||
|
|
||||||
|
if vis.calls != 1 {
|
||||||
|
t.Fatalf("3 张图应合为 1 次调用,实际 %d", vis.calls)
|
||||||
|
}
|
||||||
|
if res.Converted != 3 {
|
||||||
|
t.Fatalf("应计入 3 个已转写,实际 %d", res.Converted)
|
||||||
|
}
|
||||||
|
if res.Skipped != 0 {
|
||||||
|
t.Fatalf("不该有跳过,实际 %d", res.Skipped)
|
||||||
|
}
|
||||||
|
// 多张时提示词应补上张数,否则模型容易只描述第一张
|
||||||
|
prompt := vis.lastReq.Messages[0].Blocks[0].Text
|
||||||
|
if !strings.Contains(prompt, "3 张") {
|
||||||
|
t.Fatalf("多张提示词应声明张数,得到 %q", prompt)
|
||||||
|
}
|
||||||
|
// 帧序列:插件显式给了 detail 就尊重它(see_video 本来就传 low),
|
||||||
|
// 只在未指定时才由回退链按张数选默认。
|
||||||
|
for _, b := range vis.lastReq.Messages[0].Blocks {
|
||||||
|
if b.Type == "image_url" && b.ImageURL.Detail != "auto" {
|
||||||
|
t.Fatalf("应保留插件显式指定的 detail=auto,得到 %q", b.ImageURL.Detail)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTranscribeBlocks_MultiImageDefaultsToLowDetail(t *testing.T) {
|
||||||
|
// 未指定 detail 的多张图(帧序列)用 low 控住体积与耗时。
|
||||||
|
main := &stubProvider{name: "text-main"}
|
||||||
|
vis := &stubProvider{name: "vis-src", vision: true, reply: "三帧描述"}
|
||||||
|
mgr := agentAPI.NewProviderManager()
|
||||||
|
mgr.Register("text-main", main)
|
||||||
|
mgr.Register("vis-src", vis)
|
||||||
|
|
||||||
|
a := newFallbackAgent(main, mgr, types.InputProcessingConfig{
|
||||||
|
Image: types.ImageProcessingConfig{FallbackProvider: "vis-src"},
|
||||||
|
})
|
||||||
|
|
||||||
|
bare := agentAPI.ContentBlock{Type: "image_url", ImageURL: &agentAPI.ImageURL{URL: testPNG}}
|
||||||
|
a.transcribeBlocksForFallback([]agentAPI.ContentBlock{bare, bare, bare})
|
||||||
|
|
||||||
|
for _, b := range vis.lastReq.Messages[0].Blocks {
|
||||||
|
if b.Type == "image_url" && b.ImageURL.Detail != "low" {
|
||||||
|
t.Fatalf("多张未指定时应默认 low,得到 %q", b.ImageURL.Detail)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTranscribeBlocks_SingleImageUsesHighDetail(t *testing.T) {
|
||||||
|
// 单张图(see_picture)要看清细节,不吝惜 token。
|
||||||
|
main := &stubProvider{name: "text-main"}
|
||||||
|
vis := &stubProvider{name: "vis-src", vision: true, reply: "一只橘猫"}
|
||||||
|
mgr := agentAPI.NewProviderManager()
|
||||||
|
mgr.Register("text-main", main)
|
||||||
|
mgr.Register("vis-src", vis)
|
||||||
|
|
||||||
|
a := newFallbackAgent(main, mgr, types.InputProcessingConfig{
|
||||||
|
Image: types.ImageProcessingConfig{FallbackProvider: "vis-src"},
|
||||||
|
})
|
||||||
|
|
||||||
|
// Detail 置空,让回退链自己定
|
||||||
|
a.transcribeBlocksForFallback([]agentAPI.ContentBlock{
|
||||||
|
{Type: "image_url", ImageURL: &agentAPI.ImageURL{URL: testPNG}},
|
||||||
|
})
|
||||||
|
|
||||||
|
for _, b := range vis.lastReq.Messages[0].Blocks {
|
||||||
|
if b.Type == "image_url" && b.ImageURL.Detail != "high" {
|
||||||
|
t.Fatalf("单张应用 high detail,得到 %q", b.ImageURL.Detail)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolveModalFallback_RejectsProviderNotDeclaringCapability(t *testing.T) {
|
||||||
|
// 配置指向一个没声明 vision 的源:照用只会重演静默剥离。
|
||||||
|
// 应拒绝它,并继续找真正声明了能力的源。
|
||||||
|
main := &stubProvider{name: "text-main"}
|
||||||
|
wrong := &stubProvider{name: "wrong-src"} // vision=false
|
||||||
|
right := &stubProvider{name: "right-src", vision: true}
|
||||||
|
mgr := agentAPI.NewProviderManager()
|
||||||
|
mgr.Register("text-main", main)
|
||||||
|
mgr.Register("wrong-src", wrong)
|
||||||
|
mgr.Register("right-src", right)
|
||||||
|
|
||||||
|
a := newFallbackAgent(main, mgr, types.InputProcessingConfig{
|
||||||
|
Image: types.ImageProcessingConfig{FallbackProvider: "wrong-src"},
|
||||||
|
})
|
||||||
|
|
||||||
|
p, name := a.resolveModalFallback("image")
|
||||||
|
if name != "right-src" {
|
||||||
|
t.Fatalf("应跳过未声明能力的 wrong-src 而选中 right-src,得到 %q", name)
|
||||||
|
}
|
||||||
|
if p == nil {
|
||||||
|
t.Fatal("应返回可用 provider")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolveModalFallback_ScansForCapableSourceWhenUnconfigured(t *testing.T) {
|
||||||
|
// 用户可能只在源上声明了 vision 却忘了填 fallback_provider。
|
||||||
|
// 静默失败比多找一个能用的源更糟。
|
||||||
|
main := &stubProvider{name: "text-main"}
|
||||||
|
vis := &stubProvider{name: "some-vision-src", vision: true}
|
||||||
|
mgr := agentAPI.NewProviderManager()
|
||||||
|
mgr.Register("text-main", main)
|
||||||
|
mgr.Register("some-vision-src", vis)
|
||||||
|
|
||||||
|
a := newFallbackAgent(main, mgr, types.InputProcessingConfig{})
|
||||||
|
|
||||||
|
_, name := a.resolveModalFallback("image")
|
||||||
|
if name != "some-vision-src" {
|
||||||
|
t.Fatalf("未配置时应扫出声明了能力的源,得到 %q", name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPrepareToolBlocks_TextBlocksAlwaysPassThrough(t *testing.T) {
|
||||||
|
main := &stubProvider{name: "text-main"}
|
||||||
|
mgr := agentAPI.NewProviderManager()
|
||||||
|
mgr.Register("text-main", main)
|
||||||
|
a := newFallbackAgent(main, mgr, types.InputProcessingConfig{})
|
||||||
|
|
||||||
|
native, fallbackText := a.prepareToolBlocks([]agentAPI.ContentBlock{
|
||||||
|
{Type: "text", Text: "纯文字说明"},
|
||||||
|
})
|
||||||
|
|
||||||
|
if len(native) != 1 || native[0].Text != "纯文字说明" {
|
||||||
|
t.Fatalf("text 块应无条件直通,得到 %+v / %q", native, fallbackText)
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -4,8 +4,8 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
agentIO "gitcode.com/JianFeeeee/HomeAgent/internal/agent/io"
|
|
||||||
agentAPI "gitcode.com/JianFeeeee/HomeAgent/internal/agent/api"
|
agentAPI "gitcode.com/JianFeeeee/HomeAgent/internal/agent/api"
|
||||||
|
agentIO "gitcode.com/JianFeeeee/HomeAgent/internal/agent/io"
|
||||||
sdk "gitcode.com/JianFeeeee/HomeAgent/internal/sdk"
|
sdk "gitcode.com/JianFeeeee/HomeAgent/internal/sdk"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -78,7 +78,10 @@ func (a *Agent) executeOutputSendTool(tc agentAPI.ToolCall) string {
|
|||||||
return fmt.Sprintf("[%s] 通道发送结果未确认:%s", channel, note)
|
return fmt.Sprintf("[%s] 通道发送结果未确认:%s", channel, note)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return fmt.Sprintf("已通过 [%s] 通道发送: %v", channel, result)
|
// 成功回执:只返回极简标记,不回传完整插件响应。
|
||||||
|
// 「已通过 [qq] 通道发送: map[status:sent message_id:xxx]」这类富回执
|
||||||
|
// 会驱动模型继续调用 output_send(回声效应),是 output loop 的根源之一。
|
||||||
|
return "ok"
|
||||||
}
|
}
|
||||||
|
|
||||||
a.io.EmitTextTo("agent_io", channel, payload)
|
a.io.EmitTextTo("agent_io", channel, payload)
|
||||||
|
|||||||
@ -15,6 +15,90 @@ import (
|
|||||||
pubsdk "gitcode.com/JianFeeeee/homeagent-sdk/sdk"
|
pubsdk "gitcode.com/JianFeeeee/homeagent-sdk/sdk"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// continuationPlaceholder 是工具轮之后补的 user 占位内容。
|
||||||
|
//
|
||||||
|
// zen 兼容网关要求请求最后一条必须是 user(thinking 续写模式校验),工具轮
|
||||||
|
// 产出 assistant/tool 结尾会被 400 拒绝;首轮 system 结尾不补,否则会覆盖
|
||||||
|
// 真实用户输入。
|
||||||
|
//
|
||||||
|
// 用独立常量 + 精确等值判定,是因为这条消息是**核心自己插入的**、不是用户输入,
|
||||||
|
// 所以可以安全地按内容识别并在补位前移除上一条,保证至多一条。
|
||||||
|
const continuationPlaceholder = "请根据以上工具结果继续。"
|
||||||
|
|
||||||
|
// replyDeliveredPlaceholder 是「本批工具调用全部是输出通道发送」之后补的占位。
|
||||||
|
//
|
||||||
|
// 为何不能继续用通用的「请继续」:异步通道(qq/wechat)的回复**只能**经
|
||||||
|
// output_send__* 交付(纯文本不送达,见 buildSystemPrompt 的输出规则)。于是
|
||||||
|
// 模型「已经回复完了」的表达形式就是一个工具调用,而紧随其后的
|
||||||
|
// 「请根据以上工具结果继续。」会被读成「还要再做一步」——能做的「一步」恰好
|
||||||
|
// 还是再发一条消息。两者叠加成自我强化的发送循环:生产实测单轮 34 次
|
||||||
|
// output_send__qq、持续 514 秒,直到 QQ 插件自己的循环保险拒绝发送才停下。
|
||||||
|
//
|
||||||
|
// 所以这里换成一条明确的终止许可:已回复完就直接返回纯文本收尾。
|
||||||
|
const replyDeliveredPlaceholder = "若你的回复已完成,直接返回纯文本即可结束本轮,无需再调用任何工具。"
|
||||||
|
|
||||||
|
// continuationFor 选择工具轮之后补位的 user 占位文案。
|
||||||
|
// replyOnly 表示上一批工具调用全部是输出通道发送(即模型刚交付了回复)。
|
||||||
|
func continuationFor(replyOnly bool) string {
|
||||||
|
if replyOnly {
|
||||||
|
return replyDeliveredPlaceholder
|
||||||
|
}
|
||||||
|
return continuationPlaceholder
|
||||||
|
}
|
||||||
|
|
||||||
|
// isOutputDeliveryTool 判断工具是否是「向输出通道交付内容」。
|
||||||
|
// output_send__{channel}_help 只是查询用法,不算交付。
|
||||||
|
func isOutputDeliveryTool(name string) bool {
|
||||||
|
return strings.HasPrefix(name, "output_send__") && !strings.HasSuffix(name, "_help")
|
||||||
|
}
|
||||||
|
|
||||||
|
// isContinuationPlaceholder 判断一条 user 消息是否是本机制插入的占位。
|
||||||
|
// 只按两个常量精确匹配,不碰任何真实用户消息。
|
||||||
|
func isContinuationPlaceholder(m agentAPI.Message) bool {
|
||||||
|
return m.Role == "user" &&
|
||||||
|
(m.Content == continuationPlaceholder || m.Content == replyDeliveredPlaceholder)
|
||||||
|
}
|
||||||
|
|
||||||
|
// toolOutputForQuery 返回用于相关性计算的工具输出**有效内容**。
|
||||||
|
//
|
||||||
|
// 为什么要过 Cleaner 而不是直接用原始 result:ContextPolicy=prune 的入参是
|
||||||
|
// **相关性查询向量**——它决定保留/归档哪些上下文事件。原始工具输出里混着
|
||||||
|
// ANSI 转义、base64、JSON 包装等噪声,直接拿去向量化会让打分失真。
|
||||||
|
// 而 ToolDef.Cleaner 的契约本就写着“仅在向量化/jieba/蒸馏时调用”,裁剪正是
|
||||||
|
// 在向量化,所以这里必须过它(此前只在构建事件向量时用了,裁剪查询漏了)。
|
||||||
|
//
|
||||||
|
// Cleaner 未注册或 RPC 失败时回退原文(清洗是计算层优化,不能因此丢内容);
|
||||||
|
// 返回空串时也回退——空串会让查询向量退化成零向量,裁剪就失去判据。
|
||||||
|
func (a *Agent) toolOutputForQuery(toolName, raw string) string {
|
||||||
|
if a.stageHost == nil {
|
||||||
|
return raw
|
||||||
|
}
|
||||||
|
cleaner := a.stageHost.ToolDefCleaner(toolName)
|
||||||
|
if cleaner == nil {
|
||||||
|
return raw
|
||||||
|
}
|
||||||
|
if cleaned := cleaner(raw); cleaned != "" {
|
||||||
|
return cleaned
|
||||||
|
}
|
||||||
|
return raw
|
||||||
|
}
|
||||||
|
|
||||||
|
// dropContinuationPlaceholders 移除此前由本机制插入的 user 占位。
|
||||||
|
//
|
||||||
|
// 为什么必须移除而不仅仅是“不再追加”:`msgs` 在循环外创建、循环内只增不减,
|
||||||
|
// 占位是核心自己插的、不是用户说的话。不移除的话,prompt 里就会线性叠上
|
||||||
|
// N 条一模一样的“继续”,把前缀上下文(含记忆注入)往后挤。
|
||||||
|
func dropContinuationPlaceholders(msgs []agentAPI.Message) []agentAPI.Message {
|
||||||
|
out := msgs[:0]
|
||||||
|
for _, m := range msgs {
|
||||||
|
if isContinuationPlaceholder(m) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out = append(out, m)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
func (a *Agent) process(input string, stageCtx *sdk.StageContext) (response string, toolsUsed []string, toolResults []ToolResultItem, err error) {
|
func (a *Agent) process(input string, stageCtx *sdk.StageContext) (response string, toolsUsed []string, toolResults []ToolResultItem, err error) {
|
||||||
a.mu.Lock()
|
a.mu.Lock()
|
||||||
defer a.mu.Unlock()
|
defer a.mu.Unlock()
|
||||||
@ -63,6 +147,9 @@ func (a *Agent) process(input string, stageCtx *sdk.StageContext) (response stri
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// lastBatchReplyOnly 记录上一批工具调用是否全部是输出通道发送。
|
||||||
|
lastBatchReplyOnly := false
|
||||||
|
|
||||||
for turn := 0; ; turn++ {
|
for turn := 0; ; turn++ {
|
||||||
for _, interrupt := range a.drainInterrupts() {
|
for _, interrupt := range a.drainInterrupts() {
|
||||||
msgs = append(msgs, agentAPI.Message{
|
msgs = append(msgs, agentAPI.Message{
|
||||||
@ -75,10 +162,17 @@ func (a *Agent) process(input string, stageCtx *sdk.StageContext) (response stri
|
|||||||
// 工具轮产出的 tool/assistant 消息作结尾会被 400 拒绝,故补一条 user 占位。
|
// 工具轮产出的 tool/assistant 消息作结尾会被 400 拒绝,故补一条 user 占位。
|
||||||
// 注意:仅当尾部确为工具轮产物(assistant/tool)时才补位;首轮 system 上下文结尾不补,
|
// 注意:仅当尾部确为工具轮产物(assistant/tool)时才补位;首轮 system 上下文结尾不补,
|
||||||
// 否则会错误覆盖实际用户输入(如 injectSourceContext 追加的 system 说明)。
|
// 否则会错误覆盖实际用户输入(如 injectSourceContext 追加的 system 说明)。
|
||||||
|
//
|
||||||
|
// 补位前先移除前面轮次插入的同类占位,保证占位**不随轮次线性累积**——
|
||||||
|
// 占位是核心插的传输层附加物,不是用户发言,不该在 prompt 里叠成 N 条。
|
||||||
|
//
|
||||||
|
// 文案分情况:上一批全是 output_send__* 时不能说“继续”,详见
|
||||||
|
// replyDeliveredPlaceholder 的说明。
|
||||||
|
msgs = dropContinuationPlaceholders(msgs)
|
||||||
if last := msgs[len(msgs)-1]; last.Role == "assistant" || last.Role == "tool" {
|
if last := msgs[len(msgs)-1]; last.Role == "assistant" || last.Role == "tool" {
|
||||||
msgs = append(msgs, agentAPI.Message{
|
msgs = append(msgs, agentAPI.Message{
|
||||||
Role: "user",
|
Role: "user",
|
||||||
Content: "请根据以上工具结果继续。",
|
Content: continuationFor(lastBatchReplyOnly),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -243,6 +337,17 @@ func (a *Agent) process(input string, stageCtx *sdk.StageContext) (response stri
|
|||||||
return resp.Content, toolsUsed, toolResults, nil
|
return resp.Content, toolsUsed, toolResults, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 本批是否全部是输出通道发送(=模型刚交付了给用户的回复)。
|
||||||
|
// 必须在执行前判定:执行过程中的中断/拒绝分支会 continue/break,
|
||||||
|
// 放在循环里统计会漏。
|
||||||
|
replyOnly := true
|
||||||
|
for _, tc := range resp.ToolCalls {
|
||||||
|
if !isOutputDeliveryTool(tc.Name) {
|
||||||
|
replyOnly = false
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
contentOnce := true
|
contentOnce := true
|
||||||
for _, tc := range resp.ToolCalls {
|
for _, tc := range resp.ToolCalls {
|
||||||
if len(a.interceptCh) > 0 {
|
if len(a.interceptCh) > 0 {
|
||||||
@ -305,6 +410,18 @@ func (a *Agent) process(input string, stageCtx *sdk.StageContext) (response stri
|
|||||||
result = r
|
result = r
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// ContextPolicy: prune 工具调用后执行上下文裁剪(§13.8)
|
||||||
|
if def := a.stageHost.ToolDef(tc.Name); def != nil && def.ContextPolicy == "prune" {
|
||||||
|
if a.context != nil {
|
||||||
|
topK := a.maxContextSize - 1
|
||||||
|
if topK < 1 {
|
||||||
|
topK = 1
|
||||||
|
}
|
||||||
|
// 查询向量取**清洗后**的有效内容,否则噪声(ANSI/base64/JSON
|
||||||
|
// 包装)会把相关性打分带偏,裁掉本该保留的事件。
|
||||||
|
a.context.Prune(a.toolOutputForQuery(tc.Name, result), topK, a.docStore)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
msgContent := ""
|
msgContent := ""
|
||||||
if contentOnce {
|
if contentOnce {
|
||||||
@ -313,10 +430,21 @@ func (a *Agent) process(input string, stageCtx *sdk.StageContext) (response stri
|
|||||||
}
|
}
|
||||||
msgs = append(msgs, agentAPI.Message{Role: "assistant", Content: msgContent, ReasoningContent: resp.ReasoningContent, ToolCalls: []agentAPI.ToolCall{tc}})
|
msgs = append(msgs, agentAPI.Message{Role: "assistant", Content: msgContent, ReasoningContent: resp.ReasoningContent, ToolCalls: []agentAPI.ToolCall{tc}})
|
||||||
|
|
||||||
// 多模态工具结果:插件通过 SDK.SetToolBlocks 注入 image_url/audio_url block,
|
// 多模态工具结果:插件通过 SDK.SetToolBlocks 注入 image_url/audio_url block。
|
||||||
// process.go 拾起并追加到 tool message 的 content 数组(OpenAI 多模态格式),
|
//
|
||||||
// 让下一轮 LLM 请求在 tool message 里看到图/音频。
|
// 媒体不挂在 tool message 上,而是另起一条紧随其后的 user message——
|
||||||
|
// 这也是插件文案一直在说的「注入后续对话」。
|
||||||
|
// 为何不能挂 tool message:同一张图、同一模型、三轮实测——
|
||||||
|
// 图在 user message → 3/3 读到
|
||||||
|
// 图在 tool message → 0/3(模型答「没能读到这张图」)
|
||||||
|
// tool 纯文本 + 后接 user → 3/3 读到
|
||||||
|
// tool message 那轮 prompt_tokens 反而更高(7967 vs 7089),base64 确实
|
||||||
|
// 进了上游,但 role=tool 上的多模态 content 数组不被当作可视内容。
|
||||||
|
//
|
||||||
|
// 主模型不支持该模态时更不能直接塞:网关会把 image_url 静默剥离后仍
|
||||||
|
// 返回 200,模型回答「我没有看到图片」而内核以为注入成功。改走回退链。
|
||||||
toolMsg := agentAPI.Message{Role: "tool", ToolCallID: tc.ID, Content: result}
|
toolMsg := agentAPI.Message{Role: "tool", ToolCallID: tc.ID, Content: result}
|
||||||
|
var mediaMsg *agentAPI.Message
|
||||||
if rawBlocks := a.io.ConsumeToolBlocks(); len(rawBlocks) > 0 {
|
if rawBlocks := a.io.ConsumeToolBlocks(); len(rawBlocks) > 0 {
|
||||||
var blocks []agentAPI.ContentBlock
|
var blocks []agentAPI.ContentBlock
|
||||||
for _, b := range rawBlocks {
|
for _, b := range rawBlocks {
|
||||||
@ -333,10 +461,35 @@ func (a *Agent) process(input string, stageCtx *sdk.StageContext) (response stri
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
if len(blocks) > 0 {
|
if len(blocks) > 0 {
|
||||||
toolMsg.Blocks = blocks
|
// 先落进 CAS:无论下面走直视还是回退转写,媒体本体都该进记忆。
|
||||||
|
// 不存的后果是 ToolResultItem.Output 只剩那句
|
||||||
|
// "[已将图片注入后续对话] /tmp/x.png",文件一删线索就断了。
|
||||||
|
a.stageMediaDigests(a.captureBlockMedia(blocks, tc.Name)...)
|
||||||
|
|
||||||
|
if native, fallbackText := a.prepareToolBlocks(blocks); len(native) > 0 {
|
||||||
|
// 能直视:另起一条 user message 承载媒体,并补一句来源说明,
|
||||||
|
// 否则模型会把它当成用户新发的图而不是工具拉回来的。
|
||||||
|
mediaBlocks := append([]agentAPI.ContentBlock{{
|
||||||
|
Type: "text",
|
||||||
|
Text: fmt.Sprintf("[以下是 %s 注入的媒体内容]", tc.Name),
|
||||||
|
}}, native...)
|
||||||
|
mediaMsg = &agentAPI.Message{Role: "user", Blocks: mediaBlocks}
|
||||||
|
} else if fallbackText != "" {
|
||||||
|
// 回退链已把媒体转写成文字:并进 tool message 的纯文本 content,
|
||||||
|
// 不再另起消息(文字在 tool message 里本来就能被读到)。
|
||||||
|
toolMsg.Content = result + "\n\n" + fallbackText
|
||||||
|
result = toolMsg.Content
|
||||||
|
if len(toolResults) > 0 {
|
||||||
|
toolResults[len(toolResults)-1].Output = result
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
msgs = append(msgs, toolMsg)
|
msgs = append(msgs, toolMsg)
|
||||||
|
if mediaMsg != nil {
|
||||||
|
// 必须紧跟在 toolMsg 之后:中间插入其他消息会让 tool_call_id 配对断开。
|
||||||
|
msgs = append(msgs, *mediaMsg)
|
||||||
|
}
|
||||||
|
|
||||||
a.publishEvent(events.EventToolCall, map[string]interface{}{
|
a.publishEvent(events.EventToolCall, map[string]interface{}{
|
||||||
"tool": tc.Name,
|
"tool": tc.Name,
|
||||||
@ -354,6 +507,9 @@ func (a *Agent) process(input string, stageCtx *sdk.StageContext) (response stri
|
|||||||
break
|
break
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 供下一轮顶部选择补位文案。
|
||||||
|
lastBatchReplyOnly = replyOnly
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
119
internal/agent/core/prunepolicy_test.go
Normal file
119
internal/agent/core/prunepolicy_test.go
Normal file
@ -0,0 +1,119 @@
|
|||||||
|
package core
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
agentIO "gitcode.com/JianFeeeee/HomeAgent/internal/agent/io"
|
||||||
|
pubsdk "gitcode.com/JianFeeeee/homeagent-sdk/sdk"
|
||||||
|
)
|
||||||
|
|
||||||
|
// 这一组测试锁死「默认不裁剪」这条语义。
|
||||||
|
//
|
||||||
|
// 改动前:每条非中断输入都无条件 Prune 一次,没有任何声明能关掉它。
|
||||||
|
// 这是破坏性行为(低相关事件被归档并从上下文移走),却无法从调用点看出
|
||||||
|
// 「谁触发的裁剪」。改成需声明后,必须逐条验证默认值确实是不裁剪。
|
||||||
|
func TestPruneDeclared_DefaultsToNoPrune(t *testing.T) {
|
||||||
|
m := agentIO.NewIOManager()
|
||||||
|
a := &Agent{io: m}
|
||||||
|
|
||||||
|
evt := &agentIO.InputEvent{Source: "unknown_source", Payload: map[string]interface{}{}}
|
||||||
|
if a.pruneDeclared(evt) {
|
||||||
|
t.Fatal("既没有通道声明也没有注入声明的输入,默认必须不裁剪")
|
||||||
|
}
|
||||||
|
|
||||||
|
// 通道注册了、但策略是 none / 空:仍然不裁剪。
|
||||||
|
m.RegisterInputChannel("quiet", pubsdk.ChannelDef{ContextPolicy: pubsdk.ContextPolicyNone})
|
||||||
|
if a.pruneDeclared(&agentIO.InputEvent{Source: "quiet", Payload: map[string]interface{}{}}) {
|
||||||
|
t.Fatal("ChannelDef.ContextPolicy=none 不应裁剪")
|
||||||
|
}
|
||||||
|
m.RegisterInputChannel("empty", pubsdk.ChannelDef{})
|
||||||
|
if a.pruneDeclared(&agentIO.InputEvent{Source: "empty", Payload: map[string]interface{}{}}) {
|
||||||
|
t.Fatal("ChannelDef 未设 ContextPolicy 不应裁剪")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 通道显式声明 prune 才裁剪。
|
||||||
|
func TestPruneDeclared_ChannelOptIn(t *testing.T) {
|
||||||
|
m := agentIO.NewIOManager()
|
||||||
|
m.RegisterInputChannel("noisy", pubsdk.ChannelDef{ContextPolicy: pubsdk.ContextPolicyPrune})
|
||||||
|
a := &Agent{io: m}
|
||||||
|
|
||||||
|
if !a.pruneDeclared(&agentIO.InputEvent{Source: "noisy", Payload: map[string]interface{}{}}) {
|
||||||
|
t.Fatal("通道声明 prune 后应裁剪")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 注入点声明的优先级高于通道定义:同一通道下的不同注入可以有不同意图。
|
||||||
|
func TestPruneDeclared_InjectionOverridesChannel(t *testing.T) {
|
||||||
|
m := agentIO.NewIOManager()
|
||||||
|
a := &Agent{io: m}
|
||||||
|
m.RegisterInputChannel("chan", pubsdk.ChannelDef{ContextPolicy: pubsdk.ContextPolicyPrune})
|
||||||
|
|
||||||
|
// 注入点说 none → 即使通道说 prune 也不裁。
|
||||||
|
evt := &agentIO.InputEvent{Source: "chan", Payload: map[string]interface{}{
|
||||||
|
"context_policy": pubsdk.ContextPolicyNone,
|
||||||
|
}}
|
||||||
|
if a.pruneDeclared(evt) {
|
||||||
|
t.Fatal("注入点声明 none 应覆盖通道的 prune")
|
||||||
|
}
|
||||||
|
|
||||||
|
// 通道没说,注入点说 prune → 裁。
|
||||||
|
m.RegisterInputChannel("plain", pubsdk.ChannelDef{})
|
||||||
|
evt = &agentIO.InputEvent{Source: "plain", Payload: map[string]interface{}{
|
||||||
|
"context_policy": pubsdk.ContextPolicyPrune,
|
||||||
|
}}
|
||||||
|
if !a.pruneDeclared(evt) {
|
||||||
|
t.Fatal("注入点声明 prune 应生效")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 没有 context 时不能 panic,也不该裁剪。
|
||||||
|
func TestPruneOnInput_NilContextIsSafe(t *testing.T) {
|
||||||
|
m := agentIO.NewIOManager()
|
||||||
|
m.RegisterInputChannel("noisy", pubsdk.ChannelDef{ContextPolicy: pubsdk.ContextPolicyPrune})
|
||||||
|
a := &Agent{io: m}
|
||||||
|
if got := a.pruneOnInput(&agentIO.InputEvent{Source: "noisy", Payload: map[string]interface{}{}}, "x"); got != 0 {
|
||||||
|
t.Fatalf("nil context 应返回 0,实际 %d", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// cleanInputFor 的优先级:注入点声明的 cleaner > 按 source 查的 cleaner > 原文。
|
||||||
|
func TestCleanInputFor_Priority(t *testing.T) {
|
||||||
|
m := agentIO.NewIOManager()
|
||||||
|
m.RegisterInputChannel("src", pubsdk.ChannelDef{
|
||||||
|
Cleaner: func(s string) string { return "by-source:" + s },
|
||||||
|
})
|
||||||
|
m.RegisterInputChannel("explicit", pubsdk.ChannelDef{
|
||||||
|
Cleaner: func(s string) string { return "by-name:" + s },
|
||||||
|
})
|
||||||
|
a := &Agent{io: m}
|
||||||
|
|
||||||
|
// 无声明 → 用 source 的 cleaner
|
||||||
|
evt := &agentIO.InputEvent{Source: "src", Payload: map[string]interface{}{}}
|
||||||
|
if got := a.cleanInputFor(evt, "raw"); got != "by-source:raw" {
|
||||||
|
t.Fatalf("应回退到 source 的 cleaner,实际 %q", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 注入点指定 cleaner_name → 覆盖 source 的
|
||||||
|
evt = &agentIO.InputEvent{Source: "src", Payload: map[string]interface{}{"cleaner_name": "explicit"}}
|
||||||
|
if got := a.cleanInputFor(evt, "raw"); got != "by-name:raw" {
|
||||||
|
t.Fatalf("注入点声明的 cleaner 应优先,实际 %q", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 完全没有 cleaner → 原文
|
||||||
|
evt = &agentIO.InputEvent{Source: "nobody", Payload: map[string]interface{}{}}
|
||||||
|
if got := a.cleanInputFor(evt, "raw"); got != "raw" {
|
||||||
|
t.Fatalf("没有 cleaner 时应返回原文,实际 %q", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 声明的名字查不到 → 回退到 source 的 cleaner(并记日志),不能 panic、不能丢内容
|
||||||
|
evt = &agentIO.InputEvent{Source: "src", Payload: map[string]interface{}{"cleaner_name": "missing"}}
|
||||||
|
if got := a.cleanInputFor(evt, "raw"); got != "by-source:raw" {
|
||||||
|
t.Fatalf("未知 cleaner_name 应回退,实际 %q", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
// nil IOManager 不能 panic
|
||||||
|
if got := (&Agent{}).cleanInputFor(evt, "raw"); got != "raw" {
|
||||||
|
t.Fatalf("nil io 应返回原文,实际 %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
65
internal/agent/core/prunequery_test.go
Normal file
65
internal/agent/core/prunequery_test.go
Normal file
@ -0,0 +1,65 @@
|
|||||||
|
package core
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
sdk "gitcode.com/JianFeeeee/HomeAgent/internal/sdk"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ContextPolicy=prune 的查询向量必须取**清洗后**的有效内容。
|
||||||
|
//
|
||||||
|
// 裁剪的入参是相关性查询向量,它决定保留/归档哪些上下文事件。原始工具输出里
|
||||||
|
// 混着 ANSI 转义、base64、JSON 包装等噪声,直接向量化会让打分失真,裁掉本该
|
||||||
|
// 保留的事件。ToolDef.Cleaner 的契约本就写着「仅在向量化/jieba/蒸馏时调用」,
|
||||||
|
// 裁剪正是在向量化——此前只在构建事件向量时用了它,裁剪查询漏了。
|
||||||
|
func TestToolOutputForQueryAppliesCleaner(t *testing.T) {
|
||||||
|
host := NewStageHost()
|
||||||
|
called := 0
|
||||||
|
if err := host.RegisterTool("demo_tool", sdk.ToolDef{
|
||||||
|
Name: "demo_tool",
|
||||||
|
Cleaner: func(s string) string {
|
||||||
|
called++
|
||||||
|
return "cleaned:" + s
|
||||||
|
},
|
||||||
|
}, func(map[string]interface{}) (interface{}, error) { return nil, nil }); err != nil {
|
||||||
|
t.Fatalf("RegisterTool: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
a := &Agent{stageHost: host}
|
||||||
|
raw := "\x1b[31mresult\x1b[0m"
|
||||||
|
|
||||||
|
got := a.toolOutputForQuery("demo_tool", raw)
|
||||||
|
if called != 1 {
|
||||||
|
t.Fatalf("Cleaner 应被调用恰好一次,实际 %d", called)
|
||||||
|
}
|
||||||
|
if got != "cleaned:"+raw {
|
||||||
|
t.Fatalf("查询应使用清洗结果,实际 %q", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 未注册 Cleaner 的工具:回退原文。
|
||||||
|
if got := a.toolOutputForQuery("no_such_tool", raw); got != raw {
|
||||||
|
t.Fatalf("无 Cleaner 应回退原文,实际 %q", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 无 StageHost(如裸 Agent):不能 panic,回退原文。
|
||||||
|
if got := (&Agent{}).toolOutputForQuery("demo_tool", raw); got != raw {
|
||||||
|
t.Fatalf("nil stageHost 应回退原文,实际 %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Cleaner 返回空串时必须回退原文:空串会让查询向量退化成零向量,
|
||||||
|
// 所有事件相关性相同,裁剪就失去判据(等于随机裁)。
|
||||||
|
func TestToolOutputForQueryEmptyCleanFallsBack(t *testing.T) {
|
||||||
|
host := NewStageHost()
|
||||||
|
if err := host.RegisterTool("t", sdk.ToolDef{
|
||||||
|
Name: "t",
|
||||||
|
Cleaner: func(string) string { return "" },
|
||||||
|
}, func(map[string]interface{}) (interface{}, error) { return nil, nil }); err != nil {
|
||||||
|
t.Fatalf("RegisterTool: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
a := &Agent{stageHost: host}
|
||||||
|
if got := a.toolOutputForQuery("t", "raw"); got != "raw" {
|
||||||
|
t.Fatalf("Cleaner 返回空应回退原文,实际 %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -9,6 +9,42 @@ import (
|
|||||||
agentAPI "gitcode.com/JianFeeeee/HomeAgent/internal/agent/api"
|
agentAPI "gitcode.com/JianFeeeee/HomeAgent/internal/agent/api"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// childTaskState 是一个子任务的生命周期状态。
|
||||||
|
//
|
||||||
|
// delivered 代替了早期的“读到即删”:完成通知会写进持久上下文
|
||||||
|
// (formatMergedTimeline 每轮重新注入),模型之后还会再查。读一次就删的
|
||||||
|
// 话,第二次查询返回“不存在或已过期”——那是一个**永远不会成功的可操作
|
||||||
|
// 信号**,模型只能一遍遍地重试/汇报,循环永不结束。
|
||||||
|
type childTaskState struct {
|
||||||
|
running bool
|
||||||
|
result string
|
||||||
|
delivered bool // 结果是否已交付过(用于幂等应答)
|
||||||
|
seq int64 // 完成顺序,用于有界淘汰
|
||||||
|
}
|
||||||
|
|
||||||
|
// maxRetainedChildTasks 是保留的已完成子任务上限(防结果无限占用内存)。
|
||||||
|
const maxRetainedChildTasks = 20
|
||||||
|
|
||||||
|
// evictChildTasksLocked 淘汰最旧的已完成子任务。调用方必须持有 childMu。
|
||||||
|
func (a *Agent) evictChildTasksLocked() {
|
||||||
|
for len(a.childTasks) > maxRetainedChildTasks {
|
||||||
|
oldestID := ""
|
||||||
|
var oldestSeq int64
|
||||||
|
for id, st := range a.childTasks {
|
||||||
|
if st.running {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if oldestID == "" || st.seq < oldestSeq {
|
||||||
|
oldestID, oldestSeq = id, st.seq
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if oldestID == "" {
|
||||||
|
return // 剩下全是运行中的,不淘汰
|
||||||
|
}
|
||||||
|
delete(a.childTasks, oldestID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func (a *Agent) executeSpawnChild(tc agentAPI.ToolCall) string {
|
func (a *Agent) executeSpawnChild(tc agentAPI.ToolCall) string {
|
||||||
task, _ := tc.Arguments["task"].(string)
|
task, _ := tc.Arguments["task"].(string)
|
||||||
if task == "" {
|
if task == "" {
|
||||||
@ -41,11 +77,11 @@ func (a *Agent) executeSpawnChild(tc agentAPI.ToolCall) string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
a.childMu.Lock()
|
a.childMu.Lock()
|
||||||
a.childRunning[taskID] = true
|
a.childTasks[taskID] = &childTaskState{running: true}
|
||||||
a.childMu.Unlock()
|
a.childMu.Unlock()
|
||||||
go a.runChildTask(taskID, task, parentChannel, maxTurns)
|
go a.runChildTask(taskID, task, parentChannel, maxTurns)
|
||||||
|
|
||||||
return fmt.Sprintf("子任务已启动(ID: %s,最多 %d 轮),完成后会自动通知你,届时请使用 child_result 工具查看输出", taskID, maxTurns)
|
return fmt.Sprintf("子任务已启动(ID: %s,最多 %d 轮)。完成后会自动通知你,届时用 child_result 查看输出即可(**只需查询一次**)", taskID, maxTurns)
|
||||||
}
|
}
|
||||||
|
|
||||||
// defaultChildMaxTurns 子 Agent 默认工具轮数(可被 spawn_child 的 max_turns 参数覆盖)。
|
// defaultChildMaxTurns 子 Agent 默认工具轮数(可被 spawn_child 的 max_turns 参数覆盖)。
|
||||||
@ -126,19 +162,30 @@ func (a *Agent) runChildTask(taskID, task string, parentChannel string, maxTurns
|
|||||||
}
|
}
|
||||||
|
|
||||||
a.childMu.Lock()
|
a.childMu.Lock()
|
||||||
a.childResults[taskID] = finalResult
|
if st := a.childTasks[taskID]; st != nil {
|
||||||
delete(a.childRunning, taskID)
|
st.running = false
|
||||||
|
st.result = finalResult
|
||||||
|
a.childSeq++
|
||||||
|
st.seq = a.childSeq
|
||||||
|
}
|
||||||
|
a.evictChildTasksLocked()
|
||||||
a.childMu.Unlock()
|
a.childMu.Unlock()
|
||||||
|
|
||||||
log.Printf("[child] %s done: %s", taskID, truncateStr(finalResult, 100))
|
log.Printf("[child] %s done: %s", taskID, truncateStr(finalResult, 100))
|
||||||
|
|
||||||
notification := fmt.Sprintf("子任务 %s 已完成,请调用 child_result 工具查看输出", taskID)
|
notification := fmt.Sprintf("子任务 %s 已完成。请用 child_result 工具查看输出(只需查询一次;重复查询不会返回失败)。", taskID)
|
||||||
a.injectSelfChannel(selfInputMsg{
|
a.injectSelfChannel(selfInputMsg{
|
||||||
text: notification,
|
text: notification,
|
||||||
channel: parentChannel, // 回到父对话通道,正常处理(写入上下文 + emit 响应)
|
channel: parentChannel, // 回到父对话通道,正常处理(写入上下文 + emit 响应)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// executeChildResultTool 取回子任务结果。
|
||||||
|
//
|
||||||
|
// **幂等**:结果不会被“读到即删”,重复查询返回同一结果或一条明确提示。
|
||||||
|
// 这一点至关重要——完成通知会长期留在持久上下文里(formatMergedTimeline
|
||||||
|
// 每轮重新注入),如果重复查询返回“不存在”这种失败信号,模型会认定任务
|
||||||
|
// 未完成而无限重试(实测单轮 35 次工具调用、持续 514 秒)。
|
||||||
func (a *Agent) executeChildResultTool(tc agentAPI.ToolCall) string {
|
func (a *Agent) executeChildResultTool(tc agentAPI.ToolCall) string {
|
||||||
taskID, _ := tc.Arguments["task_id"].(string)
|
taskID, _ := tc.Arguments["task_id"].(string)
|
||||||
if taskID == "" {
|
if taskID == "" {
|
||||||
@ -146,18 +193,25 @@ func (a *Agent) executeChildResultTool(tc agentAPI.ToolCall) string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
a.childMu.Lock()
|
a.childMu.Lock()
|
||||||
result, ok := a.childResults[taskID]
|
st, ok := a.childTasks[taskID]
|
||||||
if ok {
|
if !ok {
|
||||||
delete(a.childResults, taskID)
|
|
||||||
a.childMu.Unlock()
|
a.childMu.Unlock()
|
||||||
return fmt.Sprintf("【子任务 %s 结果】\n%s", taskID, result)
|
return fmt.Sprintf("子任务 %s 不存在:从未创建该 ID(请核对 spawn_child 返回的 ID 拼写)", taskID)
|
||||||
}
|
}
|
||||||
if a.childRunning[taskID] {
|
if st.running {
|
||||||
a.childMu.Unlock()
|
a.childMu.Unlock()
|
||||||
return fmt.Sprintf("子任务 %s 仍在运行中,尚未完成。请等待完成通知后再查询。", taskID)
|
return fmt.Sprintf("子任务 %s 仍在运行中,尚未完成。请等待完成通知后再查询。", taskID)
|
||||||
}
|
}
|
||||||
|
first := !st.delivered
|
||||||
|
st.delivered = true
|
||||||
|
result := st.result
|
||||||
a.childMu.Unlock()
|
a.childMu.Unlock()
|
||||||
return fmt.Sprintf("子任务 %s 不存在或已过期", taskID)
|
|
||||||
|
if first {
|
||||||
|
return fmt.Sprintf("【子任务 %s 结果】\n%s", taskID, result)
|
||||||
|
}
|
||||||
|
// 重复查询不是失败:明确告诉模型“任务已完成、结果已给过”,让它停止重试。
|
||||||
|
return fmt.Sprintf("【子任务 %s 已完成】结果已在上文提供(见先前的 child_result 工具结果),无需重复查询;请直接基于上文结果继续。", taskID)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (a *Agent) executeLLMTool(tc agentAPI.ToolCall) string {
|
func (a *Agent) executeLLMTool(tc agentAPI.ToolCall) string {
|
||||||
|
|||||||
88
internal/agent/core/spawn_test.go
Normal file
88
internal/agent/core/spawn_test.go
Normal file
@ -0,0 +1,88 @@
|
|||||||
|
package core
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
agentAPI "gitcode.com/JianFeeeee/HomeAgent/internal/agent/api"
|
||||||
|
)
|
||||||
|
|
||||||
|
// child_result 必须幂等——这是 "任务已结束但核心循环不结束" 的根因修复。
|
||||||
|
//
|
||||||
|
// 子任务完成通知会写进持久上下文(formatMergedTimeline 每轮重新注入),
|
||||||
|
// 模型之后还会再查。若第二次查询返回 "不存在或已过期" 这种**永久失败信号**,
|
||||||
|
// 模型会认定任务未完成而无限重试/汇报(生产实测:单轮 35 次工具调用、
|
||||||
|
// 持续 514 秒)。
|
||||||
|
func TestChildResultIsIdempotent(t *testing.T) {
|
||||||
|
a := New(AgentConfig{ID: "t"})
|
||||||
|
|
||||||
|
a.childMu.Lock()
|
||||||
|
a.childTasks["child_1"] = &childTaskState{result: "任务完成:已创建 3 个日程", seq: 1}
|
||||||
|
a.childMu.Unlock()
|
||||||
|
|
||||||
|
call := func(id string) string {
|
||||||
|
return a.executeChildResultTool(agentAPI.ToolCall{
|
||||||
|
Name: "child_result",
|
||||||
|
Arguments: map[string]interface{}{"task_id": id},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
first := call("child_1")
|
||||||
|
if !strings.Contains(first, "任务完成:已创建 3 个日程") {
|
||||||
|
t.Fatalf("首次查询应返回结果,实际: %q", first)
|
||||||
|
}
|
||||||
|
|
||||||
|
second := call("child_1")
|
||||||
|
if strings.Contains(second, "不存在") {
|
||||||
|
t.Fatalf("重复查询不能返回失败信号(会驱动模型无限重试),实际: %q", second)
|
||||||
|
}
|
||||||
|
if !strings.Contains(second, "已完成") {
|
||||||
|
t.Fatalf("重复查询应明确告知「已完成、结果已提供」,实际: %q", second)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 只有从未创建过的 ID 才应报 "不存在"。
|
||||||
|
missing := call("child_999")
|
||||||
|
if !strings.Contains(missing, "不存在") {
|
||||||
|
t.Fatalf("未知 ID 应报不存在,实际: %q", missing)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 运行中与已完成必须给出不同答复,否则模型无法判断该等还是该继续。
|
||||||
|
func TestChildResultRunningVsDone(t *testing.T) {
|
||||||
|
a := New(AgentConfig{ID: "t"})
|
||||||
|
|
||||||
|
a.childMu.Lock()
|
||||||
|
a.childTasks["child_run"] = &childTaskState{running: true}
|
||||||
|
a.childMu.Unlock()
|
||||||
|
|
||||||
|
got := a.executeChildResultTool(agentAPI.ToolCall{
|
||||||
|
Name: "child_result",
|
||||||
|
Arguments: map[string]interface{}{"task_id": "child_run"},
|
||||||
|
})
|
||||||
|
if !strings.Contains(got, "仍在运行中") {
|
||||||
|
t.Fatalf("运行中的任务应提示仍在运行,实际: %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 保留的结果必须有界,不能随子任务数量无限增长。
|
||||||
|
func TestChildTaskRetentionBounded(t *testing.T) {
|
||||||
|
a := New(AgentConfig{ID: "t"})
|
||||||
|
|
||||||
|
a.childMu.Lock()
|
||||||
|
for i := 0; i < maxRetainedChildTasks*3; i++ {
|
||||||
|
a.childSeq++
|
||||||
|
a.childTasks[fmt.Sprintf("child_%d", i)] = &childTaskState{result: "r", seq: a.childSeq}
|
||||||
|
}
|
||||||
|
a.evictChildTasksLocked()
|
||||||
|
n := len(a.childTasks)
|
||||||
|
a.childMu.Unlock()
|
||||||
|
|
||||||
|
if n > maxRetainedChildTasks {
|
||||||
|
t.Fatalf("保留子任务数=%d,超过上限 %d", n, maxRetainedChildTasks)
|
||||||
|
}
|
||||||
|
// 淘汰应保留最新的:最早的那批必须已不在
|
||||||
|
if _, ok := a.childTasks["child_0"]; ok {
|
||||||
|
t.Fatal("淘汰应优先丢弃最旧的已完成任务")
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -73,6 +73,8 @@ func collectKernelStatus(
|
|||||||
BuildTime: meta.BuildTime,
|
BuildTime: meta.BuildTime,
|
||||||
SDKCompatible: meta.SDKCompatibleVersion,
|
SDKCompatible: meta.SDKCompatibleVersion,
|
||||||
KernelName: meta.KernelName,
|
KernelName: meta.KernelName,
|
||||||
|
// AGPL-3.0 §13:状态页向网络使用者展示取得源码的入口。
|
||||||
|
SourceURL: meta.SourceURL,
|
||||||
},
|
},
|
||||||
Runtime: RuntimeStatus{
|
Runtime: RuntimeStatus{
|
||||||
Goroutines: runtime.NumGoroutine(),
|
Goroutines: runtime.NumGoroutine(),
|
||||||
|
|||||||
@ -150,6 +150,14 @@ func (a *Agent) executeMemoryTool(tc agentAPI.ToolCall) string {
|
|||||||
}
|
}
|
||||||
parts = append(parts, fmt.Sprintf("- %s →(%s)→ %s", r.SourceName, r.RelationType, r.TargetName))
|
parts = append(parts, fmt.Sprintf("- %s →(%s)→ %s", r.SourceName, r.RelationType, r.TargetName))
|
||||||
}
|
}
|
||||||
|
// 命中的关系若挂着媒体块,把媒体说明附在结果末尾。
|
||||||
|
//
|
||||||
|
// 关系行只有实体名和关系类型,看不出"这条记忆当时还带了一张图"。
|
||||||
|
// 媒体块以结构边与句子相连,需经关系→句子反查。
|
||||||
|
// 不附上的后果:agent 显式查了图记忆,却仍然不知道有图。
|
||||||
|
if mc := a.mediaContextForRelations(result.Relations); mc != "" {
|
||||||
|
parts = append(parts, "", "关联媒体:", mc)
|
||||||
|
}
|
||||||
return strings.Join(parts, "\n")
|
return strings.Join(parts, "\n")
|
||||||
|
|
||||||
case "memory_block_merge":
|
case "memory_block_merge":
|
||||||
@ -177,9 +185,21 @@ func (a *Agent) executeMemoryTool(tc agentAPI.ToolCall) string {
|
|||||||
for _, td := range triplesData {
|
for _, td := range triplesData {
|
||||||
if m, ok := td.(map[string]interface{}); ok {
|
if m, ok := td.(map[string]interface{}); ok {
|
||||||
t := memory.Triple{
|
t := memory.Triple{
|
||||||
Subject: getString(m, "subject"),
|
Subject: getString(m, "subject"),
|
||||||
Relation: getString(m, "relation"),
|
Relation: getString(m, "relation"),
|
||||||
Object: getString(m, "object"),
|
Object: getString(m, "object"),
|
||||||
|
SentenceText: getString(m, "sentence_text"),
|
||||||
|
}
|
||||||
|
// 模型显式关联的媒体:结构化字段随三元组一起提交,
|
||||||
|
// 由 commitTriplesWithMedia 变成 L3 一等块并与句子建边——
|
||||||
|
// 不再把 marker 写进句子文本。
|
||||||
|
if digests := getStringSlice(m, "media_digests"); len(digests) > 0 {
|
||||||
|
t.MediaDigests = a.resolveMediaDigests(digests)
|
||||||
|
// 块边需要句子作端点。模型没给原句时用三元组本身拼一句
|
||||||
|
// 自然语言——不能造一段 marker 文本,那正是被废弃的东西。
|
||||||
|
if t.SentenceText == "" && len(t.MediaDigests) > 0 {
|
||||||
|
t.SentenceText = fmt.Sprintf("%s%s%s。", t.Subject, t.Relation, t.Object)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if t.Subject != "" && t.Relation != "" && t.Object != "" {
|
if t.Subject != "" && t.Relation != "" && t.Object != "" {
|
||||||
triples = append(triples, t)
|
triples = append(triples, t)
|
||||||
@ -189,10 +209,15 @@ func (a *Agent) executeMemoryTool(tc agentAPI.ToolCall) string {
|
|||||||
if len(triples) == 0 {
|
if len(triples) == 0 {
|
||||||
return "没有有效的三元组"
|
return "没有有效的三元组"
|
||||||
}
|
}
|
||||||
ec, rc, err := a.memory.Commit(triples, string(a.id), 0)
|
// remember 工具是用户/模型显式写入,不涉及归档删除,
|
||||||
|
// 因此不需要 mediaBound——没有旧引用要释放。
|
||||||
|
ec, rc, mb, err := a.commitTriplesWithMedia(triples, string(a.id), 0, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Sprintf("记忆写入失败: %v", err)
|
return fmt.Sprintf("记忆写入失败: %v", err)
|
||||||
}
|
}
|
||||||
|
if mb > 0 {
|
||||||
|
return fmt.Sprintf("已写入 %d 个实体和 %d 条关系,关联 %d 份媒体", ec, rc, mb)
|
||||||
|
}
|
||||||
return fmt.Sprintf("已写入 %d 个实体和 %d 条关系", ec, rc)
|
return fmt.Sprintf("已写入 %d 个实体和 %d 条关系", ec, rc)
|
||||||
|
|
||||||
case "memory_introspect":
|
case "memory_introspect":
|
||||||
@ -511,6 +536,11 @@ func (a *Agent) executeDocTool(tc agentAPI.ToolCall) string {
|
|||||||
if len(content) > 2000 {
|
if len(content) > 2000 {
|
||||||
content = content[:2000] + "..."
|
content = content[:2000] + "..."
|
||||||
}
|
}
|
||||||
|
// 媒体块标签单独一行进冷存事件:正文可能被上面的 2000 字截断,
|
||||||
|
// 截掉之后模型就不知道这篇文档带过图。
|
||||||
|
if labels := a.blockLabelsForDoc(d); labels != "" {
|
||||||
|
content = content + "\n关联媒体: " + labels
|
||||||
|
}
|
||||||
a.context.InsertByTimestamp(ContextEvent{
|
a.context.InsertByTimestamp(ContextEvent{
|
||||||
Timestamp: d.CreatedAt,
|
Timestamp: d.CreatedAt,
|
||||||
Source: "cold_storage",
|
Source: "cold_storage",
|
||||||
@ -541,14 +571,27 @@ func (a *Agent) executeDocTool(tc agentAPI.ToolCall) string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
doc := &document.Doc{
|
doc := &document.Doc{
|
||||||
Summary: summary,
|
Summary: summary,
|
||||||
Content: content,
|
Content: content,
|
||||||
Tags: tags,
|
Tags: tags,
|
||||||
Source: "manual",
|
Source: "manual",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 模型显式关联的媒体:直接变成文档持有的一等块。
|
||||||
|
// 不再往正文写 marker——文档向量会融合这些块的媒体向量,
|
||||||
|
// 图片按自己的向量被检索。
|
||||||
|
for _, d := range a.resolveMediaDigests(getStringSlice(tc.Arguments, "media_digests")) {
|
||||||
|
if b, ok := a.blockFromDigest(d); ok {
|
||||||
|
doc.Blocks = append(doc.Blocks, b)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if err := a.docStore.Insert(doc); err != nil {
|
if err := a.docStore.Insert(doc); err != nil {
|
||||||
return fmt.Sprintf("文档写入失败: %v", err)
|
return fmt.Sprintf("文档写入失败: %v", err)
|
||||||
}
|
}
|
||||||
|
if n := len(doc.Blocks); n > 0 {
|
||||||
|
return fmt.Sprintf("文档已提交 (id: %s, 摘要: %s, 关联 %d 份媒体)", doc.ID, summary, n)
|
||||||
|
}
|
||||||
return fmt.Sprintf("文档已提交 (id: %s, 摘要: %s)", doc.ID, summary)
|
return fmt.Sprintf("文档已提交 (id: %s, 摘要: %s)", doc.ID, summary)
|
||||||
|
|
||||||
default:
|
default:
|
||||||
|
|||||||
@ -13,6 +13,24 @@ func (a *Agent) buildMemoryContext(input string, maxTokens int) string {
|
|||||||
}
|
}
|
||||||
injected := a.indexer.BuildContext(input)
|
injected := a.indexer.BuildContext(input)
|
||||||
s := a.indexer.FormatContext(injected)
|
s := a.indexer.FormatContext(injected)
|
||||||
|
|
||||||
|
// 图库召回命中的实体若关联着带媒体的句子,把媒体说明一并注入。
|
||||||
|
//
|
||||||
|
// 不做这一步的后果:媒体描述进了 L3,agent 却拿不出来。图库句子里
|
||||||
|
// 写着 [image/png a1b2c3d4e5f6] 这样的短标记,但没有任何东西告诉
|
||||||
|
// 模型那份内容是否还在、能否重新查看——描述永存而 blob 可能已被
|
||||||
|
// 删除,两者状态不同,必须显式告知。
|
||||||
|
//
|
||||||
|
// 注意不能直接用 injected.Relations:BuildContext 刻意把它置为 nil
|
||||||
|
//(自动注入只给实体索引以省 token,细节留给 memory_recall)。
|
||||||
|
// 因此这里用命中的实体名再查一次关系,只为拿到 sentence_id。
|
||||||
|
if mc := a.mediaContextForInjectedEntities(injected); mc != "" {
|
||||||
|
if s != "" {
|
||||||
|
s += "\n"
|
||||||
|
}
|
||||||
|
s += "【关联媒体】\n" + mc
|
||||||
|
}
|
||||||
|
|
||||||
if maxTokens > 0 {
|
if maxTokens > 0 {
|
||||||
s = TruncateByTokens(s, maxTokens)
|
s = TruncateByTokens(s, maxTokens)
|
||||||
}
|
}
|
||||||
@ -37,15 +55,13 @@ func (a *Agent) buildSystemPrompt(memContext string, userInput string) string {
|
|||||||
|
|
||||||
prompt += "\n\n【记忆清理指令】当用户要求整理或清理记忆时,你必须实际调用 memory_ 工具执行操作,不能只回复文本。先用 memory_introspect 查看概况,再用 memory_recall 获取详情。有同义实体则用 memory_merge 合并(source 会被彻底删除),有无用噪音实体则用 memory_delete_entity 直接删除,也可用 memory_purge 批量清理,用 memory_edit 修正错误,用 memory_block_merge 标记不合并。如果工具执行成功,把结果告知用户;不要只描述计划而不执行。"
|
prompt += "\n\n【记忆清理指令】当用户要求整理或清理记忆时,你必须实际调用 memory_ 工具执行操作,不能只回复文本。先用 memory_introspect 查看概况,再用 memory_recall 获取详情。有同义实体则用 memory_merge 合并(source 会被彻底删除),有无用噪音实体则用 memory_delete_entity 直接删除,也可用 memory_purge 批量清理,用 memory_edit 修正错误,用 memory_block_merge 标记不合并。如果工具执行成功,把结果告知用户;不要只描述计划而不执行。"
|
||||||
|
|
||||||
|
// 跨模态召回:文本路(fastText/TF-IDF 文档层,媒体描述文本已随记忆进入)
|
||||||
|
// + 视觉路(多模态文本编码 → 媒体库坐标)两路归一化融合。
|
||||||
|
// 未配置多模态空间时视觉路为空,等价旧的 docStore.Query。
|
||||||
if a.docStore != nil {
|
if a.docStore != nil {
|
||||||
docs := a.docStore.Query(userInput, 3)
|
hits := a.retrieveCrossModal(userInput, 3, a.fusionCfg)
|
||||||
if len(docs) > 0 {
|
if md := a.crossModalMarkdown(hits); md != "" {
|
||||||
var parts []string
|
prompt += "\n\n" + md
|
||||||
parts = append(parts, "【相关记忆文档】")
|
|
||||||
for i, d := range docs {
|
|
||||||
parts = append(parts, fmt.Sprintf(" [%d] %s", i+1, d.Summary))
|
|
||||||
}
|
|
||||||
prompt += "\n\n" + strings.Join(parts, "\n")
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -57,7 +73,7 @@ func (a *Agent) buildSystemPrompt(memContext string, userInput string) string {
|
|||||||
prompt += "- 同步通道(webui / cli / 终端):直接返回纯文本,内核会把文本交给等待方显示,无需调用工具。\n"
|
prompt += "- 同步通道(webui / cli / 终端):直接返回纯文本,内核会把文本交给等待方显示,无需调用工具。\n"
|
||||||
prompt += "- 异步通道(qq / wechat / 群聊等):返回纯文本**【不会】**自动送达用户,必须调用 output_send__{通道名} 工具(注意 meta 里带上正确的 user_id 或 group_id)才能真正把消息发出去。\n"
|
prompt += "- 异步通道(qq / wechat / 群聊等):返回纯文本**【不会】**自动送达用户,必须调用 output_send__{通道名} 工具(注意 meta 里带上正确的 user_id 或 group_id)才能真正把消息发出去。\n"
|
||||||
prompt += "- 不确定当前通道的发送方式时,先用 output_send__{通道名}_help 查看该通道的 meta 格式和 type 枚举,再决定。\n"
|
prompt += "- 不确定当前通道的发送方式时,先用 output_send__{通道名}_help 查看该通道的 meta 格式和 type 枚举,再决定。\n"
|
||||||
prompt += "- 同一轮对话中可多次调用输出门工具。长消息应当分多次发出,而不是一口气发完。\n"
|
prompt += "- 每轮对话**通常只需调用一次** output_send__{通道名} 即可完成回复。仅在内容确实超过单条消息长度上限(如 >4000 字)时才拆分为多条;拆分时每条应是完整段落,不要碎片化。\n"
|
||||||
prompt += "- 需要多步执行的长任务:**必须先**向当前对话通道发一条确认消息告诉用户已收到(异步通道用输出门工具,同步通道直接返回文本),**然后再**执行具体排查工具。确认消息不代表任务完成,发出后仍需继续执行实际工具并最终汇报结果。\n"
|
prompt += "- 需要多步执行的长任务:**必须先**向当前对话通道发一条确认消息告诉用户已收到(异步通道用输出门工具,同步通道直接返回文本),**然后再**执行具体排查工具。确认消息不代表任务完成,发出后仍需继续执行实际工具并最终汇报结果。\n"
|
||||||
prompt += "- 用户从其他渠道发来「在哪里/怎么样了」这类追问时,先回忆上次任务的通道与上下文,再回同一通道。"
|
prompt += "- 用户从其他渠道发来「在哪里/怎么样了」这类追问时,先回忆上次任务的通道与上下文,再回同一通道。"
|
||||||
|
|
||||||
@ -108,8 +124,8 @@ func (a *Agent) buildToolCatalog() string {
|
|||||||
}
|
}
|
||||||
// 仅注入插件/通道能力摘要,避免全量工具定义污染 system prompt。
|
// 仅注入插件/通道能力摘要,避免全量工具定义污染 system prompt。
|
||||||
// 每个插件列:名称 + 能力描述 + 工具数。完整工具定义由 get_plugin_tools 按需拉取。
|
// 每个插件列:名称 + 能力描述 + 工具数。完整工具定义由 get_plugin_tools 按需拉取。
|
||||||
byPlugin := map[string]int{} // plugin -> 工具数
|
byPlugin := map[string]int{} // plugin -> 工具数
|
||||||
pluginDesc := map[string]string{} // plugin -> 首个工具描述(作能力概览)
|
pluginDesc := map[string]string{} // plugin -> 首个工具描述(作能力概览)
|
||||||
var order []string
|
var order []string
|
||||||
for _, t := range defs {
|
for _, t := range defs {
|
||||||
fn, ok := t.(map[string]interface{})["function"].(map[string]interface{})
|
fn, ok := t.(map[string]interface{})["function"].(map[string]interface{})
|
||||||
@ -302,7 +318,7 @@ func (a *Agent) buildToolDefs() []interface{} {
|
|||||||
"parameters": map[string]interface{}{
|
"parameters": map[string]interface{}{
|
||||||
"type": "object",
|
"type": "object",
|
||||||
"properties": map[string]interface{}{
|
"properties": map[string]interface{}{
|
||||||
"name": map[string]interface{}{"type": "string", "description": "知识名称(用作目录名)"},
|
"name": map[string]interface{}{"type": "string", "description": "知识名称(用作目录名)"},
|
||||||
"content": map[string]interface{}{"type": "string", "description": "知识内容,支持 Markdown"},
|
"content": map[string]interface{}{"type": "string", "description": "知识内容,支持 Markdown"},
|
||||||
},
|
},
|
||||||
"required": []string{"name", "content"},
|
"required": []string{"name", "content"},
|
||||||
@ -356,6 +372,11 @@ func (a *Agent) buildToolDefs() []interface{} {
|
|||||||
"description": "标签列表",
|
"description": "标签列表",
|
||||||
"items": map[string]interface{}{"type": "string"},
|
"items": map[string]interface{}{"type": "string"},
|
||||||
},
|
},
|
||||||
|
"media_digests": map[string]interface{}{
|
||||||
|
"type": "array",
|
||||||
|
"description": "可选:这篇文档关联的媒体 digest(对话或 memory_recall 的「关联媒体」里显示的十六进制串,短的即可)。填了以后检索到这篇文档就能看到并取回原图/音频。",
|
||||||
|
"items": map[string]interface{}{"type": "string"},
|
||||||
|
},
|
||||||
},
|
},
|
||||||
"required": []string{"content"},
|
"required": []string{"content"},
|
||||||
},
|
},
|
||||||
|
|||||||
117
internal/agent/core/tooloop_test.go
Normal file
117
internal/agent/core/tooloop_test.go
Normal file
@ -0,0 +1,117 @@
|
|||||||
|
package core
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
agentAPI "gitcode.com/JianFeeeee/HomeAgent/internal/agent/api"
|
||||||
|
)
|
||||||
|
|
||||||
|
// appendPlaceholder 复刻 process() 循环顶部的补位逻辑。
|
||||||
|
func appendPlaceholder(msgs []agentAPI.Message, replyOnly bool) []agentAPI.Message {
|
||||||
|
msgs = dropContinuationPlaceholders(msgs)
|
||||||
|
if last := msgs[len(msgs)-1]; last.Role == "assistant" || last.Role == "tool" {
|
||||||
|
msgs = append(msgs, agentAPI.Message{Role: "user", Content: continuationFor(replyOnly)})
|
||||||
|
}
|
||||||
|
return msgs
|
||||||
|
}
|
||||||
|
|
||||||
|
func countPlaceholders(msgs []agentAPI.Message) int {
|
||||||
|
n := 0
|
||||||
|
for _, m := range msgs {
|
||||||
|
if isContinuationPlaceholder(m) {
|
||||||
|
n++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return n
|
||||||
|
}
|
||||||
|
|
||||||
|
// 占位是核心插入的传输层附加物,不是用户发言——它不能随轮次线性累积。
|
||||||
|
//
|
||||||
|
// 旧实现每轮无条件追加而从不移除,跑 N 轮 prompt 里就叠了 N 条一模一样的
|
||||||
|
// “继续”,把前缀上下文(含记忆注入)往后挤。
|
||||||
|
func TestPlaceholderDoesNotAccumulate(t *testing.T) {
|
||||||
|
msgs := []agentAPI.Message{{Role: "user", Content: "用户请求"}}
|
||||||
|
|
||||||
|
const rounds = 20
|
||||||
|
for turn := 0; turn < rounds; turn++ {
|
||||||
|
msgs = append(msgs, agentAPI.Message{Role: "assistant", Content: "调用工具"})
|
||||||
|
msgs = append(msgs, agentAPI.Message{Role: "tool", Content: "结果"})
|
||||||
|
|
||||||
|
// 交替普通工具轮 / 纯发送轮,确保两种文案都参与去重。
|
||||||
|
msgs = appendPlaceholder(msgs, turn%2 == 1)
|
||||||
|
|
||||||
|
if n := countPlaceholders(msgs); n != 1 {
|
||||||
|
t.Fatalf("第 %d 轮后占位数=%d,期望恰好 1 条(旧实现会累积到 %d 条)", turn+1, n, turn+1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 末尾那一轮是纯发送轮,留下的应是“允许收尾”的文案。
|
||||||
|
if last := msgs[len(msgs)-1]; last.Content != replyDeliveredPlaceholder {
|
||||||
|
t.Fatalf("最后应是回复已交付的文案,实际: %q", last.Content)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 首轮 system/真实用户输入结尾不补位:补了会覆盖实际用户输入。
|
||||||
|
func TestPlaceholderNotAppendedOnFirstTurn(t *testing.T) {
|
||||||
|
msgs := []agentAPI.Message{
|
||||||
|
{Role: "system", Content: "系统说明"},
|
||||||
|
{Role: "user", Content: "真实用户输入"},
|
||||||
|
}
|
||||||
|
got := appendPlaceholder(msgs, false)
|
||||||
|
if len(got) != 2 {
|
||||||
|
t.Fatalf("首轮不应补位,得到 %d 条: %+v", len(got), got)
|
||||||
|
}
|
||||||
|
if got[1].Content != "真实用户输入" {
|
||||||
|
t.Fatalf("真实用户输入被覆盖: %q", got[1].Content)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 内容相近的真实用户消息不能被当作占位删掉。
|
||||||
|
func TestDropOnlyExactPlaceholder(t *testing.T) {
|
||||||
|
msgs := []agentAPI.Message{
|
||||||
|
{Role: "user", Content: continuationPlaceholder + "补充"},
|
||||||
|
{Role: "user", Content: replyDeliveredPlaceholder + "补充"},
|
||||||
|
{Role: "user", Content: continuationPlaceholder},
|
||||||
|
}
|
||||||
|
got := dropContinuationPlaceholders(msgs)
|
||||||
|
if len(got) != 2 {
|
||||||
|
t.Fatalf("只应删掉精确匹配的那条,得到 %d 条: %+v", len(got), got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 纯输出通道调用之后的补位不能再是「请继续」。
|
||||||
|
//
|
||||||
|
// 异步通道的回复只能经 output_send__* 交付,所以模型「已完成回复」的形式就是
|
||||||
|
// 一个工具调用;紧跟一句「请继续」会被读成「还要再做一步」,而能做的
|
||||||
|
// 「一步」恰好还是再发一条消息。(生产实测:单轮 34 次发送、514 秒)
|
||||||
|
func TestContinuationForReplyDoesNotPushToContinue(t *testing.T) {
|
||||||
|
reply := continuationFor(true)
|
||||||
|
if reply == continuationPlaceholder {
|
||||||
|
t.Fatal("回复已交付后不应再补「请继续」,会驱动重复发送")
|
||||||
|
}
|
||||||
|
if !strings.Contains(reply, "纯文本") || !strings.Contains(reply, "结束") {
|
||||||
|
t.Fatalf("应明确告知可返回纯文本收尾,实际: %q", reply)
|
||||||
|
}
|
||||||
|
|
||||||
|
if got := continuationFor(false); got != continuationPlaceholder {
|
||||||
|
t.Fatalf("普通工具轮补位应保持不变,实际: %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 只有真正的发送动作算「交付回复」;_help 是查询用法。
|
||||||
|
func TestIsOutputDeliveryTool(t *testing.T) {
|
||||||
|
cases := map[string]bool{
|
||||||
|
"output_send__qq": true,
|
||||||
|
"output_send__webui": true,
|
||||||
|
"output_send__qq_help": false,
|
||||||
|
"output_list_channels": false,
|
||||||
|
"cmd_run": false,
|
||||||
|
"qq_get_message": false,
|
||||||
|
}
|
||||||
|
for name, want := range cases {
|
||||||
|
if got := isOutputDeliveryTool(name); got != want {
|
||||||
|
t.Errorf("isOutputDeliveryTool(%q) = %v, want %v", name, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -29,6 +29,24 @@ func getFloat(m map[string]interface{}, key string) float64 {
|
|||||||
return 0
|
return 0
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// getStringSlice 从工具参数里取字符串数组。
|
||||||
|
//
|
||||||
|
// 需要单独一个 helper 而不是直接断言 []string:LLM 的参数经 JSON 解码后是
|
||||||
|
// []interface{},直接断言 []string 恒失败——静默拿到 nil,参数像没传一样。
|
||||||
|
func getStringSlice(m map[string]interface{}, key string) []string {
|
||||||
|
raw, ok := m[key].([]interface{})
|
||||||
|
if !ok {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
var out []string
|
||||||
|
for _, v := range raw {
|
||||||
|
if s, ok := v.(string); ok && s != "" {
|
||||||
|
out = append(out, s)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
func truncateStr(s string, max int) string {
|
func truncateStr(s string, max int) string {
|
||||||
if utf8.RuneCountInString(s) <= max {
|
if utf8.RuneCountInString(s) <= max {
|
||||||
return s
|
return s
|
||||||
|
|||||||
@ -25,11 +25,11 @@ const (
|
|||||||
type OutputCapability int
|
type OutputCapability int
|
||||||
|
|
||||||
const (
|
const (
|
||||||
CapText OutputCapability = 1 << iota // 文本
|
CapText OutputCapability = 1 << iota // 文本
|
||||||
CapFile // 文件
|
CapFile // 文件
|
||||||
CapImage // 图片
|
CapImage // 图片
|
||||||
CapAudio // 音频
|
CapAudio // 音频
|
||||||
CapStructured // 结构化数据(JSON/卡片)
|
CapStructured // 结构化数据(JSON/卡片)
|
||||||
)
|
)
|
||||||
|
|
||||||
func (c OutputCapability) Supports(cap OutputCapability) bool {
|
func (c OutputCapability) Supports(cap OutputCapability) bool {
|
||||||
@ -242,6 +242,52 @@ func (m *IOManager) InjectInputSyncTo(source, outputChannel, eventType string, p
|
|||||||
return <-ch
|
return <-ch
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// InjectOptions 声明一次注入在记忆层与上下文层的表现。
|
||||||
|
//
|
||||||
|
// 零值 = 记入记忆 + 不裁剪上下文,与历史的三参数注入方法完全一致。
|
||||||
|
// 别名到公共 SDK 而非另建一套:内置插件与外部插件必须用同一套结构,
|
||||||
|
// 否则内核要认两种类型,而漏认会静默丢失标志位。
|
||||||
|
type InjectOptions = pubsdk.InjectOptions
|
||||||
|
|
||||||
|
// applyInjectOpts 把注入标志位写进事件 payload。
|
||||||
|
//
|
||||||
|
// 只在非零时写:零值与旧 payload 逐字节一致,事件订阅方与旧内核
|
||||||
|
// (不认识这两个键)都不会受影响。
|
||||||
|
//
|
||||||
|
// 为什么不把标志位当独立参数传到底:eventloop 与各注入路径都按 payload 取字段
|
||||||
|
// (no_memory 本来就是这么走的),payload 是这里唯一已有的携带面。
|
||||||
|
func applyInjectOpts(payload map[string]interface{}, opts InjectOptions) {
|
||||||
|
if opts.NoMemory {
|
||||||
|
payload["no_memory"] = true
|
||||||
|
}
|
||||||
|
if opts.ContextPolicy != "" {
|
||||||
|
payload["context_policy"] = opts.ContextPolicy
|
||||||
|
}
|
||||||
|
if opts.CleanerName != "" {
|
||||||
|
payload["cleaner_name"] = opts.CleanerName
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *IOManager) InjectInputOpts(source, eventType string, payload map[string]interface{}, opts InjectOptions) {
|
||||||
|
applyInjectOpts(payload, opts)
|
||||||
|
m.InjectInput(source, eventType, payload)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *IOManager) InjectInputToOpts(source, outputChannel, eventType string, payload map[string]interface{}, opts InjectOptions) {
|
||||||
|
applyInjectOpts(payload, opts)
|
||||||
|
m.InjectInputTo(source, outputChannel, eventType, payload)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *IOManager) InjectInputSyncToOpts(source, outputChannel, eventType string, payload map[string]interface{}, opts InjectOptions) *OutputEvent {
|
||||||
|
applyInjectOpts(payload, opts)
|
||||||
|
return m.InjectInputSyncTo(source, outputChannel, eventType, payload)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *IOManager) InjectInterruptOpts(source, channel string, payload map[string]interface{}, opts InjectOptions) {
|
||||||
|
applyInjectOpts(payload, opts)
|
||||||
|
m.InjectInterrupt(source, channel, payload)
|
||||||
|
}
|
||||||
|
|
||||||
func (m *IOManager) InjectText(source string, text string) {
|
func (m *IOManager) InjectText(source string, text string) {
|
||||||
m.InjectInput(source, "text", map[string]interface{}{
|
m.InjectInput(source, "text", map[string]interface{}{
|
||||||
"content": text,
|
"content": text,
|
||||||
@ -293,10 +339,31 @@ func (m *IOManager) InjectInterrupt(source, channel string, payload map[string]i
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (m *IOManager) InjectInterruptText(source, channel, text string) {
|
func (m *IOManager) InjectInterruptText(source, channel, text string) {
|
||||||
m.InjectInterrupt(source, channel, map[string]interface{}{
|
m.InjectInterruptTextOpts(source, channel, text, InjectOptions{})
|
||||||
|
}
|
||||||
|
|
||||||
|
// InjectInterruptTextOpts 注入中断文本,并声明本次注入的记忆/裁剪行为。
|
||||||
|
//
|
||||||
|
// 中断也允许声明 ContextPolicyPrune:中断同样携带内容进入上下文。
|
||||||
|
func (m *IOManager) InjectInterruptTextOpts(source, channel, text string, opts InjectOptions) {
|
||||||
|
m.InjectInterruptOpts(source, channel, map[string]interface{}{
|
||||||
"type": "text",
|
"type": "text",
|
||||||
"content": text,
|
"content": text,
|
||||||
})
|
}, opts)
|
||||||
|
}
|
||||||
|
|
||||||
|
// InjectTextOpts 注入排队文本,并声明本次注入的记忆/裁剪行为。
|
||||||
|
func (m *IOManager) InjectTextOpts(source, channel, text string, opts InjectOptions) {
|
||||||
|
m.InjectInputToOpts(source, channel, "text", map[string]interface{}{
|
||||||
|
"content": text,
|
||||||
|
}, opts)
|
||||||
|
}
|
||||||
|
|
||||||
|
// InjectTextSyncOpts 同步注入文本并声明记忆/裁剪行为。
|
||||||
|
func (m *IOManager) InjectTextSyncOpts(source, outputChannel, text string, opts InjectOptions) *OutputEvent {
|
||||||
|
return m.InjectInputSyncToOpts(source, outputChannel, "text", map[string]interface{}{
|
||||||
|
"content": text,
|
||||||
|
}, opts)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *IOManager) InputInterruptChan() <-chan *InputEvent { return m.interruptCh }
|
func (m *IOManager) InputInterruptChan() <-chan *InputEvent { return m.interruptCh }
|
||||||
@ -308,6 +375,33 @@ func (m *IOManager) InjectTextSyncTo(source, outputChannel, text string) *Output
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ---- 带标志位的注入(记忆/裁剪行为由调用点声明)----
|
||||||
|
|
||||||
|
// InjectInputMediaOpts 注入带媒体块的输入,并声明记忆/裁剪行为。
|
||||||
|
func (m *IOManager) InjectInputMediaOpts(source, outputChannel, text string, blocks []pubsdk.ContentBlock, opts InjectOptions) {
|
||||||
|
m.InjectInputToOpts(source, outputChannel, "text", map[string]interface{}{
|
||||||
|
"content": text,
|
||||||
|
"media_blocks": blocks,
|
||||||
|
}, opts)
|
||||||
|
}
|
||||||
|
|
||||||
|
// InjectInputMediaSyncOpts 注入带媒体块的输入并同步等待回复,同时声明记忆/裁剪行为。
|
||||||
|
func (m *IOManager) InjectInputMediaSyncOpts(source, outputChannel, text string, blocks []pubsdk.ContentBlock, opts InjectOptions) *OutputEvent {
|
||||||
|
return m.InjectInputSyncToOpts(source, outputChannel, "text", map[string]interface{}{
|
||||||
|
"content": text,
|
||||||
|
"media_blocks": blocks,
|
||||||
|
}, opts)
|
||||||
|
}
|
||||||
|
|
||||||
|
// InjectInterruptMediaOpts 注入带媒体块的中断,并声明记忆/裁剪行为。
|
||||||
|
func (m *IOManager) InjectInterruptMediaOpts(source, channel, text string, blocks []pubsdk.ContentBlock, opts InjectOptions) {
|
||||||
|
m.InjectInterruptOpts(source, channel, map[string]interface{}{
|
||||||
|
"type": "text",
|
||||||
|
"content": text,
|
||||||
|
"media_blocks": blocks,
|
||||||
|
}, opts)
|
||||||
|
}
|
||||||
|
|
||||||
func (m *IOManager) EmitOutput(target string, outputType string, payload map[string]interface{}) {
|
func (m *IOManager) EmitOutput(target string, outputType string, payload map[string]interface{}) {
|
||||||
m.outputCh <- &OutputEvent{
|
m.outputCh <- &OutputEvent{
|
||||||
RequestID: "",
|
RequestID: "",
|
||||||
@ -343,7 +437,7 @@ func (m *IOManager) EmitTextTo(target, outputChannel, text string) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *IOManager) InputChan() <-chan *InputEvent { return m.inputCh }
|
func (m *IOManager) InputChan() <-chan *InputEvent { return m.inputCh }
|
||||||
func (m *IOManager) OutputChan() <-chan *OutputEvent { return m.outputCh }
|
func (m *IOManager) OutputChan() <-chan *OutputEvent { return m.outputCh }
|
||||||
|
|
||||||
// RegisterInputChannel 注册输入通道的记忆行为
|
// RegisterInputChannel 注册输入通道的记忆行为
|
||||||
@ -463,12 +557,14 @@ func NewMicrophone(name string, sampleRate int, io *IOManager) *Microphone {
|
|||||||
return &Microphone{name: name, sampleRate: sampleRate, io: io}
|
return &Microphone{name: name, sampleRate: sampleRate, io: io}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (d *Microphone) Name() string { return d.name }
|
func (d *Microphone) Name() string { return d.name }
|
||||||
func (d *Microphone) Type() DeviceType { return DeviceInput }
|
func (d *Microphone) Type() DeviceType { return DeviceInput }
|
||||||
func (d *Microphone) OutputCapabilities() OutputCapability { return 0 } // 纯输入
|
func (d *Microphone) OutputCapabilities() OutputCapability { return 0 } // 纯输入
|
||||||
func (d *Microphone) Description() string { return fmt.Sprintf("麦克风 (%s, %dHz)", d.name, d.sampleRate) }
|
func (d *Microphone) Description() string {
|
||||||
func (d *Microphone) Start() error { return nil }
|
return fmt.Sprintf("麦克风 (%s, %dHz)", d.name, d.sampleRate)
|
||||||
func (d *Microphone) Stop() error { return nil }
|
}
|
||||||
|
func (d *Microphone) Start() error { return nil }
|
||||||
|
func (d *Microphone) Stop() error { return nil }
|
||||||
func (d *Microphone) ChannelDef() ChannelDef { return ChannelDef{} }
|
func (d *Microphone) ChannelDef() ChannelDef { return ChannelDef{} }
|
||||||
|
|
||||||
func (d *Microphone) Tools() []ToolDef {
|
func (d *Microphone) Tools() []ToolDef {
|
||||||
@ -498,13 +594,13 @@ func NewSpeaker(name string, io *IOManager) *Speaker {
|
|||||||
return &Speaker{name: name, io: io}
|
return &Speaker{name: name, io: io}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (d *Speaker) Name() string { return d.name }
|
func (d *Speaker) Name() string { return d.name }
|
||||||
func (d *Speaker) Type() DeviceType { return DeviceOutput }
|
func (d *Speaker) Type() DeviceType { return DeviceOutput }
|
||||||
func (d *Speaker) OutputCapabilities() OutputCapability { return CapText | CapAudio }
|
func (d *Speaker) OutputCapabilities() OutputCapability { return CapText | CapAudio }
|
||||||
func (d *Speaker) Description() string { return fmt.Sprintf("扬声器 (%s)", d.name) }
|
func (d *Speaker) Description() string { return fmt.Sprintf("扬声器 (%s)", d.name) }
|
||||||
func (d *Speaker) Start() error { return nil }
|
func (d *Speaker) Start() error { return nil }
|
||||||
func (d *Speaker) Stop() error { return nil }
|
func (d *Speaker) Stop() error { return nil }
|
||||||
func (d *Speaker) ChannelDef() ChannelDef { return ChannelDef{} }
|
func (d *Speaker) ChannelDef() ChannelDef { return ChannelDef{} }
|
||||||
|
|
||||||
func (d *Speaker) Tools() []ToolDef {
|
func (d *Speaker) Tools() []ToolDef {
|
||||||
return []ToolDef{{
|
return []ToolDef{{
|
||||||
@ -535,13 +631,13 @@ func NewCamera(name string, io *IOManager) *Camera {
|
|||||||
return &Camera{name: name, io: io}
|
return &Camera{name: name, io: io}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (d *Camera) Name() string { return d.name }
|
func (d *Camera) Name() string { return d.name }
|
||||||
func (d *Camera) Type() DeviceType { return DeviceInput }
|
func (d *Camera) Type() DeviceType { return DeviceInput }
|
||||||
func (d *Camera) OutputCapabilities() OutputCapability { return CapImage } // 可返回图片
|
func (d *Camera) OutputCapabilities() OutputCapability { return CapImage } // 可返回图片
|
||||||
func (d *Camera) Description() string { return fmt.Sprintf("摄像头 (%s)", d.name) }
|
func (d *Camera) Description() string { return fmt.Sprintf("摄像头 (%s)", d.name) }
|
||||||
func (d *Camera) Start() error { return nil }
|
func (d *Camera) Start() error { return nil }
|
||||||
func (d *Camera) Stop() error { return nil }
|
func (d *Camera) Stop() error { return nil }
|
||||||
func (d *Camera) ChannelDef() ChannelDef { return ChannelDef{} }
|
func (d *Camera) ChannelDef() ChannelDef { return ChannelDef{} }
|
||||||
|
|
||||||
func (d *Camera) Tools() []ToolDef {
|
func (d *Camera) Tools() []ToolDef {
|
||||||
return []ToolDef{
|
return []ToolDef{
|
||||||
@ -583,13 +679,13 @@ func NewRobotArm(name string, io *IOManager) *RobotArm {
|
|||||||
return &RobotArm{name: name, io: io}
|
return &RobotArm{name: name, io: io}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (d *RobotArm) Name() string { return d.name }
|
func (d *RobotArm) Name() string { return d.name }
|
||||||
func (d *RobotArm) Type() DeviceType { return DeviceIO }
|
func (d *RobotArm) Type() DeviceType { return DeviceIO }
|
||||||
func (d *RobotArm) OutputCapabilities() OutputCapability { return CapStructured }
|
func (d *RobotArm) OutputCapabilities() OutputCapability { return CapStructured }
|
||||||
func (d *RobotArm) Description() string { return fmt.Sprintf("机械臂 (%s)", d.name) }
|
func (d *RobotArm) Description() string { return fmt.Sprintf("机械臂 (%s)", d.name) }
|
||||||
func (d *RobotArm) Start() error { return nil }
|
func (d *RobotArm) Start() error { return nil }
|
||||||
func (d *RobotArm) Stop() error { return nil }
|
func (d *RobotArm) Stop() error { return nil }
|
||||||
func (d *RobotArm) ChannelDef() ChannelDef { return ChannelDef{} }
|
func (d *RobotArm) ChannelDef() ChannelDef { return ChannelDef{} }
|
||||||
|
|
||||||
func (d *RobotArm) Tools() []ToolDef {
|
func (d *RobotArm) Tools() []ToolDef {
|
||||||
return []ToolDef{
|
return []ToolDef{
|
||||||
@ -635,13 +731,13 @@ func NewGPIODevice(name string, pins []int, io *IOManager) *GPIODevice {
|
|||||||
return &GPIODevice{name: name, pins: pins, io: io}
|
return &GPIODevice{name: name, pins: pins, io: io}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (d *GPIODevice) Name() string { return d.name }
|
func (d *GPIODevice) Name() string { return d.name }
|
||||||
func (d *GPIODevice) Type() DeviceType { return DeviceIO }
|
func (d *GPIODevice) Type() DeviceType { return DeviceIO }
|
||||||
func (d *GPIODevice) OutputCapabilities() OutputCapability { return CapStructured }
|
func (d *GPIODevice) OutputCapabilities() OutputCapability { return CapStructured }
|
||||||
func (d *GPIODevice) Description() string { return "GPIO 通用引脚" }
|
func (d *GPIODevice) Description() string { return "GPIO 通用引脚" }
|
||||||
func (d *GPIODevice) Start() error { return nil }
|
func (d *GPIODevice) Start() error { return nil }
|
||||||
func (d *GPIODevice) Stop() error { return nil }
|
func (d *GPIODevice) Stop() error { return nil }
|
||||||
func (d *GPIODevice) ChannelDef() ChannelDef { return ChannelDef{} }
|
func (d *GPIODevice) ChannelDef() ChannelDef { return ChannelDef{} }
|
||||||
|
|
||||||
func (d *GPIODevice) Tools() []ToolDef {
|
func (d *GPIODevice) Tools() []ToolDef {
|
||||||
return []ToolDef{
|
return []ToolDef{
|
||||||
|
|||||||
111
internal/agent/io/injectopts_test.go
Normal file
111
internal/agent/io/injectopts_test.go
Normal file
@ -0,0 +1,111 @@
|
|||||||
|
package io
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
pubsdk "gitcode.com/JianFeeeee/homeagent-sdk/sdk"
|
||||||
|
)
|
||||||
|
|
||||||
|
// drainOne 取出一条注入事件;没有则 Fatal。
|
||||||
|
func drainOne(t *testing.T, ch <-chan *InputEvent) *InputEvent {
|
||||||
|
t.Helper()
|
||||||
|
select {
|
||||||
|
case evt := <-ch:
|
||||||
|
return evt
|
||||||
|
default:
|
||||||
|
t.Fatal("没有拿到注入事件")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 零值 InjectOptions 必须与历史的三参数注入产出**完全一致**的 payload。
|
||||||
|
//
|
||||||
|
// 这是兼容性底线:任何按 payload 取字段的下游(事件订阅方、旧内核、
|
||||||
|
// 工具链测试)都不能因为这次改造而看到新键。
|
||||||
|
func TestInjectTextOpts_ZeroValueMatchesLegacyPayload(t *testing.T) {
|
||||||
|
m := NewIOManager()
|
||||||
|
m.InjectText("src", "hello")
|
||||||
|
legacy := drainOne(t, m.InputChan())
|
||||||
|
|
||||||
|
m2 := NewIOManager()
|
||||||
|
m2.InjectTextOpts("src", "chan", "hello", InjectOptions{})
|
||||||
|
withOpts := drainOne(t, m2.InputChan())
|
||||||
|
|
||||||
|
if len(withOpts.Payload) != len(legacy.Payload) {
|
||||||
|
t.Fatalf("零值注入多出了键:legacy=%v opts=%v", legacy.Payload, withOpts.Payload)
|
||||||
|
}
|
||||||
|
for k, v := range legacy.Payload {
|
||||||
|
if withOpts.Payload[k] != v {
|
||||||
|
t.Fatalf("键 %q 不一致:legacy=%v opts=%v", k, v, withOpts.Payload[k])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 标志位必须出现在事件 payload 上——eventloop 就是从那里读的。
|
||||||
|
func TestInjectTextOpts_CarriesFlags(t *testing.T) {
|
||||||
|
m := NewIOManager()
|
||||||
|
m.InjectTextOpts("src", "chan", "hello", InjectOptions{
|
||||||
|
NoMemory: true,
|
||||||
|
ContextPolicy: "prune",
|
||||||
|
CleanerName: "clean_me",
|
||||||
|
})
|
||||||
|
evt := drainOne(t, m.InputChan())
|
||||||
|
|
||||||
|
if evt.Payload["no_memory"] != true {
|
||||||
|
t.Errorf("no_memory 未传递: %v", evt.Payload["no_memory"])
|
||||||
|
}
|
||||||
|
if evt.Payload["context_policy"] != "prune" {
|
||||||
|
t.Errorf("context_policy 未传递: %v", evt.Payload["context_policy"])
|
||||||
|
}
|
||||||
|
if evt.Payload["cleaner_name"] != "clean_me" {
|
||||||
|
t.Errorf("cleaner_name 未传递: %v", evt.Payload["cleaner_name"])
|
||||||
|
}
|
||||||
|
if evt.Payload["content"] != "hello" {
|
||||||
|
t.Errorf("content 丢失: %v", evt.Payload["content"])
|
||||||
|
}
|
||||||
|
if evt.OutputChannel != "chan" {
|
||||||
|
t.Errorf("输出通道 = %q,期望 chan", evt.OutputChannel)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 中断注入走另一条队列,标志位同样要带上(用户已确认中断允许声明 prune)。
|
||||||
|
func TestInjectInterruptTextOpts_CarriesFlags(t *testing.T) {
|
||||||
|
m := NewIOManager()
|
||||||
|
m.InjectInterruptTextOpts("src", "chan", "alert", InjectOptions{ContextPolicy: "prune"})
|
||||||
|
evt := drainOne(t, m.InputInterruptChan())
|
||||||
|
|
||||||
|
if evt.Payload["context_policy"] != "prune" {
|
||||||
|
t.Errorf("中断注入的 context_policy 未传递: %v", evt.Payload)
|
||||||
|
}
|
||||||
|
if evt.Payload["type"] != "text" || evt.Payload["content"] != "alert" {
|
||||||
|
t.Errorf("中断注入的基本字段不对: %v", evt.Payload)
|
||||||
|
}
|
||||||
|
if _, has := evt.Payload["no_memory"]; has {
|
||||||
|
t.Errorf("未声明的 no_memory 不应出现: %v", evt.Payload)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 带媒体的注入同样要带标志位。
|
||||||
|
func TestInjectInputMediaOpts_CarriesFlags(t *testing.T) {
|
||||||
|
m := NewIOManager()
|
||||||
|
blocks := []pubsdk.ContentBlock{{Type: "image_url", ImageURL: &pubsdk.ImageURL{URL: "data:image/png;base64,AA"}}}
|
||||||
|
m.InjectInputMediaOpts("src", "chan", "看图", blocks, InjectOptions{NoMemory: true})
|
||||||
|
evt := drainOne(t, m.InputChan())
|
||||||
|
|
||||||
|
if evt.Payload["no_memory"] != true {
|
||||||
|
t.Errorf("媒体的 no_memory 未传递: %v", evt.Payload)
|
||||||
|
}
|
||||||
|
if _, ok := evt.Payload["media_blocks"]; !ok {
|
||||||
|
t.Errorf("媒体块丢失: %v", evt.Payload)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 旧方法必须继续等价工作(它们是 Opts 变体的零值糖)。
|
||||||
|
func TestLegacyNoMemoryMethodStillSetsFlag(t *testing.T) {
|
||||||
|
m := NewIOManager()
|
||||||
|
m.InjectTextNoMemoryTo("src", "chan", "quiet")
|
||||||
|
evt := drainOne(t, m.InputChan())
|
||||||
|
if evt.Payload["no_memory"] != true {
|
||||||
|
t.Fatalf("旧 NoMemory 方法应置位: %v", evt.Payload)
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -188,6 +188,8 @@ var sourceFieldDefs = []struct {
|
|||||||
{"adapter_path", "string", "适配器路径"},
|
{"adapter_path", "string", "适配器路径"},
|
||||||
{"max_concurrent", "int", "并发上限"},
|
{"max_concurrent", "int", "并发上限"},
|
||||||
{"priority", "int", "AUTO 优先级(大者优先)"},
|
{"priority", "int", "AUTO 优先级(大者优先)"},
|
||||||
|
{"vision", "bool", "支持图片"},
|
||||||
|
{"audio", "bool", "支持音频"},
|
||||||
}
|
}
|
||||||
|
|
||||||
// registerSourceDefs 注册 core.llm.sources.<name>.* 的 ConfigDef
|
// registerSourceDefs 注册 core.llm.sources.<name>.* 的 ConfigDef
|
||||||
@ -480,9 +482,30 @@ func (r *ConfigRegistry) SeedDefaults(dataDir string) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (r *ConfigRegistry) seedDBValues(dataDir string) {
|
func (r *ConfigRegistry) seedDBValues(dataDir string) {
|
||||||
var count int
|
// 新鲜度判据不能是「config 表非空」。
|
||||||
r.db.QueryRow(`SELECT COUNT(*) FROM config`).Scan(&count)
|
//
|
||||||
if count > 0 {
|
// 发行包的 postinst 会先跑 setup.sh → initconfig,而 initconfig 会写一行
|
||||||
|
// webui.listen_addr。于是**全新安装**的 DB 看上去"已经有内容",整个默认值
|
||||||
|
// 播种被跳过:core.plugin.dir、core.memory.*、多模态 provider 一个都没写。
|
||||||
|
// 现场表现是装完 0 个插件、随包的模型与运行库成死重量。
|
||||||
|
//
|
||||||
|
// 也不能改成"每次都补缺键":老安装升级时被注进新默认值,会让它突然
|
||||||
|
// 去加载一个 1.8GB 的模型——那是刻意要避免的行为(静默变重)。
|
||||||
|
//
|
||||||
|
// 故用显式标记区分三种情形:
|
||||||
|
// 有标记 → 已经播过种,直接返回
|
||||||
|
// 无标记但有 core.daemon.data_dir → 老安装(本键历来由播种写入),
|
||||||
|
// 只补标记、不播种
|
||||||
|
// 两者都没有 → 全新安装,播种并打标记
|
||||||
|
const markerKey = "core.internal.seed_version"
|
||||||
|
var hasMarker, hasLegacy int
|
||||||
|
r.db.QueryRow(`SELECT COUNT(*) FROM config WHERE key = ?`, markerKey).Scan(&hasMarker)
|
||||||
|
if hasMarker > 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
r.db.QueryRow(`SELECT COUNT(*) FROM config WHERE key = 'core.daemon.data_dir'`).Scan(&hasLegacy)
|
||||||
|
if hasLegacy > 0 {
|
||||||
|
r.db.Exec(`INSERT OR IGNORE INTO config (key, value) VALUES (?, ?)`, markerKey, "1")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -542,6 +565,14 @@ func (r *ConfigRegistry) seedDBValues(dataDir string) {
|
|||||||
set("core.memory.graph", filepath.Join(dataDir, "memory", "graph.db"))
|
set("core.memory.graph", filepath.Join(dataDir, "memory", "graph.db"))
|
||||||
set("core.memory.text", filepath.Join(dataDir, "memory", "text"))
|
set("core.memory.text", filepath.Join(dataDir, "memory", "text"))
|
||||||
set("core.memory.documents", filepath.Join(dataDir, "memory", "documents"))
|
set("core.memory.documents", filepath.Join(dataDir, "memory", "documents"))
|
||||||
|
set("core.memory.media.dir", filepath.Join(dataDir, "memory", "media"))
|
||||||
|
// 发行版默认启用本地向量空间。用 chineseclip(text+image、512 维、实测
|
||||||
|
// 常驻 1.15GB、Apache-2.0)而不是 qwen3vl(9.4GB):多数机器装不下后者。
|
||||||
|
// 产物不在仓库里,用 scripts/export_chineseclip_onnx.py 生成到这个路径;
|
||||||
|
// 产物缺失时 homed 会打印明确错误并退回 fastText 文本路径(不静默假装启用)。
|
||||||
|
set("core.memory.multimodal_space.provider", "chineseclip")
|
||||||
|
set("core.memory.multimodal_space.options.model_dir",
|
||||||
|
filepath.Join(dataDir, "models", "chinese-clip-vit-b16-onnx"))
|
||||||
set("core.knowledge.path", filepath.Join(dataDir, "knowledge"))
|
set("core.knowledge.path", filepath.Join(dataDir, "knowledge"))
|
||||||
set("core.log.path", filepath.Join(dataDir, "log"))
|
set("core.log.path", filepath.Join(dataDir, "log"))
|
||||||
|
|
||||||
@ -594,6 +625,8 @@ WebUI 概览页展示你的立绘,可通过 /mascot.webp 直接访问。如输
|
|||||||
set("core.input_processing.audio.fallback_model", "")
|
set("core.input_processing.audio.fallback_model", "")
|
||||||
set("core.input_processing.audio.describe_prompt", "请转写这段音频的内容。")
|
set("core.input_processing.audio.describe_prompt", "请转写这段音频的内容。")
|
||||||
|
|
||||||
|
set(markerKey, "1")
|
||||||
|
|
||||||
tx.Commit()
|
tx.Commit()
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -646,6 +679,16 @@ func (r *ConfigRegistry) seedCoreDefs(dataDir string) {
|
|||||||
reg(ConfigDef{Key: "core.memory.graph", Default: filepath.Join(dataDir, "memory", "graph.db"), Type: "string", DisplayName: "图数据库路径", Description: "长期记忆(图数据库)存储路径", Category: "paths"})
|
reg(ConfigDef{Key: "core.memory.graph", Default: filepath.Join(dataDir, "memory", "graph.db"), Type: "string", DisplayName: "图数据库路径", Description: "长期记忆(图数据库)存储路径", Category: "paths"})
|
||||||
reg(ConfigDef{Key: "core.memory.text", Default: filepath.Join(dataDir, "memory", "text"), Type: "string", DisplayName: "文本记忆路径", Description: "短期文本记忆存储目录", Category: "paths"})
|
reg(ConfigDef{Key: "core.memory.text", Default: filepath.Join(dataDir, "memory", "text"), Type: "string", DisplayName: "文本记忆路径", Description: "短期文本记忆存储目录", Category: "paths"})
|
||||||
reg(ConfigDef{Key: "core.memory.documents", Default: filepath.Join(dataDir, "memory", "documents"), Type: "string", DisplayName: "文档记忆路径", Description: "文档记忆存储目录", Category: "paths"})
|
reg(ConfigDef{Key: "core.memory.documents", Default: filepath.Join(dataDir, "memory", "documents"), Type: "string", DisplayName: "文档记忆路径", Description: "文档记忆存储目录", Category: "paths"})
|
||||||
|
reg(ConfigDef{Key: "core.memory.media.enabled", Default: "true", Type: "bool", DisplayName: "媒体记忆", Description: "把对话里出现的图片/音频变成一等记忆块,内容按 sha256 落盘去重。关闭后媒体仅在当前对话内可见,下一轮起只剩路径或 alt 文本", Category: "memory"})
|
||||||
|
reg(ConfigDef{Key: "core.memory.media.dir", Default: filepath.Join(dataDir, "memory", "media"), Type: "string", DisplayName: "媒体存储路径", Description: "媒体内容寻址存储目录(内含 media.db 与 blobs/)", Category: "paths"})
|
||||||
|
reg(ConfigDef{Key: "core.memory.multimodal_space.provider", Default: "chineseclip", Type: "string", DisplayName: "多模态向量 provider", Description: "从公共 provider 注册表(pkg/embedding)按名字打开的多模态向量空间。内置:chineseclip(默认,text+image,512 维,实测常驻 1.15GB,Apache-2.0)、qwen3vl(text+image,2048 维,常驻 9.4GB;视频已实现但未纳入契约)、http(外部向量 API)。也可是第三方注册的名字。两者均需 onnxruntime 构建标签。留空禁用多模态向量检索,只保留 fastText 文本路径。provider 的模型文件、预处理与运行时全在 provider 内部,核心不做任何模型假设。修改后需重启生效。", Category: "memory"})
|
||||||
|
reg(ConfigDef{Key: "core.memory.multimodal_space.options.model_dir", Default: filepath.Join(dataDir, "models", "chinese-clip-vit-b16-onnx"), Type: "string", DisplayName: "provider 模型目录", Description: "provider 自定义选项(以 options. 开头的键会去掉前缀后原样传给 provider,核心不解释其含义)。对内置 chineseclip:Chinese-CLIP 产物目录(用 scripts/export_chineseclip_onnx.py 生成)。对内置 qwen3vl:Qwen3-VL ONNX 产物目录(用 scripts/export_qwen3vl_embedding_onnx.py 生成)。", Category: "memory"})
|
||||||
|
reg(ConfigDef{Key: "core.memory.multimodal_space.options.endpoint", Default: "", Type: "string", DisplayName: "provider 服务端点", Description: "provider 自定义选项。对内置 http:外部多模态向量服务的端点 URL(POST,接受 modality/side/text/data/mime,返回 embedding)。", Category: "memory"})
|
||||||
|
reg(ConfigDef{Key: "core.memory.multimodal_space.options.api_key", Default: "", Type: "password", DisplayName: "provider 服务密钥", Description: "provider 自定义选项。对内置 http:作为 Bearer token 发送。可选。", Category: "memory"})
|
||||||
|
reg(ConfigDef{Key: "core.memory.multimodal_space.options.model", Default: "", Type: "string", DisplayName: "provider 模型标识", Description: "provider 自定义选项。对内置 http:外部服务使用的模型名,作为 vec_model 持久化。", Category: "memory"})
|
||||||
|
reg(ConfigDef{Key: "core.memory.multimodal_space.options.dimension", Default: "0", Type: "int", DisplayName: "provider 向量维度", Description: "provider 自定义选项。对内置 http:服务返回的特征向量维度,必须与实际返回值一致。", Category: "memory"})
|
||||||
|
reg(ConfigDef{Key: "core.memory.multimodal_space.options.timeout", Default: "30s", Type: "duration", DisplayName: "provider 请求超时", Description: "provider 自定义选项。对内置 http:单次向量请求的超时时间。", Category: "memory"})
|
||||||
|
reg(ConfigDef{Key: "core.memory.multimodal_space.options.fingerprint", Default: "", Type: "string", DisplayName: "provider 空间指纹", Description: "provider 自定义选项。对内置 http:向量空间版本标识(留空时根据 model+dim 自动生成)。指纹变化会触发历史向量重算。", Category: "memory"})
|
||||||
reg(ConfigDef{Key: "core.knowledge.path", Default: filepath.Join(dataDir, "knowledge"), Type: "string", DisplayName: "知识库路径", Description: "知识库存储目录", Category: "paths"})
|
reg(ConfigDef{Key: "core.knowledge.path", Default: filepath.Join(dataDir, "knowledge"), Type: "string", DisplayName: "知识库路径", Description: "知识库存储目录", Category: "paths"})
|
||||||
reg(ConfigDef{Key: "core.log.path", Default: filepath.Join(dataDir, "log"), Type: "string", DisplayName: "日志目录", Description: "日志文件输出目录", Category: "paths"})
|
reg(ConfigDef{Key: "core.log.path", Default: filepath.Join(dataDir, "log"), Type: "string", DisplayName: "日志目录", Description: "日志文件输出目录", Category: "paths"})
|
||||||
|
|
||||||
@ -861,6 +904,8 @@ func (r *ConfigRegistry) ToConfig() *types.Config {
|
|||||||
MaxConcurrent: readInt(p+".max_concurrent", 8),
|
MaxConcurrent: readInt(p+".max_concurrent", 8),
|
||||||
Priority: readInt(p+".priority", 0),
|
Priority: readInt(p+".priority", 0),
|
||||||
ThinkingEnabled: readBool(p+".thinking_enabled", false),
|
ThinkingEnabled: readBool(p+".thinking_enabled", false),
|
||||||
|
Vision: readBool(p+".vision", false),
|
||||||
|
Audio: readBool(p+".audio", false),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -188,6 +188,65 @@ func TestSeedDefaultsToConfig(t *testing.T) {
|
|||||||
r.Close()
|
r.Close()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 发行包全新安装:postinst 先跑 setup.sh → initconfig,而 initconfig 只写
|
||||||
|
// webui.listen_addr。于是 config 表已经非空,旧实现据此判定“已有配置”并整体
|
||||||
|
// 跳过播种——装完没有 core.plugin.dir(0 个插件)、也没有随包模型对应的
|
||||||
|
// 多模态 provider(754MB 产物 + 24MB 运行库全成死重量)。
|
||||||
|
func TestSeedDefaultsAfterInitconfigPrepopulate(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
path := filepath.Join(dir, "config.db")
|
||||||
|
|
||||||
|
r := NewConfigRegistry(path)
|
||||||
|
// 精确复现 initconfig 的唯一一笔写入
|
||||||
|
if _, err := r.db.Exec(`INSERT INTO config (key, value) VALUES ('webui.listen_addr', ':8080')`); err != nil {
|
||||||
|
t.Fatalf("预置 initconfig 行: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
r.SeedDefaults(dir)
|
||||||
|
|
||||||
|
for _, k := range []string{"core.daemon.data_dir", "core.plugin.dir", "core.memory.multimodal_space.provider"} {
|
||||||
|
if r.GetString(k, "") == "" {
|
||||||
|
t.Fatalf("全新安装(initconfig 已写 webui.listen_addr)后 %s 仍为空:默认值播种被跳过", k)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if got := r.GetString("core.memory.multimodal_space.provider", ""); got != "chineseclip" {
|
||||||
|
t.Fatalf("随包默认 provider 应为 chineseclip,实为 %q", got)
|
||||||
|
}
|
||||||
|
r.Close()
|
||||||
|
}
|
||||||
|
|
||||||
|
// 老安装升级:绝不能因为新版本加了默认值就把它注进现有 DB——那会让升级即
|
||||||
|
// 静默加载一个 1.8GB 的模型。判据是 core.daemon.data_dir 在场(老安装由播种
|
||||||
|
// 写入)而 seed 标记缺失。
|
||||||
|
func TestSeedDefaultsDoesNotInjectIntoLegacyInstall(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
path := filepath.Join(dir, "config.db")
|
||||||
|
|
||||||
|
r := NewConfigRegistry(path)
|
||||||
|
if _, err := r.db.Exec(`INSERT INTO config (key, value) VALUES ('core.daemon.data_dir', ?)`, dir); err != nil {
|
||||||
|
t.Fatalf("预置老安装行: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
r.SeedDefaults(dir)
|
||||||
|
|
||||||
|
if got := r.GetString("core.memory.multimodal_space.provider", ""); got != "" {
|
||||||
|
t.Fatalf("老安装升级被注入新默认值 provider=%q(升级后会静默加载大模型)", got)
|
||||||
|
}
|
||||||
|
if got := r.GetString("core.plugin.dir", ""); got != "" {
|
||||||
|
t.Fatalf("老安装升级被注入新默认值 core.plugin.dir=%q", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 但标记必须补上,否则每次启动都会重走判断
|
||||||
|
var n int
|
||||||
|
if err := r.db.QueryRow(`SELECT COUNT(*) FROM config WHERE key = 'core.internal.seed_version'`).Scan(&n); err != nil {
|
||||||
|
t.Fatalf("查 seed 标记: %v", err)
|
||||||
|
}
|
||||||
|
if n != 1 {
|
||||||
|
t.Fatalf("老安装应补上 seed 标记,实际 count=%d", n)
|
||||||
|
}
|
||||||
|
r.Close()
|
||||||
|
}
|
||||||
|
|
||||||
func TestGetHelpers(t *testing.T) {
|
func TestGetHelpers(t *testing.T) {
|
||||||
r := NewConfigRegistry("")
|
r := NewConfigRegistry("")
|
||||||
r.Set("str_key", "hello")
|
r.Set("str_key", "hello")
|
||||||
|
|||||||
@ -201,6 +201,13 @@ func (s *Store) Add(name, content string) error {
|
|||||||
}
|
}
|
||||||
s.items[id] = k
|
s.items[id] = k
|
||||||
|
|
||||||
|
// 覆盖同名条目时必须先摘掉旧向量。
|
||||||
|
//
|
||||||
|
// vector.Store.Insert 是**追加**语义(s.docs = append + index.Add),不按 id
|
||||||
|
// 去重。少了这一步,更新一条知识会在向量索引里留下上一版的副本:条目数看起来
|
||||||
|
// 是对的,只有向量数比条目数多——而检索可能因此命中已被替换掉的旧内容。
|
||||||
|
s.vec.Remove(id)
|
||||||
|
|
||||||
vec := s.vectorize(name + " " + content)
|
vec := s.vectorize(name + " " + content)
|
||||||
s.vec.Insert(id, name+": "+content, vec, map[string]string{
|
s.vec.Insert(id, name+": "+content, vec, map[string]string{
|
||||||
"name": name, "path": path,
|
"name": name, "path": path,
|
||||||
|
|||||||
@ -44,6 +44,51 @@ func TestAddAndSearch(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 覆盖同名条目必须把旧向量摘掉,而不是再插一份。
|
||||||
|
//
|
||||||
|
// 这条是从一次真实的知识库更新里发现的:在线上实例更新一个已有条目后,
|
||||||
|
// knowledge_count=32 但 vector_count=33 ——多出来的那一条是上一版的副本。
|
||||||
|
// 成因是 vector.Store.Insert 为追加语义(s.docs = append + index.Add),不按 id 去重。
|
||||||
|
// 危害不在于多占一份内存:检索可能命中**已被替换掉的旧内容**。
|
||||||
|
func TestAddOverwriteReplacesVector(t *testing.T) {
|
||||||
|
dir, err := os.MkdirTemp("", "know_overwrite_*")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer os.RemoveAll(dir)
|
||||||
|
|
||||||
|
s := NewStore(dir)
|
||||||
|
s.Start()
|
||||||
|
defer s.Stop()
|
||||||
|
|
||||||
|
if err := s.Add("recent", "第一版内容:旧的多模态描述式索引"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got := s.Stats()["vector_count"].(int); got != 1 {
|
||||||
|
t.Fatalf("首次写入后 vector_count 应为 1,实为 %d", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := s.Add("recent", "第二版内容:媒体已成为图记忆的一等节点"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if n := s.Stats()["knowledge_count"].(int); n != 1 {
|
||||||
|
t.Fatalf("同名覆盖后 knowledge_count 应为 1,实为 %d", n)
|
||||||
|
}
|
||||||
|
if n := s.Stats()["vector_count"].(int); n != 1 {
|
||||||
|
t.Fatalf("同名覆盖后 vector_count 应为 1(多了就是旧版没被摘掉),实为 %d", n)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 目录里也只应有一份内容,且是新的那份
|
||||||
|
b, err := os.ReadFile(dir + "/recent/content.md")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if string(b) != "第二版内容:媒体已成为图记忆的一等节点" {
|
||||||
|
t.Fatalf("content.md 未被新内容覆盖,实为 %q", string(b))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestList(t *testing.T) {
|
func TestList(t *testing.T) {
|
||||||
dir, err := os.MkdirTemp("", "know_list_*")
|
dir, err := os.MkdirTemp("", "know_list_*")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@ -7,11 +7,11 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
type bilingualEvent struct {
|
type bilingualEvent struct {
|
||||||
idx int
|
idx int
|
||||||
source string
|
source string
|
||||||
topic string
|
topic string
|
||||||
text string // cleaned text for vectorization
|
text string // cleaned text for vectorization
|
||||||
label string // short description
|
label string // short description
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestBilingualPruningAccuracy(t *testing.T) {
|
func TestBilingualPruningAccuracy(t *testing.T) {
|
||||||
@ -40,16 +40,16 @@ func TestBilingualPruningAccuracy(t *testing.T) {
|
|||||||
t.Logf("%s: %d words", cfg.name, len(e.words))
|
t.Logf("%s: %d words", cfg.name, len(e.words))
|
||||||
|
|
||||||
type scored struct {
|
type scored struct {
|
||||||
idx int
|
idx int
|
||||||
topic string
|
topic string
|
||||||
label string
|
label string
|
||||||
score float64
|
score float64
|
||||||
}
|
}
|
||||||
|
|
||||||
queries := []struct {
|
queries := []struct {
|
||||||
q string
|
q string
|
||||||
qTopic string
|
qTopic string
|
||||||
desc string
|
desc string
|
||||||
}{
|
}{
|
||||||
{"老大说了关于 React 组件的事情", "老大私聊", "中英混合:老大+React"},
|
{"老大说了关于 React 组件的事情", "老大私聊", "中英混合:老大+React"},
|
||||||
{"帮我查一下 Nginx 反向代理配置", "服务器运维", "中英混合:Nginx+反向代理"},
|
{"帮我查一下 Nginx 反向代理配置", "服务器运维", "中英混合:Nginx+反向代理"},
|
||||||
@ -189,10 +189,10 @@ func TestBilingualVectorizeClean(t *testing.T) {
|
|||||||
|
|
||||||
func genBilingualEvents() []bilingualEvent {
|
func genBilingualEvents() []bilingualEvent {
|
||||||
entries := []struct {
|
entries := []struct {
|
||||||
topic string
|
topic string
|
||||||
zh string // Chinese description
|
zh string // Chinese description
|
||||||
en string // English terms mixed in
|
en string // English terms mixed in
|
||||||
source string
|
source string
|
||||||
}{
|
}{
|
||||||
{"大学招生", "河南医药大学录取分数线", "", "qq"},
|
{"大学招生", "河南医药大学录取分数线", "", "qq"},
|
||||||
{"大学招生", "医学院专业排名", "medical university ranking", "agent"},
|
{"大学招生", "医学院专业排名", "medical university ranking", "agent"},
|
||||||
|
|||||||
271
internal/memory/block.go
Normal file
271
internal/memory/block.go
Normal file
@ -0,0 +1,271 @@
|
|||||||
|
package memory
|
||||||
|
|
||||||
|
import (
|
||||||
|
"database/sql"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// BlockModality 是一等记忆块的原生模态。
|
||||||
|
type BlockModality string
|
||||||
|
|
||||||
|
const (
|
||||||
|
BlockText BlockModality = "text"
|
||||||
|
BlockImage BlockModality = "image"
|
||||||
|
BlockVideo BlockModality = "video"
|
||||||
|
BlockAudio BlockModality = "audio"
|
||||||
|
)
|
||||||
|
|
||||||
|
// MemoryBlock 是 Context、Document、Graph 三层共同使用的记忆块值。
|
||||||
|
//
|
||||||
|
// 它不携带 Layer、Owner 或 RefCount:块当前由哪个层的容器持有,哪个层就是
|
||||||
|
// 唯一事实源。Context→Document→Graph 迁移的是这个值本身,不建立平行保活账本。
|
||||||
|
// PayloadDigest 仅用于定位内容寻址的原始字节,不表示另一条逻辑记忆。
|
||||||
|
type MemoryBlock struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
Modality BlockModality `json:"modality"`
|
||||||
|
Text string `json:"text,omitempty"`
|
||||||
|
PayloadDigest string `json:"payload_digest,omitempty"`
|
||||||
|
MIME string `json:"mime,omitempty"`
|
||||||
|
Size int64 `json:"size,omitempty"`
|
||||||
|
Width int `json:"width,omitempty"`
|
||||||
|
Height int `json:"height,omitempty"`
|
||||||
|
Vector []float64 `json:"vector,omitempty"`
|
||||||
|
Fingerprint string `json:"fingerprint,omitempty"`
|
||||||
|
Source string `json:"source,omitempty"`
|
||||||
|
Tool string `json:"tool,omitempty"`
|
||||||
|
CreatedAt time.Time `json:"created_at"`
|
||||||
|
UpdatedAt time.Time `json:"updated_at"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// MemoryBlockEdge 是 L3 中连接一等记忆节点的结构化语义边。
|
||||||
|
// source/target kind 当前允许 block、entity、sentence、document。
|
||||||
|
type MemoryBlockEdge struct {
|
||||||
|
ID int64 `json:"id"`
|
||||||
|
SourceKind string `json:"source_kind"`
|
||||||
|
SourceID string `json:"source_id"`
|
||||||
|
TargetKind string `json:"target_kind"`
|
||||||
|
TargetID string `json:"target_id"`
|
||||||
|
Type string `json:"type"`
|
||||||
|
CreatedAt time.Time `json:"created_at"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func validBlockModality(modality BlockModality) bool {
|
||||||
|
return modality == BlockText || modality == BlockImage || modality == BlockVideo || modality == BlockAudio
|
||||||
|
}
|
||||||
|
|
||||||
|
// PutMemoryBlocks 将完成 L2→L3 迁移的块写成 GraphDB 原生节点。
|
||||||
|
// 调用方只有在本事务成功后才能从 Document 删除这些块。
|
||||||
|
func (g *GraphDB) PutMemoryBlocks(blocks []MemoryBlock) error {
|
||||||
|
if len(blocks) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
g.mu.Lock()
|
||||||
|
defer g.mu.Unlock()
|
||||||
|
tx, err := g.db.Begin()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer tx.Rollback()
|
||||||
|
|
||||||
|
for _, block := range blocks {
|
||||||
|
if block.ID == "" {
|
||||||
|
return fmt.Errorf("memory block id is required")
|
||||||
|
}
|
||||||
|
if !validBlockModality(block.Modality) {
|
||||||
|
return fmt.Errorf("memory block %s has invalid modality %q", block.ID, block.Modality)
|
||||||
|
}
|
||||||
|
vectorJSON, err := json.Marshal(block.Vector)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("marshal memory block %s vector: %w", block.ID, err)
|
||||||
|
}
|
||||||
|
now := time.Now()
|
||||||
|
if block.CreatedAt.IsZero() {
|
||||||
|
block.CreatedAt = now
|
||||||
|
}
|
||||||
|
_, err = tx.Exec(`INSERT INTO memory_blocks (
|
||||||
|
id, modality, text_content, payload_digest, mime, size, width, height,
|
||||||
|
vector, fingerprint, source, tool, created_at, updated_at
|
||||||
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||||
|
ON CONFLICT(id) DO UPDATE SET
|
||||||
|
modality = excluded.modality,
|
||||||
|
text_content = excluded.text_content,
|
||||||
|
payload_digest = excluded.payload_digest,
|
||||||
|
mime = excluded.mime,
|
||||||
|
size = excluded.size,
|
||||||
|
width = excluded.width,
|
||||||
|
height = excluded.height,
|
||||||
|
vector = excluded.vector,
|
||||||
|
fingerprint = excluded.fingerprint,
|
||||||
|
source = excluded.source,
|
||||||
|
tool = excluded.tool,
|
||||||
|
updated_at = excluded.updated_at`,
|
||||||
|
block.ID, block.Modality, block.Text, block.PayloadDigest, block.MIME,
|
||||||
|
block.Size, block.Width, block.Height, string(vectorJSON), block.Fingerprint,
|
||||||
|
block.Source, block.Tool, block.CreatedAt, now)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("put memory block %s: %w", block.ID, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return tx.Commit()
|
||||||
|
}
|
||||||
|
|
||||||
|
// PutDocumentNode 在 L3 登记一个文档节点,作为 document --contains--> block
|
||||||
|
// 结构边的端点。文档正文已蒸馏为实体/关系,这里只保留身份与摘要。
|
||||||
|
func (g *GraphDB) PutDocumentNode(id, summary string) error {
|
||||||
|
if id == "" {
|
||||||
|
return fmt.Errorf("document node id is required")
|
||||||
|
}
|
||||||
|
g.mu.Lock()
|
||||||
|
defer g.mu.Unlock()
|
||||||
|
_, err := g.db.Exec(`INSERT INTO documents (id, summary) VALUES (?, ?)
|
||||||
|
ON CONFLICT(id) DO UPDATE SET summary = excluded.summary`, id, summary)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// MemoryBlocks 查询 Graph 层实际持有的一等记忆节点。
|
||||||
|
func (g *GraphDB) MemoryBlocks() ([]MemoryBlock, error) {
|
||||||
|
g.mu.RLock()
|
||||||
|
defer g.mu.RUnlock()
|
||||||
|
rows, err := g.db.Query(`SELECT id, modality, text_content, payload_digest, mime,
|
||||||
|
size, width, height, vector, fingerprint, source, tool, created_at, updated_at
|
||||||
|
FROM memory_blocks ORDER BY created_at, id`)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
|
||||||
|
var blocks []MemoryBlock
|
||||||
|
for rows.Next() {
|
||||||
|
var block MemoryBlock
|
||||||
|
var vectorJSON string
|
||||||
|
if err := rows.Scan(&block.ID, &block.Modality, &block.Text, &block.PayloadDigest,
|
||||||
|
&block.MIME, &block.Size, &block.Width, &block.Height, &vectorJSON,
|
||||||
|
&block.Fingerprint, &block.Source, &block.Tool, &block.CreatedAt,
|
||||||
|
&block.UpdatedAt); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if vectorJSON != "" && vectorJSON != "null" {
|
||||||
|
if err := json.Unmarshal([]byte(vectorJSON), &block.Vector); err != nil {
|
||||||
|
return nil, fmt.Errorf("decode memory block %s vector: %w", block.ID, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
blocks = append(blocks, block)
|
||||||
|
}
|
||||||
|
return blocks, rows.Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
func validGraphNodeKind(kind string) bool {
|
||||||
|
return kind == "block" || kind == "entity" || kind == "sentence" || kind == "document"
|
||||||
|
}
|
||||||
|
|
||||||
|
func graphNodeExists(tx *sql.Tx, kind, id string) (bool, error) {
|
||||||
|
var n int
|
||||||
|
var err error
|
||||||
|
switch kind {
|
||||||
|
case "block":
|
||||||
|
err = tx.QueryRow(`SELECT COUNT(*) FROM memory_blocks WHERE id = ?`, id).Scan(&n)
|
||||||
|
case "entity":
|
||||||
|
err = tx.QueryRow(`SELECT COUNT(*) FROM entities WHERE CAST(id AS TEXT) = ?`, id).Scan(&n)
|
||||||
|
case "sentence":
|
||||||
|
err = tx.QueryRow(`SELECT COUNT(*) FROM sentences WHERE CAST(id AS TEXT) = ?`, id).Scan(&n)
|
||||||
|
case "document":
|
||||||
|
err = tx.QueryRow(`SELECT COUNT(*) FROM documents WHERE id = ?`, id).Scan(&n)
|
||||||
|
default:
|
||||||
|
return false, fmt.Errorf("invalid graph node kind %q", kind)
|
||||||
|
}
|
||||||
|
return n == 1, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// AddMemoryBlockEdge 建立 contains、depicts、derived_from 等原生图边。
|
||||||
|
// 端点必须是真实 Graph 节点,不能用 owner 字符串伪装关系。
|
||||||
|
func (g *GraphDB) AddMemoryBlockEdge(sourceKind, sourceID, targetKind, targetID, edgeType string) error {
|
||||||
|
if !validGraphNodeKind(sourceKind) || !validGraphNodeKind(targetKind) {
|
||||||
|
return fmt.Errorf("invalid memory block edge kinds %q -> %q", sourceKind, targetKind)
|
||||||
|
}
|
||||||
|
if sourceID == "" || targetID == "" || edgeType == "" {
|
||||||
|
return fmt.Errorf("memory block edge endpoints and type are required")
|
||||||
|
}
|
||||||
|
g.mu.Lock()
|
||||||
|
defer g.mu.Unlock()
|
||||||
|
tx, err := g.db.Begin()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer tx.Rollback()
|
||||||
|
for _, endpoint := range []struct{ kind, id string }{{sourceKind, sourceID}, {targetKind, targetID}} {
|
||||||
|
exists, err := graphNodeExists(tx, endpoint.kind, endpoint.id)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !exists {
|
||||||
|
return fmt.Errorf("%s graph node %s does not exist", endpoint.kind, endpoint.id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_, err = tx.Exec(`INSERT OR IGNORE INTO memory_block_edges
|
||||||
|
(source_kind, source_id, target_kind, target_id, edge_type)
|
||||||
|
VALUES (?, ?, ?, ?, ?)`, sourceKind, sourceID, targetKind, targetID, edgeType)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return tx.Commit()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (g *GraphDB) MemoryBlockEdges() ([]MemoryBlockEdge, error) {
|
||||||
|
g.mu.RLock()
|
||||||
|
defer g.mu.RUnlock()
|
||||||
|
rows, err := g.db.Query(`SELECT id, source_kind, source_id, target_kind, target_id,
|
||||||
|
edge_type, created_at FROM memory_block_edges ORDER BY id`)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
var edges []MemoryBlockEdge
|
||||||
|
for rows.Next() {
|
||||||
|
var edge MemoryBlockEdge
|
||||||
|
if err := rows.Scan(&edge.ID, &edge.SourceKind, &edge.SourceID, &edge.TargetKind,
|
||||||
|
&edge.TargetID, &edge.Type, &edge.CreatedAt); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
edges = append(edges, edge)
|
||||||
|
}
|
||||||
|
return edges, rows.Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
// BlocksForNode 返回与某个图节点通过任意边相连的一等记忆块。
|
||||||
|
// 例:sentence --contains--> block;entity --depicts--> block。
|
||||||
|
func (g *GraphDB) BlocksForNode(nodeKind, nodeID string) ([]MemoryBlock, error) {
|
||||||
|
g.mu.RLock()
|
||||||
|
defer g.mu.RUnlock()
|
||||||
|
rows, err := g.db.Query(`SELECT b.id, b.modality, b.text_content, b.payload_digest,
|
||||||
|
b.mime, b.size, b.width, b.height, b.vector, b.fingerprint, b.source, b.tool,
|
||||||
|
b.created_at, b.updated_at
|
||||||
|
FROM memory_block_edges e
|
||||||
|
JOIN memory_blocks b ON (
|
||||||
|
(e.source_kind = 'block' AND e.source_id = b.id AND e.target_kind = ? AND e.target_id = ?)
|
||||||
|
OR (e.target_kind = 'block' AND e.target_id = b.id AND e.source_kind = ? AND e.source_id = ?))
|
||||||
|
ORDER BY b.created_at, b.id`, nodeKind, nodeID, nodeKind, nodeID)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
var blocks []MemoryBlock
|
||||||
|
for rows.Next() {
|
||||||
|
var block MemoryBlock
|
||||||
|
var vectorJSON string
|
||||||
|
if err := rows.Scan(&block.ID, &block.Modality, &block.Text, &block.PayloadDigest,
|
||||||
|
&block.MIME, &block.Size, &block.Width, &block.Height, &vectorJSON,
|
||||||
|
&block.Fingerprint, &block.Source, &block.Tool, &block.CreatedAt,
|
||||||
|
&block.UpdatedAt); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if vectorJSON != "" && vectorJSON != "null" {
|
||||||
|
if err := json.Unmarshal([]byte(vectorJSON), &block.Vector); err != nil {
|
||||||
|
return nil, fmt.Errorf("decode memory block %s vector: %w", block.ID, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
blocks = append(blocks, block)
|
||||||
|
}
|
||||||
|
return blocks, rows.Err()
|
||||||
|
}
|
||||||
@ -28,13 +28,13 @@ func cleanQQTemplate(text string) string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type cleanTestEvent struct {
|
type cleanTestEvent struct {
|
||||||
idx int
|
idx int
|
||||||
source string
|
source string
|
||||||
input string
|
input string
|
||||||
response string
|
response string
|
||||||
rawText string
|
rawText string
|
||||||
cleanedText string
|
cleanedText string
|
||||||
topic string
|
topic string
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestCleanStressPrecision(t *testing.T) {
|
func TestCleanStressPrecision(t *testing.T) {
|
||||||
@ -59,53 +59,53 @@ func TestCleanStressPrecision(t *testing.T) {
|
|||||||
}
|
}
|
||||||
t.Logf("topics: %v, events: %d", usedTopics, len(events))
|
t.Logf("topics: %v, events: %d", usedTopics, len(events))
|
||||||
|
|
||||||
for _, qTopic := range usedTopics {
|
for _, qTopic := range usedTopics {
|
||||||
query := queryForTopic(qTopic)
|
query := queryForTopic(qTopic)
|
||||||
qVec := e.Vectorize(query)
|
qVec := e.Vectorize(query)
|
||||||
|
|
||||||
type scored struct {
|
type scored struct {
|
||||||
idx int
|
idx int
|
||||||
topic string
|
topic string
|
||||||
text string
|
text string
|
||||||
score float64
|
score float64
|
||||||
}
|
}
|
||||||
all := make([]scored, len(events))
|
all := make([]scored, len(events))
|
||||||
for i, ev := range events {
|
for i, ev := range events {
|
||||||
text := ev.rawText
|
text := ev.rawText
|
||||||
if cleanMode {
|
if cleanMode {
|
||||||
text = ev.cleanedText
|
text = ev.cleanedText
|
||||||
|
}
|
||||||
|
vec := e.Vectorize(text)
|
||||||
|
all[i] = scored{idx: i, topic: ev.topic, text: text, score: cosineSim(qVec, vec)}
|
||||||
|
}
|
||||||
|
sort.Slice(all, func(i, j int) bool { return all[i].score > all[j].score })
|
||||||
|
|
||||||
|
topK := len(usedTopics) * 2
|
||||||
|
if topK > len(all) {
|
||||||
|
topK = len(all)
|
||||||
|
}
|
||||||
|
|
||||||
|
intraHits := 0
|
||||||
|
for _, s := range all[:topK] {
|
||||||
|
if s.topic == qTopic {
|
||||||
|
intraHits++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
expected := countTopicEvents(events, qTopic)
|
||||||
|
if expected > topK {
|
||||||
|
expected = topK
|
||||||
|
}
|
||||||
|
recall := float64(intraHits) / float64(expected)
|
||||||
|
|
||||||
|
if recall < 0.3 {
|
||||||
|
t.Logf(" [LOW] query=%q topK=%d intra=%d/%d recall=%.2f", qTopic, topK, intraHits, expected, recall)
|
||||||
|
for _, s := range all[:8] {
|
||||||
|
t.Logf(" [%.4f] %s", s.score, trimLen(s.text, 60))
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
t.Logf(" [OK] query=%q topK=%d intra=%d/%d recall=%.2f", qTopic, topK, intraHits, expected, recall)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
vec := e.Vectorize(text)
|
|
||||||
all[i] = scored{idx: i, topic: ev.topic, text: text, score: cosineSim(qVec, vec)}
|
|
||||||
}
|
|
||||||
sort.Slice(all, func(i, j int) bool { return all[i].score > all[j].score })
|
|
||||||
|
|
||||||
topK := len(usedTopics) * 2
|
|
||||||
if topK > len(all) {
|
|
||||||
topK = len(all)
|
|
||||||
}
|
|
||||||
|
|
||||||
intraHits := 0
|
|
||||||
for _, s := range all[:topK] {
|
|
||||||
if s.topic == qTopic {
|
|
||||||
intraHits++
|
|
||||||
}
|
|
||||||
}
|
|
||||||
expected := countTopicEvents(events, qTopic)
|
|
||||||
if expected > topK {
|
|
||||||
expected = topK
|
|
||||||
}
|
|
||||||
recall := float64(intraHits) / float64(expected)
|
|
||||||
|
|
||||||
if recall < 0.3 {
|
|
||||||
t.Logf(" [LOW] query=%q topK=%d intra=%d/%d recall=%.2f", qTopic, topK, intraHits, expected, recall)
|
|
||||||
for _, s := range all[:8] {
|
|
||||||
t.Logf(" [%.4f] %s", s.score, trimLen(s.text, 60))
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
t.Logf(" [OK] query=%q topK=%d intra=%d/%d recall=%.2f", qTopic, topK, intraHits, expected, recall)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -252,11 +252,11 @@ func genStressEvents(n int) []cleanTestEvent {
|
|||||||
keywords []string
|
keywords []string
|
||||||
sources []string
|
sources []string
|
||||||
}{
|
}{
|
||||||
"大学招生": {[]string{"河南医药大学", "录取分数线", "专业排名", "高考志愿", "招生简章"}, []string{"qq", "qq", "agent"}},
|
"大学招生": {[]string{"河南医药大学", "录取分数线", "专业排名", "高考志愿", "招生简章"}, []string{"qq", "qq", "agent"}},
|
||||||
"老大私聊": {[]string{"老大私聊消息", "回复老大", "任务安排", "汇报工作", "收到"}, []string{"qq", "agent", "agent"}},
|
"老大私聊": {[]string{"老大私聊消息", "回复老大", "任务安排", "汇报工作", "收到"}, []string{"qq", "agent", "agent"}},
|
||||||
"前端开发": {[]string{"前端组件封装", "页面路由配置", "界面布局设计", "交互逻辑开发", "代码调试优化"}, []string{"cli", "cli", "agent"}},
|
"前端开发": {[]string{"前端组件封装", "页面路由配置", "界面布局设计", "交互逻辑开发", "代码调试优化"}, []string{"cli", "cli", "agent"}},
|
||||||
"服务器运维": {[]string{"反向代理配置", "容器部署方案", "证书续期", "数据库备份恢复", "监控告警处理"}, []string{"cli", "agent", "agent"}},
|
"服务器运维": {[]string{"反向代理配置", "容器部署方案", "证书续期", "数据库备份恢复", "监控告警处理"}, []string{"cli", "agent", "agent"}},
|
||||||
"股票基金": {[]string{"基金定投策略", "股票涨跌分析", "理财收益计算", "市场行情分析", "投资风险管理"}, []string{"qq", "qq", "agent"}},
|
"股票基金": {[]string{"基金定投策略", "股票涨跌分析", "理财收益计算", "市场行情分析", "投资风险管理"}, []string{"qq", "qq", "agent"}},
|
||||||
}
|
}
|
||||||
|
|
||||||
for i := 0; i < n; i++ {
|
for i := 0; i < n; i++ {
|
||||||
@ -292,13 +292,13 @@ func genStressEvents(n int) []cleanTestEvent {
|
|||||||
cleaned := cleanEventText(src, input, response)
|
cleaned := cleanEventText(src, input, response)
|
||||||
raw := rawEventText(src, input, response)
|
raw := rawEventText(src, input, response)
|
||||||
events = append(events, cleanTestEvent{
|
events = append(events, cleanTestEvent{
|
||||||
idx: i,
|
idx: i,
|
||||||
source: src,
|
source: src,
|
||||||
input: input,
|
input: input,
|
||||||
response: response,
|
response: response,
|
||||||
rawText: raw,
|
rawText: raw,
|
||||||
cleanedText: cleaned,
|
cleanedText: cleaned,
|
||||||
topic: tp,
|
topic: tp,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
return events
|
return events
|
||||||
|
|||||||
@ -13,26 +13,26 @@ import (
|
|||||||
|
|
||||||
// contentPOS 有实义的词性标签:只保留名词/动词/形容词/专名等
|
// contentPOS 有实义的词性标签:只保留名词/动词/形容词/专名等
|
||||||
var contentPOS = map[string]bool{
|
var contentPOS = map[string]bool{
|
||||||
"n": true, // 普通名词
|
"n": true, // 普通名词
|
||||||
"nr": true, // 人名
|
"nr": true, // 人名
|
||||||
"ns": true, // 地名
|
"ns": true, // 地名
|
||||||
"nt": true, // 机构名
|
"nt": true, // 机构名
|
||||||
"nw": true, // 作品名/URL
|
"nw": true, // 作品名/URL
|
||||||
"nz": true, // 其他专名
|
"nz": true, // 其他专名
|
||||||
"v": true, // 动词
|
"v": true, // 动词
|
||||||
"vd": true, // 副动词
|
"vd": true, // 副动词
|
||||||
"vn": true, // 名动词
|
"vn": true, // 名动词
|
||||||
"a": true, // 形容词
|
"a": true, // 形容词
|
||||||
"ad": true, // 副形词
|
"ad": true, // 副形词
|
||||||
"an": true, // 名形词
|
"an": true, // 名形词
|
||||||
"i": true, // 成语
|
"i": true, // 成语
|
||||||
"l": true, // 习用语
|
"l": true, // 习用语
|
||||||
"j": true, // 简称
|
"j": true, // 简称
|
||||||
"s": true, // 处所词
|
"s": true, // 处所词
|
||||||
"f": true, // 方位词
|
"f": true, // 方位词
|
||||||
"b": true, // 区别词
|
"b": true, // 区别词
|
||||||
"z": true, // 状态词
|
"z": true, // 状态词
|
||||||
"t": true, // 时间词
|
"t": true, // 时间词
|
||||||
"eng": true, // 英文
|
"eng": true, // 英文
|
||||||
"x": true, // 非语素字
|
"x": true, // 非语素字
|
||||||
"zg": true, // 其他
|
"zg": true, // 其他
|
||||||
@ -53,7 +53,7 @@ func GetJieba() *gojieba.Jieba {
|
|||||||
}()
|
}()
|
||||||
d := jiebaDictDir()
|
d := jiebaDictDir()
|
||||||
if d == "" {
|
if d == "" {
|
||||||
log.Printf("[jieba] no dictionary directory found, jieba disabled")
|
log.Printf("[jieba] 未找到词库目录(内嵌落盘失败且模块缓存也不存在),jieba disabled")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
jiebaInst = gojieba.NewJieba(
|
jiebaInst = gojieba.NewJieba(
|
||||||
@ -68,9 +68,24 @@ func GetJieba() *gojieba.Jieba {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func jiebaDictDir() string {
|
func jiebaDictDir() string {
|
||||||
|
// 首选内嵌词库:它是产物的一部分,与二进制同版本、不依赖宿主环境。
|
||||||
|
//
|
||||||
|
// 以前这里只猜 GOMODCACHE/GOPATH/~/go/pkg/mod,部署机上通常没有 Go 模块缓存,
|
||||||
|
// 于是分词与关键词提取会**静默退回空列表**(详见 jieba_embed.go 的说明)。
|
||||||
|
if dir, err := materializeJiebaDict(); err == nil && dir != "" {
|
||||||
|
return dir
|
||||||
|
}
|
||||||
|
log.Printf("[jieba] 内嵌词库落盘失败,回退到模块缓存查找(内嵌失败通常意味着缓存目录不可写)")
|
||||||
|
|
||||||
|
// 回退:开发机上存在的模块缓存(仅作为兵底,不应依赖它)。
|
||||||
|
//
|
||||||
// GOMODCACHE is typically $GOPATH/pkg/mod. When set, Go writes modules
|
// GOMODCACHE is typically $GOPATH/pkg/mod. When set, Go writes modules
|
||||||
// under <GOMODCACHE>/github.com/... . Look first at GOMODCACHE, then
|
// under <GOMODCACHE>/github.com/... . Look first at GOMODCACHE, then
|
||||||
// derive from GOPATH, then try common locations.
|
// derive from GOPATH, then try common locations.
|
||||||
|
return jiebaDictDirFromModuleCache()
|
||||||
|
}
|
||||||
|
|
||||||
|
func jiebaDictDirFromModuleCache() string {
|
||||||
candidates := []string{
|
candidates := []string{
|
||||||
os.Getenv("GOMODCACHE"),
|
os.Getenv("GOMODCACHE"),
|
||||||
}
|
}
|
||||||
|
|||||||
@ -4,6 +4,7 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"log"
|
"log"
|
||||||
|
"math"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"sort"
|
"sort"
|
||||||
@ -13,6 +14,7 @@ import (
|
|||||||
|
|
||||||
"gitcode.com/JianFeeeee/HomeAgent/internal/memory"
|
"gitcode.com/JianFeeeee/HomeAgent/internal/memory"
|
||||||
"gitcode.com/JianFeeeee/HomeAgent/internal/memory/vector"
|
"gitcode.com/JianFeeeee/HomeAgent/internal/memory/vector"
|
||||||
|
"gitcode.com/JianFeeeee/HomeAgent/internal/tfidf"
|
||||||
)
|
)
|
||||||
|
|
||||||
// ChannelCleaner 按事件来源查找输入通道的 Cleaner 函数。
|
// ChannelCleaner 按事件来源查找输入通道的 Cleaner 函数。
|
||||||
@ -21,63 +23,77 @@ type ChannelCleaner func(source string) func(string) string
|
|||||||
|
|
||||||
// Doc — 记忆文档:由上下文提炼而来
|
// Doc — 记忆文档:由上下文提炼而来
|
||||||
type Doc struct {
|
type Doc struct {
|
||||||
ID string `json:"id"`
|
ID string `json:"id"`
|
||||||
Summary string `json:"summary"`
|
Summary string `json:"summary"`
|
||||||
Content string `json:"content"`
|
Content string `json:"content"`
|
||||||
Tags []string `json:"tags"`
|
Tags []string `json:"tags"`
|
||||||
Entities []string `json:"entities"`
|
Entities []string `json:"entities"`
|
||||||
CreatedAt time.Time `json:"created_at"`
|
CreatedAt time.Time `json:"created_at"`
|
||||||
UpdatedAt time.Time `json:"updated_at"`
|
UpdatedAt time.Time `json:"updated_at"`
|
||||||
Source string `json:"source"` // context / graph / manual
|
Source string `json:"source"`
|
||||||
Meta map[string]string `json:"meta,omitempty"`
|
Meta map[string]string `json:"meta,omitempty"`
|
||||||
AccessCount int `json:"access_count"` // 访问次数
|
AccessCount int `json:"access_count"`
|
||||||
LastAccess time.Time `json:"last_access"` // 最后访问时间
|
LastAccess time.Time `json:"last_access"`
|
||||||
Vector vector.Vector `json:"vector,omitempty"` // 预计算向量(与 context 同空间),nil 则用 TF-IDF 兜底
|
Blocks []memory.MemoryBlock `json:"blocks,omitempty"` // 一等记忆块(text/image/video/audio)
|
||||||
|
Vector tfidf.Vector `json:"vector,omitempty"` // TF-IDF 稀疏向量(fallback 时持久化)
|
||||||
|
DenseVec []float64 `json:"dense_vec,omitempty"` // 多模态稠密向量(主路径)
|
||||||
|
DenseFP string `json:"dense_fp,omitempty"` // DenseVec 所属统一空间指纹,变化时触发重算
|
||||||
}
|
}
|
||||||
|
|
||||||
// Store — 文档记忆存储,包含向量索引
|
// Store — 文档记忆存储。
|
||||||
|
// 主路径:denseSpace(稠密多模态向量,与媒体共享空间)。
|
||||||
|
// Fallback:tfidf(TF-IDF 倒排索引,仅稠密空间不可用时加载)。
|
||||||
type Store struct {
|
type Store struct {
|
||||||
dir string
|
dir string
|
||||||
vec *vector.Store
|
mu sync.RWMutex
|
||||||
veczer *vector.TFIDFVectorizer
|
docs map[string]*Doc
|
||||||
mu sync.RWMutex
|
|
||||||
|
|
||||||
docs map[string]*Doc
|
|
||||||
summaries []string // 用于训练向量化器,最大 10000 条
|
|
||||||
vectorizer vector.Vectorizer // 可选:与 context 同空间的向量化器
|
|
||||||
|
|
||||||
dirty bool
|
dirty bool
|
||||||
}
|
|
||||||
|
|
||||||
func (s *Store) SetVectorizer(v vector.Vectorizer) {
|
// fallback 路径(仅稠密空间不可用时加载)
|
||||||
s.vectorizer = v
|
tfidfEmb *tfidf.Embedder
|
||||||
}
|
tfidfIdx *tfidf.SearchableIndex
|
||||||
|
trainTexts []string // 缓存训练文本,延迟训练
|
||||||
|
tfidfOnce sync.Once
|
||||||
|
|
||||||
// ReindexWithVectorizer 用给定的向量化器重建所有文档的向量索引
|
// 主路径
|
||||||
func (s *Store) ReindexWithVectorizer(v vector.Vectorizer) {
|
denseSpace vector.MultimodalEmbedder
|
||||||
s.mu.Lock()
|
|
||||||
defer s.mu.Unlock()
|
|
||||||
|
|
||||||
log.Printf("[document memory] reindex with vectorizer (%d docs)", len(s.docs))
|
|
||||||
s.vec = vector.NewStore()
|
|
||||||
for _, doc := range s.docs {
|
|
||||||
doc.Vector = v.Vectorize(doc.Summary + " " + doc.Content)
|
|
||||||
s.vec.Insert(doc.ID, doc.Summary, doc.Vector, doc.Meta)
|
|
||||||
}
|
|
||||||
log.Printf("[document memory] reindex with vectorizer complete (%d vectors)", s.vec.Size())
|
|
||||||
}
|
}
|
||||||
|
|
||||||
const maxSummaries = 10000
|
const maxSummaries = 10000
|
||||||
|
|
||||||
func NewStore(dir string) *Store {
|
// NewStore 创建文档存储。tokenizer 由外层注入(如 jieba),核心不直接依赖分词库。
|
||||||
|
func NewStore(dir string, tokenizer tfidf.Tokenizer) *Store {
|
||||||
return &Store{
|
return &Store{
|
||||||
dir: dir,
|
dir: dir,
|
||||||
vec: vector.NewStore(),
|
docs: make(map[string]*Doc),
|
||||||
veczer: vector.NewTFIDFVectorizer(memory.TokenizeWords),
|
// tfidf 延迟初始化:只在需要 fallback 时创建
|
||||||
docs: make(map[string]*Doc),
|
tfidfEmb: tfidf.NewEmbedder(tokenizer, 4096),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ensureTFIDF 延迟初始化 TF-IDF 索引(仅 fallback 路径)。
|
||||||
|
// 调用方已持有 s.mu。
|
||||||
|
func (s *Store) ensureTFIDF() {
|
||||||
|
s.tfidfOnce.Do(func() {
|
||||||
|
s.tfidfIdx = tfidf.NewSearchableIndex(s.tfidfEmb)
|
||||||
|
// 延迟训练:用缓存的文本建立索引
|
||||||
|
texts := make(map[string]string, len(s.trainTexts)/2)
|
||||||
|
for i := 0; i+1 < len(s.trainTexts); i += 2 {
|
||||||
|
texts[s.trainTexts[i]] = s.trainTexts[i+1]
|
||||||
|
}
|
||||||
|
s.tfidfIdx.Train(texts)
|
||||||
|
s.trainTexts = nil // 释放缓存
|
||||||
|
s.tfidfEmb.Train(func() []string {
|
||||||
|
out := make([]string, 0, len(texts))
|
||||||
|
for _, t := range texts {
|
||||||
|
out = append(out, t)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}())
|
||||||
|
log.Printf("[document memory] tfidf fallback loaded: %d docs", len(texts))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
func (s *Store) Start() error {
|
func (s *Store) Start() error {
|
||||||
if err := os.MkdirAll(s.dir, 0755); err != nil {
|
if err := os.MkdirAll(s.dir, 0755); err != nil {
|
||||||
return fmt.Errorf("document store dir: %w", err)
|
return fmt.Errorf("document store dir: %w", err)
|
||||||
@ -85,15 +101,82 @@ func (s *Store) Start() error {
|
|||||||
if err := s.loadAll(); err != nil {
|
if err := s.loadAll(); err != nil {
|
||||||
log.Printf("[document memory] load error: %v", err)
|
log.Printf("[document memory] load error: %v", err)
|
||||||
}
|
}
|
||||||
log.Printf("[document memory] started with %d docs, %d vectors", len(s.docs), s.vec.Size())
|
log.Printf("[document memory] started with %d docs", len(s.docs))
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Store) Stop() {
|
func (s *Store) Stop() { s.flush() }
|
||||||
s.flush()
|
|
||||||
|
// SetDenseSpace 设置稠密多模态向量空间(主路径)。
|
||||||
|
func (s *Store) SetDenseSpace(ds vector.MultimodalEmbedder) {
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
s.denseSpace = ds
|
||||||
|
}
|
||||||
|
|
||||||
|
// BuildDenseIndex 为所有文档计算稠密向量(文本 ⊕ 媒体块)。
|
||||||
|
func (s *Store) BuildDenseIndex(ds vector.MultimodalEmbedder) {
|
||||||
|
if ds == nil || !ds.Loaded() {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
log.Printf("[document memory] building dense index for %d docs (dim=%d)", len(s.docs), ds.Dim())
|
||||||
|
count := 0
|
||||||
|
for _, doc := range s.docs {
|
||||||
|
if doc.DenseVec != nil && len(doc.DenseVec) == ds.Dim() && doc.DenseFP == ds.Fingerprint() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
vec := s.denseFor(doc)
|
||||||
|
if vec == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
doc.DenseVec = vec
|
||||||
|
doc.DenseFP = ds.Fingerprint()
|
||||||
|
count++
|
||||||
|
}
|
||||||
|
log.Printf("[document memory] dense index built: %d new vectors", count)
|
||||||
|
}
|
||||||
|
|
||||||
|
// denseFor 计算文档的稠密向量:文本向量与其一等记忆块的媒体向量融合。
|
||||||
|
//
|
||||||
|
// 只有与当前统一空间同指纹的块向量才参与融合:不同模型/维度的旧向量
|
||||||
|
// 属于另一个坐标系,混进去会算出一个两边都不像的方向。
|
||||||
|
// 任意一路缺失时退化为另一路;都不可用返回 nil。
|
||||||
|
func (s *Store) denseFor(doc *Doc) []float64 {
|
||||||
|
if s.denseSpace == nil || !s.denseSpace.Loaded() {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
fp := s.denseSpace.Fingerprint()
|
||||||
|
var parts [][]float64
|
||||||
|
if tv, err := s.denseSpace.VectorizeDense(doc.Summary + " " + doc.Content); err == nil && len(tv) > 0 {
|
||||||
|
parts = append(parts, tv)
|
||||||
|
}
|
||||||
|
for _, b := range doc.Blocks {
|
||||||
|
if len(b.Vector) > 0 && b.Fingerprint == fp {
|
||||||
|
parts = append(parts, b.Vector)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return vector.FuseVectors(parts...)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Reindex 重建 TF-IDF 索引(fallback 路径变更时调用)。
|
||||||
|
func (s *Store) Reindex() {
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
s.tfidfOnce = sync.Once{} // 重置延迟初始化
|
||||||
|
texts := make(map[string]string, len(s.docs))
|
||||||
|
for _, doc := range s.docs {
|
||||||
|
texts[doc.ID] = doc.Summary + " " + doc.Content
|
||||||
|
}
|
||||||
|
// 缓存文本供 ensureTFIDF 延迟训练
|
||||||
|
s.trainTexts = make([]string, 0, len(texts)*2)
|
||||||
|
for id, t := range texts {
|
||||||
|
s.trainTexts = append(s.trainTexts, id, t)
|
||||||
|
}
|
||||||
|
s.ensureTFIDF()
|
||||||
}
|
}
|
||||||
|
|
||||||
// Insert 创建/更新文档
|
|
||||||
func (s *Store) Insert(doc *Doc) error {
|
func (s *Store) Insert(doc *Doc) error {
|
||||||
s.mu.Lock()
|
s.mu.Lock()
|
||||||
defer s.mu.Unlock()
|
defer s.mu.Unlock()
|
||||||
@ -107,36 +190,35 @@ func (s *Store) Insert(doc *Doc) error {
|
|||||||
if doc.AccessCount == 0 {
|
if doc.AccessCount == 0 {
|
||||||
doc.AccessCount = 1
|
doc.AccessCount = 1
|
||||||
}
|
}
|
||||||
|
|
||||||
s.docs[doc.ID] = doc
|
s.docs[doc.ID] = doc
|
||||||
|
|
||||||
// 增量训练向量化器并加入向量索引
|
text := doc.Summary + " " + doc.Content
|
||||||
s.addSummary(doc.Summary)
|
|
||||||
vec := doc.Vector
|
// 主路径:稠密向量(文本 ⊕ 媒体块)
|
||||||
if vec == nil {
|
if s.denseSpace != nil && s.denseSpace.Loaded() && len(doc.DenseVec) == 0 {
|
||||||
vec = s.veczer.Vectorize(doc.Summary + " " + doc.Content)
|
doc.DenseVec = s.denseFor(doc)
|
||||||
}
|
doc.DenseFP = s.denseSpace.Fingerprint()
|
||||||
s.vec.Insert(doc.ID, doc.Summary, vec, doc.Meta)
|
}
|
||||||
|
|
||||||
|
// Fallback 路径:缓存文本,延迟训练
|
||||||
|
if s.tfidfIdx != nil {
|
||||||
|
s.tfidfIdx.Add(doc.ID, text)
|
||||||
|
} else {
|
||||||
|
s.trainTexts = append(s.trainTexts, doc.ID, text)
|
||||||
|
}
|
||||||
|
|
||||||
// 立即写盘
|
|
||||||
path := filepath.Join(s.dir, doc.ID+".json")
|
path := filepath.Join(s.dir, doc.ID+".json")
|
||||||
data, _ := json.MarshalIndent(doc, "", " ")
|
data, _ := json.MarshalIndent(doc, "", " ")
|
||||||
os.WriteFile(path, data, 0644)
|
os.WriteFile(path, data, 0644)
|
||||||
|
|
||||||
s.dirty = true
|
s.dirty = true
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// ContextToDoc — 将一段上下文对话历史提炼为文档(带内容去重)
|
// ContextToDoc 将上下文对话历史提炼为文档。
|
||||||
// cleanFn 可选,在计算层前统一过滤文本,不影响原文存储。
|
func (s *Store) ContextToDoc(source string, entries []ContextEntry, _ interface{}, cleanFn func(string) string, toolCleanFn func(name, output string) string, channelCleaner ChannelCleaner) (*Doc, error) {
|
||||||
// toolCleanFn 可选,func(name, output string) string,按工具名对输出进行过滤/清洗:
|
|
||||||
// - 返回 "" → 跳过该工具输出(NoMemory)
|
|
||||||
// - 返回清洗后文本 → 用于计算层(Cleaner),原文不受影响
|
|
||||||
func (s *Store) ContextToDoc(source string, entries []ContextEntry, vec vector.Vectorizer, cleanFn func(string) string, toolCleanFn func(name, output string) string, channelCleaner ChannelCleaner) (*Doc, error) {
|
|
||||||
if len(entries) == 0 {
|
if len(entries) == 0 {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
if cleanFn == nil {
|
if cleanFn == nil {
|
||||||
cleanFn = func(text string) string { return text }
|
cleanFn = func(text string) string { return text }
|
||||||
}
|
}
|
||||||
@ -154,14 +236,13 @@ func (s *Store) ContextToDoc(source string, entries []ContextEntry, vec vector.V
|
|||||||
}
|
}
|
||||||
content := strings.Join(parts, "\n")
|
content := strings.Join(parts, "\n")
|
||||||
contentHash := simpleHash(content)
|
contentHash := simpleHash(content)
|
||||||
|
|
||||||
summary := summarizeEntries(entries, cleanFn, toolCleanFn, channelCleaner)
|
summary := summarizeEntries(entries, cleanFn, toolCleanFn, channelCleaner)
|
||||||
tags := extractTags(entries, cleanFn, toolCleanFn, channelCleaner)
|
tags := extractTags(entries, cleanFn, toolCleanFn, channelCleaner)
|
||||||
entities := extractEntities(entries, cleanFn, toolCleanFn, channelCleaner)
|
entities := extractEntities(entries, cleanFn, toolCleanFn, channelCleaner)
|
||||||
|
|
||||||
s.mu.Lock()
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
|
||||||
// 去重
|
|
||||||
for _, d := range s.docs {
|
for _, d := range s.docs {
|
||||||
if d.Meta != nil && d.Meta["content_hash"] == contentHash {
|
if d.Meta != nil && d.Meta["content_hash"] == contentHash {
|
||||||
d.UpdatedAt = time.Now()
|
d.UpdatedAt = time.Now()
|
||||||
@ -171,66 +252,73 @@ func (s *Store) ContextToDoc(source string, entries []ContextEntry, vec vector.V
|
|||||||
d.Summary = summary
|
d.Summary = summary
|
||||||
d.Tags = tags
|
d.Tags = tags
|
||||||
d.Entities = entities
|
d.Entities = entities
|
||||||
|
d.Blocks = blocksFromEntries(entries)
|
||||||
|
d.DenseVec = s.denseFor(d)
|
||||||
|
if s.denseSpace != nil {
|
||||||
|
d.DenseFP = s.denseSpace.Fingerprint()
|
||||||
|
}
|
||||||
s.dirty = true
|
s.dirty = true
|
||||||
s.mu.Unlock()
|
|
||||||
return d, nil
|
return d, nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
id := fmt.Sprintf("doc_%d", time.Now().UnixNano())
|
id := fmt.Sprintf("doc_%d", time.Now().UnixNano())
|
||||||
var docVec vector.Vector
|
meta := map[string]string{"content_hash": contentHash}
|
||||||
if vec != nil {
|
|
||||||
docVec = vec.Vectorize(summary + " " + content)
|
|
||||||
} else {
|
|
||||||
docVec = s.veczer.Vectorize(summary + " " + content)
|
|
||||||
}
|
|
||||||
meta := map[string]string{"content_hash": contentHash}
|
|
||||||
if source == "context_archived" {
|
if source == "context_archived" {
|
||||||
meta["is_archived_context"] = "true"
|
meta["is_archived_context"] = "true"
|
||||||
}
|
}
|
||||||
doc := &Doc{
|
doc := &Doc{
|
||||||
ID: id,
|
ID: id, Summary: summary, Content: content, Tags: tags,
|
||||||
Summary: summary,
|
Entities: entities, CreatedAt: time.Now(), UpdatedAt: time.Now(),
|
||||||
Content: content,
|
LastAccess: time.Now(), AccessCount: 1, Source: source, Meta: meta,
|
||||||
Tags: tags,
|
Blocks: blocksFromEntries(entries),
|
||||||
Entities: entities,
|
|
||||||
CreatedAt: time.Now(),
|
|
||||||
UpdatedAt: time.Now(),
|
|
||||||
LastAccess: time.Now(),
|
|
||||||
AccessCount: 1,
|
|
||||||
Source: source,
|
|
||||||
Meta: meta,
|
|
||||||
Vector: docVec,
|
|
||||||
}
|
}
|
||||||
s.docs[id] = doc
|
s.docs[id] = doc
|
||||||
|
doc.DenseVec = s.denseFor(doc)
|
||||||
|
if s.denseSpace != nil {
|
||||||
|
doc.DenseFP = s.denseSpace.Fingerprint()
|
||||||
|
}
|
||||||
|
text := summary + " " + content
|
||||||
|
if s.tfidfIdx != nil {
|
||||||
|
s.tfidfIdx.Add(id, text)
|
||||||
|
} else {
|
||||||
|
s.trainTexts = append(s.trainTexts, id, text)
|
||||||
|
}
|
||||||
|
|
||||||
// 加入向量索引
|
|
||||||
s.addSummary(summary)
|
|
||||||
s.vec.Insert(id, summary, doc.Vector, nil)
|
|
||||||
|
|
||||||
s.dirty = true
|
|
||||||
s.mu.Unlock()
|
|
||||||
|
|
||||||
// 立即写盘
|
|
||||||
path := filepath.Join(s.dir, id+".json")
|
path := filepath.Join(s.dir, id+".json")
|
||||||
data, _ := json.MarshalIndent(doc, "", " ")
|
data, _ := json.MarshalIndent(doc, "", " ")
|
||||||
os.WriteFile(path, data, 0644)
|
os.WriteFile(path, data, 0644)
|
||||||
|
s.dirty = true
|
||||||
return doc, nil
|
return doc, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Consume — 向量相似度查询并移除文档(召回后即从冷存储删除,避免重复记忆)
|
// Consume 向量相似度查询并移除文档
|
||||||
func (s *Store) Consume(text string, topK int) []*Doc {
|
func (s *Store) Consume(text string, topK int) []*Doc {
|
||||||
s.mu.Lock()
|
s.mu.Lock()
|
||||||
defer s.mu.Unlock()
|
defer s.mu.Unlock()
|
||||||
|
|
||||||
if topK <= 0 {
|
if topK <= 0 {
|
||||||
topK = 5
|
topK = 5
|
||||||
}
|
}
|
||||||
|
|
||||||
vec := s.vectorizeQuery(text)
|
// 主路径:稠密检索
|
||||||
results := s.vec.Search(vec, topK)
|
if s.denseSpace != nil && s.denseSpace.Loaded() {
|
||||||
|
if qv, err := s.denseSpace.VectorizeDense(text); err == nil {
|
||||||
|
results := s.denseSearchScored(qv, topK)
|
||||||
|
var docs []*Doc
|
||||||
|
for _, r := range results {
|
||||||
|
if d, ok := s.docs[r.Doc.ID]; ok {
|
||||||
|
s.removeDoc(r.Doc.ID)
|
||||||
|
s.dirty = true
|
||||||
|
docs = append(docs, d)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return docs
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fallback:TF-IDF 倒排检索(延迟初始化)
|
||||||
|
s.ensureTFIDF()
|
||||||
|
results := s.tfidfIdx.Search(text, topK)
|
||||||
var docs []*Doc
|
var docs []*Doc
|
||||||
for _, r := range results {
|
for _, r := range results {
|
||||||
if d, ok := s.docs[r.ID]; ok {
|
if d, ok := s.docs[r.ID]; ok {
|
||||||
@ -242,75 +330,124 @@ func (s *Store) Consume(text string, topK int) []*Doc {
|
|||||||
return docs
|
return docs
|
||||||
}
|
}
|
||||||
|
|
||||||
// vectorizeQuery 用语义向量化器(首选)或 TF-IDF(兜底)处理查询文本
|
func (s *Store) Query(text string, topK int) []*Doc {
|
||||||
func (s *Store) vectorizeQuery(text string) vector.Vector {
|
hits := s.QueryScored(text, topK)
|
||||||
if s.vectorizer != nil {
|
out := make([]*Doc, len(hits))
|
||||||
return s.vectorizer.Vectorize(text)
|
for i, h := range hits {
|
||||||
|
out[i] = h.Doc
|
||||||
}
|
}
|
||||||
return s.veczer.Vectorize(text)
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
// Query — 向量相似度查询文档
|
// DocHit 是一篇文档记忆的相似度候选及原始分数。
|
||||||
func (s *Store) Query(text string, topK int) []*Doc {
|
type DocHit struct {
|
||||||
|
Doc *Doc
|
||||||
|
Score float64
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Store) QueryScored(text string, topK int) []DocHit {
|
||||||
s.mu.RLock()
|
s.mu.RLock()
|
||||||
defer s.mu.RUnlock()
|
defer s.mu.RUnlock()
|
||||||
|
|
||||||
if topK <= 0 {
|
if topK <= 0 {
|
||||||
topK = 5
|
topK = 5
|
||||||
}
|
}
|
||||||
|
|
||||||
vec := s.vectorizeQuery(text)
|
// 主路径
|
||||||
results := s.vec.Search(vec, topK)
|
if s.denseSpace != nil && s.denseSpace.Loaded() {
|
||||||
|
if qv, err := s.denseSpace.VectorizeDense(text); err == nil {
|
||||||
|
results := s.denseSearchScored(qv, topK)
|
||||||
|
for i := range results {
|
||||||
|
if d, ok := s.docs[results[i].Doc.ID]; ok {
|
||||||
|
d.AccessCount++
|
||||||
|
d.LastAccess = time.Now()
|
||||||
|
results[i].Doc = d
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return results
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
var docs []*Doc
|
// Fallback(需要写锁来 ensureTFIDF)
|
||||||
|
s.mu.RUnlock()
|
||||||
|
s.mu.Lock()
|
||||||
|
s.ensureTFIDF()
|
||||||
|
s.mu.Unlock()
|
||||||
|
s.mu.RLock()
|
||||||
|
|
||||||
|
results := s.tfidfIdx.Search(text, topK)
|
||||||
|
var out []DocHit
|
||||||
for _, r := range results {
|
for _, r := range results {
|
||||||
if d, ok := s.docs[r.ID]; ok {
|
if d, ok := s.docs[r.ID]; ok {
|
||||||
d.AccessCount++
|
d.AccessCount++
|
||||||
d.LastAccess = time.Now()
|
d.LastAccess = time.Now()
|
||||||
docs = append(docs, d)
|
out = append(out, DocHit{Doc: d, Score: r.Score})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return docs
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
// Reindex — 重新训练并重建向量索引
|
func (s *Store) denseSearchScored(queryVec []float64, topK int) []DocHit {
|
||||||
func (s *Store) Reindex() {
|
if len(queryVec) == 0 {
|
||||||
s.mu.Lock()
|
return nil
|
||||||
defer s.mu.Unlock()
|
|
||||||
|
|
||||||
log.Printf("[document memory] reindexing %d docs", len(s.docs))
|
|
||||||
|
|
||||||
s.veczer.Train(s.summaries)
|
|
||||||
|
|
||||||
s.vec = vector.NewStore()
|
|
||||||
for _, doc := range s.docs {
|
|
||||||
vec := doc.Vector
|
|
||||||
if vec == nil {
|
|
||||||
vec = s.veczer.Vectorize(doc.Summary + " " + doc.Content)
|
|
||||||
}
|
|
||||||
s.vec.Insert(doc.ID, doc.Summary, vec, doc.Meta)
|
|
||||||
}
|
}
|
||||||
|
type scored struct {
|
||||||
|
did string
|
||||||
|
score float64
|
||||||
|
}
|
||||||
|
var results []scored
|
||||||
|
for _, doc := range s.docs {
|
||||||
|
if len(doc.DenseVec) != len(queryVec) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
score := denseCosine(queryVec, doc.DenseVec)
|
||||||
|
if score > 0.01 {
|
||||||
|
results = append(results, scored{doc.ID, score})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(results) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
sort.Slice(results, func(i, j int) bool { return results[i].score > results[j].score })
|
||||||
|
if len(results) > topK {
|
||||||
|
results = results[:topK]
|
||||||
|
}
|
||||||
|
out := make([]DocHit, len(results))
|
||||||
|
for i, r := range results {
|
||||||
|
out[i] = DocHit{Doc: s.docs[r.did], Score: r.score}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
log.Printf("[document memory] reindex complete (%d vectors)", s.vec.Size())
|
func denseCosine(a, b []float64) float64 {
|
||||||
|
var dot, na, nb float64
|
||||||
|
for i := range a {
|
||||||
|
dot += a[i] * b[i]
|
||||||
|
na += a[i] * a[i]
|
||||||
|
nb += b[i] * b[i]
|
||||||
|
}
|
||||||
|
if na == 0 || nb == 0 {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
return dot / math.Sqrt(na*nb)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Store) Stats() map[string]interface{} {
|
func (s *Store) Stats() map[string]interface{} {
|
||||||
s.mu.RLock()
|
s.mu.RLock()
|
||||||
defer s.mu.RUnlock()
|
defer s.mu.RUnlock()
|
||||||
|
idxSize := 0
|
||||||
|
if s.tfidfIdx != nil {
|
||||||
|
idxSize = s.tfidfIdx.Size()
|
||||||
|
}
|
||||||
return map[string]interface{}{
|
return map[string]interface{}{
|
||||||
"doc_count": len(s.docs),
|
"doc_count": len(s.docs),
|
||||||
"vector_count": s.vec.Size(),
|
"index_count": idxSize,
|
||||||
"summary_count": len(s.summaries),
|
"dir": s.dir,
|
||||||
"dir": s.dir,
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// FindColdDocs — 查找冷文档:超过 maxAge 未访问且访问次数 <= minAccess
|
|
||||||
func (s *Store) FindColdDocs(maxAge time.Duration, minAccess int) []*Doc {
|
func (s *Store) FindColdDocs(maxAge time.Duration, minAccess int) []*Doc {
|
||||||
s.mu.RLock()
|
s.mu.RLock()
|
||||||
defer s.mu.RUnlock()
|
defer s.mu.RUnlock()
|
||||||
|
|
||||||
cutoff := time.Now().Add(-maxAge)
|
cutoff := time.Now().Add(-maxAge)
|
||||||
var cold []*Doc
|
var cold []*Doc
|
||||||
for _, d := range s.docs {
|
for _, d := range s.docs {
|
||||||
@ -321,60 +458,53 @@ func (s *Store) FindColdDocs(maxAge time.Duration, minAccess int) []*Doc {
|
|||||||
return cold
|
return cold
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Get 返回指定文档(不存在时为 nil)。
|
||||||
|
func (s *Store) Get(id string) *Doc {
|
||||||
|
s.mu.RLock()
|
||||||
|
defer s.mu.RUnlock()
|
||||||
|
return s.docs[id]
|
||||||
|
}
|
||||||
|
|
||||||
|
// Blocks 返回全部文档持有的一等记忆块(供跨层存活判定)。
|
||||||
|
func (s *Store) Blocks() []memory.MemoryBlock {
|
||||||
|
s.mu.RLock()
|
||||||
|
defer s.mu.RUnlock()
|
||||||
|
var out []memory.MemoryBlock
|
||||||
|
for _, d := range s.docs {
|
||||||
|
out = append(out, d.Blocks...)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
func (s *Store) RecentDocs(n int) []*Doc {
|
func (s *Store) RecentDocs(n int) []*Doc {
|
||||||
s.mu.RLock()
|
s.mu.RLock()
|
||||||
defer s.mu.RUnlock()
|
defer s.mu.RUnlock()
|
||||||
|
|
||||||
var list []*Doc
|
var list []*Doc
|
||||||
for _, d := range s.docs {
|
for _, d := range s.docs {
|
||||||
list = append(list, d)
|
list = append(list, d)
|
||||||
}
|
}
|
||||||
sort.Slice(list, func(i, j int) bool {
|
sort.Slice(list, func(i, j int) bool { return list[i].CreatedAt.After(list[j].CreatedAt) })
|
||||||
return list[i].CreatedAt.After(list[j].CreatedAt)
|
|
||||||
})
|
|
||||||
if len(list) > n {
|
if len(list) > n {
|
||||||
list = list[:n]
|
list = list[:n]
|
||||||
}
|
}
|
||||||
return list
|
return list
|
||||||
}
|
}
|
||||||
|
|
||||||
// Remove 从文档存储中删除指定 ID 的文档
|
|
||||||
func (s *Store) Remove(id string) {
|
func (s *Store) Remove(id string) {
|
||||||
s.mu.Lock()
|
s.mu.Lock()
|
||||||
defer s.mu.Unlock()
|
defer s.mu.Unlock()
|
||||||
|
|
||||||
if _, ok := s.docs[id]; ok {
|
if _, ok := s.docs[id]; ok {
|
||||||
s.removeDoc(id)
|
s.removeDoc(id)
|
||||||
s.dirty = true
|
s.dirty = true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// ——— internal ———
|
|
||||||
|
|
||||||
// addSummary 添加一条摘要到训练集,超限时截断并触发重索引。
|
|
||||||
// 调用方必须已持有 s.mu 写锁。
|
|
||||||
func (s *Store) addSummary(summary string) {
|
|
||||||
s.summaries = append(s.summaries, summary)
|
|
||||||
if len(s.summaries) > maxSummaries {
|
|
||||||
n := maxSummaries / 2
|
|
||||||
copy(s.summaries, s.summaries[len(s.summaries)-n:])
|
|
||||||
s.summaries = s.summaries[:n]
|
|
||||||
s.veczer.Train(s.summaries)
|
|
||||||
s.vec = vector.NewStore()
|
|
||||||
for _, doc := range s.docs {
|
|
||||||
vec := s.veczer.Vectorize(doc.Summary + " " + doc.Content)
|
|
||||||
s.vec.Insert(doc.ID, doc.Summary, vec, nil)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// removeDoc 从内存索引和磁盘删除文档。
|
|
||||||
// 调用方必须已持有 s.mu 写锁。
|
|
||||||
func (s *Store) removeDoc(id string) {
|
func (s *Store) removeDoc(id string) {
|
||||||
delete(s.docs, id)
|
delete(s.docs, id)
|
||||||
s.vec.Remove(id)
|
if s.tfidfIdx != nil {
|
||||||
path := filepath.Join(s.dir, id+".json")
|
s.tfidfIdx.Remove(id)
|
||||||
os.Remove(path)
|
}
|
||||||
|
os.Remove(filepath.Join(s.dir, id+".json"))
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Store) loadAll() error {
|
func (s *Store) loadAll() error {
|
||||||
@ -382,63 +512,43 @@ func (s *Store) loadAll() error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, e := range entries {
|
for _, e := range entries {
|
||||||
if !strings.HasSuffix(e.Name(), ".json") || !strings.HasPrefix(e.Name(), "doc_") {
|
if !strings.HasSuffix(e.Name(), ".json") || !strings.HasPrefix(e.Name(), "doc_") {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
path := filepath.Join(s.dir, e.Name())
|
data, err := os.ReadFile(filepath.Join(s.dir, e.Name()))
|
||||||
data, err := os.ReadFile(path)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
var doc Doc
|
var doc Doc
|
||||||
if err := json.Unmarshal(data, &doc); err != nil {
|
if json.Unmarshal(data, &doc) != nil || doc.ID == "" {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
s.docs[doc.ID] = &doc
|
s.docs[doc.ID] = &doc
|
||||||
s.summaries = append(s.summaries, doc.Summary)
|
// 缓存文本,延迟训练(确保TFIDF在首次需要时才加载)
|
||||||
|
s.trainTexts = append(s.trainTexts, doc.ID, doc.Summary+" "+doc.Content)
|
||||||
}
|
}
|
||||||
|
|
||||||
// 训练向量化器
|
|
||||||
if len(s.summaries) > 0 {
|
|
||||||
s.veczer.Train(s.summaries)
|
|
||||||
}
|
|
||||||
|
|
||||||
// 重建向量索引
|
|
||||||
for _, doc := range s.docs {
|
|
||||||
vec := doc.Vector
|
|
||||||
if vec == nil {
|
|
||||||
vec = s.veczer.Vectorize(doc.Summary + " " + doc.Content)
|
|
||||||
}
|
|
||||||
s.vec.Insert(doc.ID, doc.Summary, vec, nil)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Store) flush() {
|
func (s *Store) flush() {
|
||||||
s.mu.Lock()
|
s.mu.Lock()
|
||||||
defer s.mu.Unlock()
|
defer s.mu.Unlock()
|
||||||
|
|
||||||
if !s.dirty {
|
if !s.dirty {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, doc := range s.docs {
|
for _, doc := range s.docs {
|
||||||
path := filepath.Join(s.dir, doc.ID+".json")
|
data, _ := json.MarshalIndent(doc, "", " ")
|
||||||
data, err := json.MarshalIndent(doc, "", " ")
|
os.WriteFile(filepath.Join(s.dir, doc.ID+".json"), data, 0644)
|
||||||
if err != nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
os.WriteFile(path, data, 0644)
|
|
||||||
}
|
}
|
||||||
s.dirty = false
|
s.dirty = false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ——— 内部工具函数(从上下文提炼文档所需)———
|
||||||
|
|
||||||
type ToolResultItem struct {
|
type ToolResultItem struct {
|
||||||
Name string
|
Name string `json:"name"`
|
||||||
Output string
|
Output string `json:"output"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type ContextEntry struct {
|
type ContextEntry struct {
|
||||||
@ -447,6 +557,23 @@ type ContextEntry struct {
|
|||||||
Content string
|
Content string
|
||||||
Response string
|
Response string
|
||||||
ToolResults []ToolResultItem
|
ToolResults []ToolResultItem
|
||||||
|
Blocks []memory.MemoryBlock // 一等记忆块随事件一起迁移到文档
|
||||||
|
}
|
||||||
|
|
||||||
|
func blocksFromEntries(entries []ContextEntry) []memory.MemoryBlock {
|
||||||
|
seen := make(map[string]bool)
|
||||||
|
var out []memory.MemoryBlock
|
||||||
|
for _, e := range entries {
|
||||||
|
for i := range e.Blocks {
|
||||||
|
b := e.Blocks[i]
|
||||||
|
if b.ID == "" || seen[b.ID] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seen[b.ID] = true
|
||||||
|
out = append(out, b)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
func summarizeEntries(entries []ContextEntry, cleanText func(string) string, toolCleanFn func(name, output string) string, channelCleaner ChannelCleaner) string {
|
func summarizeEntries(entries []ContextEntry, cleanText func(string) string, toolCleanFn func(name, output string) string, channelCleaner ChannelCleaner) string {
|
||||||
@ -478,14 +605,12 @@ func summarizeEntries(entries []ContextEntry, cleanText func(string) string, too
|
|||||||
topics = append(topics, toolWords...)
|
topics = append(topics, toolWords...)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
summary := fmt.Sprintf("来自 %d 个来源的 %d 条对话", len(sources), len(entries))
|
summary := fmt.Sprintf("来自 %d 个来源的 %d 条对话", len(sources), len(entries))
|
||||||
var srcList []string
|
var srcList []string
|
||||||
for s := range sources {
|
for s := range sources {
|
||||||
srcList = append(srcList, s)
|
srcList = append(srcList, s)
|
||||||
}
|
}
|
||||||
summary += " (" + strings.Join(srcList, ", ") + ")"
|
summary += " (" + strings.Join(srcList, ", ") + ")"
|
||||||
|
|
||||||
if len(topics) > 0 {
|
if len(topics) > 0 {
|
||||||
seen := make(map[string]bool)
|
seen := make(map[string]bool)
|
||||||
var uniq []string
|
var uniq []string
|
||||||
@ -500,7 +625,6 @@ func summarizeEntries(entries []ContextEntry, cleanText func(string) string, too
|
|||||||
}
|
}
|
||||||
summary += " 涉及: " + strings.Join(uniq, ", ")
|
summary += " 涉及: " + strings.Join(uniq, ", ")
|
||||||
}
|
}
|
||||||
|
|
||||||
return summary
|
return summary
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -573,25 +697,20 @@ func extractEntities(entries []ContextEntry, cleanText func(string) string, tool
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if len(entities) > 20 {
|
|
||||||
entities = entities[:20]
|
|
||||||
}
|
|
||||||
return entities
|
return entities
|
||||||
}
|
}
|
||||||
|
|
||||||
func truncate(s string, max int) string {
|
func truncate(s string, max int) string {
|
||||||
runes := []rune(s)
|
if len([]rune(s)) <= max {
|
||||||
if len(runes) > max {
|
return s
|
||||||
return string(runes[:max]) + "..."
|
|
||||||
}
|
}
|
||||||
return s
|
return string([]rune(s)[:max]) + "..."
|
||||||
}
|
}
|
||||||
|
|
||||||
func simpleHash(s string) string {
|
func simpleHash(s string) string {
|
||||||
// 简单的基于内容的哈希,用于去重
|
h := fmt.Sprintf("%x", len(s))
|
||||||
h := 0
|
for _, c := range s {
|
||||||
for _, r := range s {
|
h += fmt.Sprintf("%x", c)
|
||||||
h = h*31 + int(r)
|
|
||||||
}
|
}
|
||||||
return fmt.Sprintf("h%08x", h)
|
return h
|
||||||
}
|
}
|
||||||
|
|||||||
@ -15,7 +15,7 @@ func TestInsertAndQuery(t *testing.T) {
|
|||||||
}
|
}
|
||||||
defer os.RemoveAll(dir)
|
defer os.RemoveAll(dir)
|
||||||
|
|
||||||
s := NewStore(dir)
|
s := NewStore(dir, memory.TokenizeWords)
|
||||||
if err := s.Start(); err != nil {
|
if err := s.Start(); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@ -48,7 +48,7 @@ func TestQuery(t *testing.T) {
|
|||||||
}
|
}
|
||||||
defer os.RemoveAll(dir)
|
defer os.RemoveAll(dir)
|
||||||
|
|
||||||
s := NewStore(dir)
|
s := NewStore(dir, memory.TokenizeWords)
|
||||||
s.Start()
|
s.Start()
|
||||||
defer s.Stop()
|
defer s.Stop()
|
||||||
|
|
||||||
@ -73,7 +73,7 @@ func TestContextToDoc(t *testing.T) {
|
|||||||
}
|
}
|
||||||
defer os.RemoveAll(dir)
|
defer os.RemoveAll(dir)
|
||||||
|
|
||||||
s := NewStore(dir)
|
s := NewStore(dir, memory.TokenizeWords)
|
||||||
s.Start()
|
s.Start()
|
||||||
defer s.Stop()
|
defer s.Stop()
|
||||||
|
|
||||||
@ -104,7 +104,7 @@ func TestFindColdDocs(t *testing.T) {
|
|||||||
}
|
}
|
||||||
defer os.RemoveAll(dir)
|
defer os.RemoveAll(dir)
|
||||||
|
|
||||||
s := NewStore(dir)
|
s := NewStore(dir, memory.TokenizeWords)
|
||||||
s.Start()
|
s.Start()
|
||||||
defer s.Stop()
|
defer s.Stop()
|
||||||
|
|
||||||
@ -136,7 +136,7 @@ func TestRecentDocs(t *testing.T) {
|
|||||||
}
|
}
|
||||||
defer os.RemoveAll(dir)
|
defer os.RemoveAll(dir)
|
||||||
|
|
||||||
s := NewStore(dir)
|
s := NewStore(dir, memory.TokenizeWords)
|
||||||
s.Start()
|
s.Start()
|
||||||
defer s.Stop()
|
defer s.Stop()
|
||||||
|
|
||||||
@ -160,7 +160,7 @@ func TestReindex(t *testing.T) {
|
|||||||
}
|
}
|
||||||
defer os.RemoveAll(dir)
|
defer os.RemoveAll(dir)
|
||||||
|
|
||||||
s := NewStore(dir)
|
s := NewStore(dir, memory.TokenizeWords)
|
||||||
s.Start()
|
s.Start()
|
||||||
defer s.Stop()
|
defer s.Stop()
|
||||||
|
|
||||||
@ -211,7 +211,7 @@ func TestInsertEmptyDoc(t *testing.T) {
|
|||||||
}
|
}
|
||||||
defer os.RemoveAll(dir)
|
defer os.RemoveAll(dir)
|
||||||
|
|
||||||
s := NewStore(dir)
|
s := NewStore(dir, memory.TokenizeWords)
|
||||||
s.Start()
|
s.Start()
|
||||||
defer s.Stop()
|
defer s.Stop()
|
||||||
|
|
||||||
@ -232,13 +232,13 @@ func TestPersistence(t *testing.T) {
|
|||||||
defer os.RemoveAll(dir)
|
defer os.RemoveAll(dir)
|
||||||
|
|
||||||
// 写
|
// 写
|
||||||
s1 := NewStore(dir)
|
s1 := NewStore(dir, memory.TokenizeWords)
|
||||||
s1.Start()
|
s1.Start()
|
||||||
s1.Insert(&Doc{Summary: "持久化测试", Content: "应该被保存到磁盘", Source: "manual"})
|
s1.Insert(&Doc{Summary: "持久化测试", Content: "应该被保存到磁盘", Source: "manual"})
|
||||||
s1.Stop()
|
s1.Stop()
|
||||||
|
|
||||||
// 读
|
// 读
|
||||||
s2 := NewStore(dir)
|
s2 := NewStore(dir, memory.TokenizeWords)
|
||||||
s2.Start()
|
s2.Start()
|
||||||
defer s2.Stop()
|
defer s2.Stop()
|
||||||
|
|
||||||
@ -269,7 +269,7 @@ func TestFlushNoDirty(t *testing.T) {
|
|||||||
}
|
}
|
||||||
defer os.RemoveAll(dir)
|
defer os.RemoveAll(dir)
|
||||||
|
|
||||||
s := NewStore(dir)
|
s := NewStore(dir, memory.TokenizeWords)
|
||||||
s.Start()
|
s.Start()
|
||||||
|
|
||||||
// 不插任何文档,flush 不应报错
|
// 不插任何文档,flush 不应报错
|
||||||
@ -283,7 +283,7 @@ func TestRemove(t *testing.T) {
|
|||||||
}
|
}
|
||||||
defer os.RemoveAll(dir)
|
defer os.RemoveAll(dir)
|
||||||
|
|
||||||
s := NewStore(dir)
|
s := NewStore(dir, memory.TokenizeWords)
|
||||||
s.Start()
|
s.Start()
|
||||||
defer s.Stop()
|
defer s.Stop()
|
||||||
|
|
||||||
@ -329,7 +329,7 @@ func TestRemoveNonexistent(t *testing.T) {
|
|||||||
}
|
}
|
||||||
defer os.RemoveAll(dir)
|
defer os.RemoveAll(dir)
|
||||||
|
|
||||||
s := NewStore(dir)
|
s := NewStore(dir, memory.TokenizeWords)
|
||||||
s.Start()
|
s.Start()
|
||||||
defer s.Stop()
|
defer s.Stop()
|
||||||
|
|
||||||
@ -437,7 +437,7 @@ func TestContextToDocContentPreservesRawToolOutput(t *testing.T) {
|
|||||||
}
|
}
|
||||||
defer os.RemoveAll(dir)
|
defer os.RemoveAll(dir)
|
||||||
|
|
||||||
s := NewStore(dir)
|
s := NewStore(dir, memory.TokenizeWords)
|
||||||
s.Start()
|
s.Start()
|
||||||
defer s.Stop()
|
defer s.Stop()
|
||||||
|
|
||||||
|
|||||||
@ -2,6 +2,7 @@ package memory
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"database/sql"
|
"database/sql"
|
||||||
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
@ -37,13 +38,17 @@ type Relation struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type Triple struct {
|
type Triple struct {
|
||||||
Subject string `json:"subject"`
|
Subject string `json:"subject"`
|
||||||
Relation string `json:"relation"`
|
Relation string `json:"relation"`
|
||||||
Object string `json:"object"`
|
Object string `json:"object"`
|
||||||
Confidence float64 `json:"confidence,omitempty"`
|
Confidence float64 `json:"confidence,omitempty"`
|
||||||
SubjectType string `json:"subject_type,omitempty"`
|
SubjectType string `json:"subject_type,omitempty"`
|
||||||
ObjectType string `json:"object_type,omitempty"`
|
ObjectType string `json:"object_type,omitempty"`
|
||||||
SentenceText string `json:"sentence_text,omitempty"` // 原始句子文本,Commit时写入sentences表
|
SentenceText string `json:"sentence_text,omitempty"` // 原始句子文本,Commit时写入sentences表
|
||||||
|
// MediaDigests 是该三元组显式携带的媒体 digest(完整或前缀)。
|
||||||
|
// 媒体不再靠正文 marker 反解:结构化字段直接给出归属,
|
||||||
|
// 由调用方(core)把它变成 L3 一等块并与句子建立结构边。
|
||||||
|
MediaDigests []string `json:"media_digests,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type GraphDB struct {
|
type GraphDB struct {
|
||||||
@ -107,6 +112,41 @@ func (g *GraphDB) initSchema() error {
|
|||||||
FOREIGN KEY (target_id) REFERENCES entities(id),
|
FOREIGN KEY (target_id) REFERENCES entities(id),
|
||||||
UNIQUE(source_id, target_id, relation_type, session_id)
|
UNIQUE(source_id, target_id, relation_type, session_id)
|
||||||
)`,
|
)`,
|
||||||
|
`CREATE TABLE IF NOT EXISTS memory_blocks (
|
||||||
|
id TEXT PRIMARY KEY,
|
||||||
|
modality TEXT NOT NULL,
|
||||||
|
text_content TEXT DEFAULT '',
|
||||||
|
payload_digest TEXT DEFAULT '',
|
||||||
|
mime TEXT DEFAULT '',
|
||||||
|
size INTEGER DEFAULT 0,
|
||||||
|
width INTEGER DEFAULT 0,
|
||||||
|
height INTEGER DEFAULT 0,
|
||||||
|
vector TEXT DEFAULT '',
|
||||||
|
fingerprint TEXT DEFAULT '',
|
||||||
|
source TEXT DEFAULT '',
|
||||||
|
tool TEXT DEFAULT '',
|
||||||
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||||
|
)`,
|
||||||
|
`CREATE TABLE IF NOT EXISTS memory_block_edges (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
source_kind TEXT NOT NULL,
|
||||||
|
source_id TEXT NOT NULL,
|
||||||
|
target_kind TEXT NOT NULL,
|
||||||
|
target_id TEXT NOT NULL,
|
||||||
|
edge_type TEXT NOT NULL,
|
||||||
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
UNIQUE(source_kind, source_id, target_kind, target_id, edge_type)
|
||||||
|
)`,
|
||||||
|
`CREATE TABLE IF NOT EXISTS documents (
|
||||||
|
id TEXT PRIMARY KEY,
|
||||||
|
summary TEXT DEFAULT '',
|
||||||
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||||
|
)`,
|
||||||
|
`CREATE INDEX IF NOT EXISTS idx_memory_blocks_modality ON memory_blocks(modality)`,
|
||||||
|
`CREATE INDEX IF NOT EXISTS idx_memory_blocks_digest ON memory_blocks(payload_digest)`,
|
||||||
|
`CREATE INDEX IF NOT EXISTS idx_memory_block_edges_source ON memory_block_edges(source_kind, source_id)`,
|
||||||
|
`CREATE INDEX IF NOT EXISTS idx_memory_block_edges_target ON memory_block_edges(target_kind, target_id)`,
|
||||||
`CREATE INDEX IF NOT EXISTS idx_entity_name ON entities(name)`,
|
`CREATE INDEX IF NOT EXISTS idx_entity_name ON entities(name)`,
|
||||||
`CREATE INDEX IF NOT EXISTS idx_entity_type ON entities(type)`,
|
`CREATE INDEX IF NOT EXISTS idx_entity_type ON entities(type)`,
|
||||||
`CREATE INDEX IF NOT EXISTS idx_relation_source ON relations(source_id)`,
|
`CREATE INDEX IF NOT EXISTS idx_relation_source ON relations(source_id)`,
|
||||||
@ -192,13 +232,37 @@ func (g *GraphDB) migrateRelationUnique(tx *sql.Tx) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Commit 把三元组写入图库,返回新建的实体数与关系数。
|
||||||
func (g *GraphDB) Commit(triples []Triple, sessionID string, turnID int) (int, int, error) {
|
func (g *GraphDB) Commit(triples []Triple, sessionID string, turnID int) (int, int, error) {
|
||||||
|
_, ec, rc, err := g.commit(triples, sessionID, turnID, false)
|
||||||
|
return ec, rc, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// CommitWithMedia 与 Commit 相同,但额外返回每条句子文本对应的 sentences.id。
|
||||||
|
//
|
||||||
|
// 为何单独开一个方法而不改 Commit 的签名:Commit 有十个非测试调用点
|
||||||
|
// 加二十多个测试调用点,为了一个多数调用方都不需要的返回值去改全部签名
|
||||||
|
// 不划算。这里让 Commit 内部转调,两者共享同一份落库逻辑。
|
||||||
|
//
|
||||||
|
// 返回的 map 只包含本次真正写入了 sentences 表的句子。调用方据此把媒体
|
||||||
|
// 变成 L3 一等块,并以 sentence --contains--> block 边与句子相连;
|
||||||
|
// 关系行本身不持有媒体。
|
||||||
|
func (g *GraphDB) CommitWithMedia(triples []Triple, sessionID string, turnID int) (map[string]int64, int, int, error) {
|
||||||
|
return g.commit(triples, sessionID, turnID, true)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (g *GraphDB) commit(triples []Triple, sessionID string, turnID int, trackSentences bool) (map[string]int64, int, int, error) {
|
||||||
g.mu.Lock()
|
g.mu.Lock()
|
||||||
defer g.mu.Unlock()
|
defer g.mu.Unlock()
|
||||||
|
|
||||||
|
var sentenceIDs map[string]int64
|
||||||
|
if trackSentences {
|
||||||
|
sentenceIDs = make(map[string]int64)
|
||||||
|
}
|
||||||
|
|
||||||
tx, err := g.db.Begin()
|
tx, err := g.db.Begin()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, 0, err
|
return nil, 0, 0, err
|
||||||
}
|
}
|
||||||
defer tx.Rollback()
|
defer tx.Rollback()
|
||||||
|
|
||||||
@ -229,24 +293,24 @@ func (g *GraphDB) Commit(triples []Triple, sessionID string, turnID int) (int, i
|
|||||||
|
|
||||||
ec, err := g.upsertEntity(tx, t.Subject, subjType)
|
ec, err := g.upsertEntity(tx, t.Subject, subjType)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, 0, err
|
return nil, 0, 0, err
|
||||||
}
|
}
|
||||||
entitiesCreated += ec
|
entitiesCreated += ec
|
||||||
|
|
||||||
ec, err = g.upsertEntity(tx, t.Object, objType)
|
ec, err = g.upsertEntity(tx, t.Object, objType)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, 0, err
|
return nil, 0, 0, err
|
||||||
}
|
}
|
||||||
entitiesCreated += ec
|
entitiesCreated += ec
|
||||||
|
|
||||||
var sourceID, targetID int64
|
var sourceID, targetID int64
|
||||||
err = tx.QueryRow("SELECT id FROM entities WHERE name = ?", t.Subject).Scan(&sourceID)
|
err = tx.QueryRow("SELECT id FROM entities WHERE name = ?", t.Subject).Scan(&sourceID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, 0, fmt.Errorf("subject %q: %w", t.Subject, err)
|
return nil, 0, 0, fmt.Errorf("subject %q: %w", t.Subject, err)
|
||||||
}
|
}
|
||||||
err = tx.QueryRow("SELECT id FROM entities WHERE name = ?", t.Object).Scan(&targetID)
|
err = tx.QueryRow("SELECT id FROM entities WHERE name = ?", t.Object).Scan(&targetID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, 0, fmt.Errorf("object %q: %w", t.Object, err)
|
return nil, 0, 0, fmt.Errorf("object %q: %w", t.Object, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// 写入/查找句子
|
// 写入/查找句子
|
||||||
@ -255,11 +319,13 @@ func (g *GraphDB) Commit(triples []Triple, sessionID string, turnID int) (int, i
|
|||||||
_, err = tx.Exec(
|
_, err = tx.Exec(
|
||||||
`INSERT OR IGNORE INTO sentences (text) VALUES (?)`, t.SentenceText)
|
`INSERT OR IGNORE INTO sentences (text) VALUES (?)`, t.SentenceText)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, 0, fmt.Errorf("insert sentence: %w", err)
|
return nil, 0, 0, fmt.Errorf("insert sentence: %w", err)
|
||||||
}
|
}
|
||||||
err = tx.QueryRow("SELECT id FROM sentences WHERE text = ?", t.SentenceText).Scan(&sentenceID)
|
err = tx.QueryRow("SELECT id FROM sentences WHERE text = ?", t.SentenceText).Scan(&sentenceID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
sentenceID = 0
|
sentenceID = 0
|
||||||
|
} else if sentenceIDs != nil {
|
||||||
|
sentenceIDs[t.SentenceText] = sentenceID
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -275,11 +341,11 @@ func (g *GraphDB) Commit(triples []Triple, sessionID string, turnID int) (int, i
|
|||||||
sourceID, targetID, t.Relation, confidence, sessionID, turnID, dateBucket, sentenceID,
|
sourceID, targetID, t.Relation, confidence, sessionID, turnID, dateBucket, sentenceID,
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, 0, err
|
return nil, 0, 0, err
|
||||||
}
|
}
|
||||||
relationsCreated++
|
relationsCreated++
|
||||||
} else if err != nil {
|
} else if err != nil {
|
||||||
return 0, 0, err
|
return nil, 0, 0, err
|
||||||
} else {
|
} else {
|
||||||
// 同一(会话内)三元组已存在:仅刷新置信度与时间戳,不重复计数
|
// 同一(会话内)三元组已存在:仅刷新置信度与时间戳,不重复计数
|
||||||
_, err = tx.Exec(
|
_, err = tx.Exec(
|
||||||
@ -288,16 +354,16 @@ func (g *GraphDB) Commit(triples []Triple, sessionID string, turnID int) (int, i
|
|||||||
confidence, sourceID, targetID, t.Relation, sessionID,
|
confidence, sourceID, targetID, t.Relation, sessionID,
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, 0, err
|
return nil, 0, 0, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := tx.Commit(); err != nil {
|
if err := tx.Commit(); err != nil {
|
||||||
return 0, 0, err
|
return nil, 0, 0, err
|
||||||
}
|
}
|
||||||
|
|
||||||
return entitiesCreated, relationsCreated, nil
|
return sentenceIDs, entitiesCreated, relationsCreated, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func validEntityName(name string) bool {
|
func validEntityName(name string) bool {
|
||||||
@ -688,9 +754,58 @@ func (g *GraphDB) GraphData() (map[string]interface{}, error) {
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
brows, err := g.db.Query(`SELECT id, modality, text_content, payload_digest, mime,
|
||||||
|
size, width, height, vector, fingerprint, source, tool, created_at, updated_at
|
||||||
|
FROM memory_blocks ORDER BY created_at, id`)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer brows.Close()
|
||||||
|
var blocks []MemoryBlock
|
||||||
|
for brows.Next() {
|
||||||
|
var block MemoryBlock
|
||||||
|
var vectorJSON string
|
||||||
|
if err := brows.Scan(&block.ID, &block.Modality, &block.Text, &block.PayloadDigest,
|
||||||
|
&block.MIME, &block.Size, &block.Width, &block.Height, &vectorJSON,
|
||||||
|
&block.Fingerprint, &block.Source, &block.Tool, &block.CreatedAt,
|
||||||
|
&block.UpdatedAt); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if vectorJSON != "" && vectorJSON != "null" {
|
||||||
|
if err := json.Unmarshal([]byte(vectorJSON), &block.Vector); err != nil {
|
||||||
|
return nil, fmt.Errorf("decode memory block %s vector: %w", block.ID, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
blocks = append(blocks, block)
|
||||||
|
}
|
||||||
|
if err := brows.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
berows, err := g.db.Query(`SELECT id, source_kind, source_id, target_kind, target_id,
|
||||||
|
edge_type, created_at FROM memory_block_edges ORDER BY id`)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer berows.Close()
|
||||||
|
var blockEdges []MemoryBlockEdge
|
||||||
|
for berows.Next() {
|
||||||
|
var edge MemoryBlockEdge
|
||||||
|
if err := berows.Scan(&edge.ID, &edge.SourceKind, &edge.SourceID,
|
||||||
|
&edge.TargetKind, &edge.TargetID, &edge.Type, &edge.CreatedAt); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
blockEdges = append(blockEdges, edge)
|
||||||
|
}
|
||||||
|
if err := berows.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
return map[string]interface{}{
|
return map[string]interface{}{
|
||||||
"nodes": entities,
|
"nodes": entities,
|
||||||
"edges": relations,
|
"edges": relations,
|
||||||
|
"memory_blocks": blocks,
|
||||||
|
"memory_block_edges": blockEdges,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -873,17 +988,42 @@ func (g *GraphDB) ClearSentenceID(relationID int64) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
// CleanupOrphanedSentences 删除没有任何关系引用的句子,返回删除数
|
// CleanupOrphanedSentences 删除既无关系引用、也无媒体块边的句子,返回删除数。
|
||||||
|
//
|
||||||
|
// 两个条件都必须看:旧媒体实体被迁移成原生块后,那些句子可能只靠
|
||||||
|
// sentence --contains--> block 存活,若只看 relations 引用就会被误删,
|
||||||
|
// 连带把块边变成悬空引用。
|
||||||
func (g *GraphDB) CleanupOrphanedSentences() (int, error) {
|
func (g *GraphDB) CleanupOrphanedSentences() (int, error) {
|
||||||
result, err := g.db.Exec(
|
g.mu.Lock()
|
||||||
`DELETE FROM sentences WHERE id NOT IN (
|
defer g.mu.Unlock()
|
||||||
SELECT DISTINCT sentence_id FROM relations WHERE sentence_id != 0
|
tx, err := g.db.Begin()
|
||||||
)`,
|
|
||||||
)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, err
|
return 0, err
|
||||||
}
|
}
|
||||||
n, _ := result.RowsAffected()
|
defer tx.Rollback()
|
||||||
|
|
||||||
|
// 先清掉指向将被删除句子的块边,避免留下悬空端点。
|
||||||
|
if _, err := tx.Exec(`DELETE FROM memory_block_edges
|
||||||
|
WHERE source_kind = 'sentence' AND source_id NOT IN (
|
||||||
|
SELECT CAST(id AS TEXT) FROM sentences
|
||||||
|
WHERE id IN (SELECT DISTINCT sentence_id FROM relations WHERE sentence_id != 0)
|
||||||
|
OR id IN (SELECT CAST(source_id AS INTEGER) FROM memory_block_edges WHERE source_kind = 'sentence')
|
||||||
|
)`); err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
|
||||||
|
res, err := tx.Exec(`DELETE FROM sentences WHERE id NOT IN (
|
||||||
|
SELECT DISTINCT sentence_id FROM relations WHERE sentence_id != 0
|
||||||
|
) AND id NOT IN (
|
||||||
|
SELECT CAST(source_id AS INTEGER) FROM memory_block_edges WHERE source_kind = 'sentence'
|
||||||
|
)`)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
n, _ := res.RowsAffected()
|
||||||
|
if err := tx.Commit(); err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
return int(n), nil
|
return int(n), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -2,8 +2,8 @@ package memory
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"database/sql"
|
"database/sql"
|
||||||
"path/filepath"
|
|
||||||
"os"
|
"os"
|
||||||
|
"path/filepath"
|
||||||
"testing"
|
"testing"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -434,6 +434,84 @@ func TestMergeEntitiesNonexistent(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestMemoryBlocksAreFirstClassGraphNodes(t *testing.T) {
|
||||||
|
g := newTestGraph(t)
|
||||||
|
defer os.Remove(g.dbPath)
|
||||||
|
defer g.Close()
|
||||||
|
|
||||||
|
image := MemoryBlock{
|
||||||
|
ID: "block_image_1", Modality: BlockImage,
|
||||||
|
PayloadDigest: "0123456789abcdef", MIME: "image/png", Size: 1234,
|
||||||
|
Width: 768, Height: 512, Vector: []float64{0.1, 0.2, 0.3},
|
||||||
|
Fingerprint: "qwen:test", Source: "qq", Tool: "upload",
|
||||||
|
}
|
||||||
|
text := MemoryBlock{ID: "block_text_1", Modality: BlockText, Text: "用户上传了一张架构图"}
|
||||||
|
if err := g.PutMemoryBlocks([]MemoryBlock{image, text}); err != nil {
|
||||||
|
t.Fatalf("PutMemoryBlocks: %v", err)
|
||||||
|
}
|
||||||
|
if err := g.AddMemoryBlockEdge("block", text.ID, "block", image.ID, "contains"); err != nil {
|
||||||
|
t.Fatalf("AddMemoryBlockEdge: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
blocks, err := g.MemoryBlocks()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(blocks) != 2 {
|
||||||
|
t.Fatalf("memory blocks=%d, want 2", len(blocks))
|
||||||
|
}
|
||||||
|
var gotImage *MemoryBlock
|
||||||
|
for i := range blocks {
|
||||||
|
if blocks[i].ID == image.ID {
|
||||||
|
gotImage = &blocks[i]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if gotImage == nil || gotImage.Modality != BlockImage || gotImage.PayloadDigest != image.PayloadDigest || gotImage.Fingerprint != image.Fingerprint || len(gotImage.Vector) != 3 {
|
||||||
|
t.Fatalf("image block not round-tripped: %+v", gotImage)
|
||||||
|
}
|
||||||
|
|
||||||
|
edges, err := g.MemoryBlockEdges()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(edges) != 1 || edges[0].Type != "contains" || edges[0].SourceID != text.ID || edges[0].TargetID != image.ID {
|
||||||
|
t.Fatalf("memory block edges=%+v", edges)
|
||||||
|
}
|
||||||
|
|
||||||
|
graph, err := g.GraphData()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
graphBlocks, ok := graph["memory_blocks"].([]MemoryBlock)
|
||||||
|
if !ok || len(graphBlocks) != 2 {
|
||||||
|
t.Fatalf("GraphData memory_blocks=%T %+v", graph["memory_blocks"], graph["memory_blocks"])
|
||||||
|
}
|
||||||
|
graphEdges, ok := graph["memory_block_edges"].([]MemoryBlockEdge)
|
||||||
|
if !ok || len(graphEdges) != 1 {
|
||||||
|
t.Fatalf("GraphData memory_block_edges=%T %+v", graph["memory_block_edges"], graph["memory_block_edges"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMemoryBlockEdgeRejectsMissingEndpoint(t *testing.T) {
|
||||||
|
g := newTestGraph(t)
|
||||||
|
defer os.Remove(g.dbPath)
|
||||||
|
defer g.Close()
|
||||||
|
|
||||||
|
if err := g.PutMemoryBlocks([]MemoryBlock{{ID: "known", Modality: BlockText, Text: "known"}}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := g.AddMemoryBlockEdge("block", "known", "block", "missing", "derived_from"); err == nil {
|
||||||
|
t.Fatal("edge to missing node must fail")
|
||||||
|
}
|
||||||
|
edges, err := g.MemoryBlockEdges()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(edges) != 0 {
|
||||||
|
t.Fatalf("failed transaction left edges: %+v", edges)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestPlaceholders(t *testing.T) {
|
func TestPlaceholders(t *testing.T) {
|
||||||
if placeholders(0) != "NULL" {
|
if placeholders(0) != "NULL" {
|
||||||
t.Errorf("expected NULL for n=0, got %s", placeholders(0))
|
t.Errorf("expected NULL for n=0, got %s", placeholders(0))
|
||||||
|
|||||||
@ -10,19 +10,19 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
type Indexer struct {
|
type Indexer struct {
|
||||||
db *GraphDB
|
db *GraphDB
|
||||||
vec *vector.Store
|
vec *vector.Store
|
||||||
veczer *vector.TFIDFVectorizer
|
veczer *vector.TFIDFVectorizer
|
||||||
mu sync.RWMutex
|
mu sync.RWMutex
|
||||||
trained bool
|
trained bool
|
||||||
recalled map[string]bool // 已通过工具调用显式召回的实体名,自动注入时跳过
|
recalled map[string]bool // 已通过工具调用显式召回的实体名,自动注入时跳过
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewIndexer(db *GraphDB) *Indexer {
|
func NewIndexer(db *GraphDB) *Indexer {
|
||||||
return &Indexer{
|
return &Indexer{
|
||||||
db: db,
|
db: db,
|
||||||
vec: vector.NewStore(),
|
vec: vector.NewStore(),
|
||||||
veczer: vector.NewTFIDFVectorizer(TokenizeWords),
|
veczer: vector.NewTFIDFVectorizer(TokenizeWords),
|
||||||
recalled: make(map[string]bool),
|
recalled: make(map[string]bool),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -174,7 +174,9 @@ func (idx *Indexer) BuildToolPrompt() string {
|
|||||||
### memory_commit
|
### memory_commit
|
||||||
将三元组写入图记忆。
|
将三元组写入图记忆。
|
||||||
参数:
|
参数:
|
||||||
- triples: [{"subject": "实体名", "relation": "关系类型", "object": "目标实体"}]
|
- triples: [{"subject": "实体名", "relation": "关系类型", "object": "目标实体",
|
||||||
|
"sentence_text": "原始句子(可选)", "media_digests": ["图片digest(可选)"]}]
|
||||||
|
填了 media_digests,日后从这条记忆就能取回当时那张图/那段音频。
|
||||||
|
|
||||||
### memory_introspect
|
### memory_introspect
|
||||||
查看记忆统计信息。
|
查看记忆统计信息。
|
||||||
@ -261,6 +263,15 @@ func (idx *Indexer) GetToolDefinitions() []map[string]interface{} {
|
|||||||
"subject": map[string]interface{}{"type": "string"},
|
"subject": map[string]interface{}{"type": "string"},
|
||||||
"relation": map[string]interface{}{"type": "string"},
|
"relation": map[string]interface{}{"type": "string"},
|
||||||
"object": map[string]interface{}{"type": "string"},
|
"object": map[string]interface{}{"type": "string"},
|
||||||
|
"sentence_text": map[string]interface{}{
|
||||||
|
"type": "string",
|
||||||
|
"description": "可选:这条三元组的原始句子。填了才能日后从图谱回到原文。",
|
||||||
|
},
|
||||||
|
"media_digests": map[string]interface{}{
|
||||||
|
"type": "array",
|
||||||
|
"description": "可选:这条记忆关联的媒体 digest(对话或 memory_recall 的「关联媒体」里显示的十六进制串,短的即可)。填了以后从这条记忆能取回原图/音频。",
|
||||||
|
"items": map[string]interface{}{"type": "string"},
|
||||||
|
},
|
||||||
},
|
},
|
||||||
"required": []string{"subject", "relation", "object"},
|
"required": []string{"subject", "relation", "object"},
|
||||||
},
|
},
|
||||||
|
|||||||
123
internal/memory/jieba_embed.go
Normal file
123
internal/memory/jieba_embed.go
Normal file
@ -0,0 +1,123 @@
|
|||||||
|
// Package memory 的 jieba 词库内嵌。
|
||||||
|
//
|
||||||
|
// 为什么要把词库嵌进二进制,而不是像以前那样去猜 Go 模块缓存路径:
|
||||||
|
//
|
||||||
|
// 原实现是 `jiebaDictDir()` 依次试 GOMODCACHE / GOPATH / ~/go/pkg/mod,去找
|
||||||
|
// `github.com/yanyiwu/gojieba@v1.4.7/deps/cppjieba/dict`。部署机上通常**没有**
|
||||||
|
// Go 模块缓存,于是返回 "",`GetJieba()` 返回 nil,四个分词/关键词函数
|
||||||
|
// **一律静默返回空列表**(只在首次打一行「jieba disabled」)。
|
||||||
|
//
|
||||||
|
// 后果不是「少了个优化」而是**能力整体消失**:图记忆的关键词提取、文档
|
||||||
|
// TF-IDF 分词、NLP 依存解析(进而 doc→graph 三元组抽取)全部退化为空。
|
||||||
|
// 而本机之所以看起来正常,只是因为开发机与生产机重合、恰好有那份模块缓存。
|
||||||
|
//
|
||||||
|
// 内嵌后词库成为产物的一部分:与二进制同版本、随二进制分发、不依赖宿主环境。
|
||||||
|
// 代价是包体大 ~11.6MB(jieba.dict.utf8 5.1M + idf.utf8 6.0M + hmm_model 0.5M + …),
|
||||||
|
// 这是可接受的——它换来的是「装到哪都能用」。
|
||||||
|
//
|
||||||
|
// 关于 POS:gojieba 的 Tag() 不读 `pos_dict/` 目录,而是从主词典每行的
|
||||||
|
// 词性列取 tag(cppjieba 的 PosTagger::LookupTag 走 dict->Find(...)->tag),
|
||||||
|
// 取不到时用 SpecialRule 按字符类型兜底。所以这 5 个文件已足够同时支撑
|
||||||
|
// Cut 与 Tag,无需再嵌 pos_dict/。
|
||||||
|
package memory
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/sha256"
|
||||||
|
"embed"
|
||||||
|
"encoding/hex"
|
||||||
|
"fmt"
|
||||||
|
"io/fs"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"sort"
|
||||||
|
)
|
||||||
|
|
||||||
|
//go:embed jiebadict/*
|
||||||
|
var jiebaDictFS embed.FS
|
||||||
|
|
||||||
|
// jiebaDictFiles 是 gojieba.NewJieba 需要的 5 个文件,顺序与它的参数一致:
|
||||||
|
// dict, hmm, user, idf, stop_words。
|
||||||
|
var jiebaDictFiles = []string{
|
||||||
|
"jieba.dict.utf8",
|
||||||
|
"hmm_model.utf8",
|
||||||
|
"user.dict.utf8",
|
||||||
|
"idf.utf8",
|
||||||
|
"stop_words.utf8",
|
||||||
|
}
|
||||||
|
|
||||||
|
// materializeJiebaDict 把内嵌词库落盘,返回目录路径。
|
||||||
|
//
|
||||||
|
// gojieba 的 C++ API 只接受**文件路径**(NewJieba 会对每个路径 os.Stat,
|
||||||
|
// 缺失就 panic),所以必须先落盘再传路径。
|
||||||
|
//
|
||||||
|
// 落盘位置与幂等性:
|
||||||
|
// - 用内容哈希命名目录:词库升级后不会复用旧文件(否则会出现「新旧词库混用」
|
||||||
|
// 这种最难查的一类问题——分词结果与版本对不上)。
|
||||||
|
// - 已存在且大小一致就跳过写入:正常启动只做几次 stat。
|
||||||
|
func materializeJiebaDict() (string, error) {
|
||||||
|
sum, err := jiebaDictDigest()
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
base, err := os.UserCacheDir()
|
||||||
|
if err != nil || base == "" {
|
||||||
|
base = os.TempDir()
|
||||||
|
}
|
||||||
|
dir := filepath.Join(base, "homeagent", "jieba-"+sum)
|
||||||
|
|
||||||
|
if jiebaDictComplete(dir) {
|
||||||
|
return dir, nil
|
||||||
|
}
|
||||||
|
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||||
|
return "", fmt.Errorf("创建词库目录: %w", err)
|
||||||
|
}
|
||||||
|
for _, name := range jiebaDictFiles {
|
||||||
|
data, err := jiebaDictFS.ReadFile("jiebadict/" + name)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("读取内嵌词库 %s: %w", name, err)
|
||||||
|
}
|
||||||
|
path := filepath.Join(dir, name)
|
||||||
|
// 先写临时文件再 rename:避免并发启动时读到写了一半的词库。
|
||||||
|
tmp := path + ".tmp"
|
||||||
|
if err := os.WriteFile(tmp, data, 0o644); err != nil {
|
||||||
|
return "", fmt.Errorf("写出词库 %s: %w", name, err)
|
||||||
|
}
|
||||||
|
if err := os.Rename(tmp, path); err != nil {
|
||||||
|
return "", fmt.Errorf("落位词库 %s: %w", name, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return dir, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// jiebaDictDigest 对全部内嵌词库内容求哈希,作为落盘目录名的一部分。
|
||||||
|
func jiebaDictDigest() (string, error) {
|
||||||
|
h := sha256.New()
|
||||||
|
// 按固定顺序喂入:embed.FS 的遍历顺序不保证稳定,顺序变了哈希就变,
|
||||||
|
// 会导致每次启动都重建一份词库。
|
||||||
|
names := append([]string(nil), jiebaDictFiles...)
|
||||||
|
sort.Strings(names)
|
||||||
|
for _, name := range names {
|
||||||
|
data, err := jiebaDictFS.ReadFile("jiebadict/" + name)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("读取内嵌词库 %s: %w", name, err)
|
||||||
|
}
|
||||||
|
fmt.Fprintf(h, "%s:%d:", name, len(data))
|
||||||
|
h.Write(data)
|
||||||
|
}
|
||||||
|
return hex.EncodeToString(h.Sum(nil))[:16], nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// jiebaDictComplete 判断目录下 5 个词库是否齐全且大小与内嵌版本一致。
|
||||||
|
func jiebaDictComplete(dir string) bool {
|
||||||
|
for _, name := range jiebaDictFiles {
|
||||||
|
want, err := fs.Stat(jiebaDictFS, "jiebadict/"+name)
|
||||||
|
if err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
got, err := os.Stat(filepath.Join(dir, name))
|
||||||
|
if err != nil || got.Size() != want.Size() {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
98
internal/memory/jieba_embed_test.go
Normal file
98
internal/memory/jieba_embed_test.go
Normal file
@ -0,0 +1,98 @@
|
|||||||
|
package memory
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// 内嵌词库必须自足:把 GOMODCACHE/GOPATH/HOME 全部指向不存在的路径,
|
||||||
|
// 分词与关键词提取仍然要工作。
|
||||||
|
//
|
||||||
|
// 这正是修复前的故障场景:原实现只去猜 Go 模块缓存,部署机上没有那份缓存时
|
||||||
|
// GetJieba() 返回 nil,四个函数静默返回空列表——关键词提取、文档 TF-IDF 分词、
|
||||||
|
// NLP 依存解析(进而 doc→graph 三元组抽取)一起失效,且只有一行日志。
|
||||||
|
func TestEmbeddedJiebaDictIsSelfContained(t *testing.T) {
|
||||||
|
t.Setenv("GOMODCACHE", filepath.Join(t.TempDir(), "nonexistent"))
|
||||||
|
t.Setenv("GOPATH", filepath.Join(t.TempDir(), "nonexistent"))
|
||||||
|
t.Setenv("HOME", filepath.Join(t.TempDir(), "nonexistent"))
|
||||||
|
// 清掉可能已被其它测试初始化过的单例。
|
||||||
|
jiebaOnce = sync.Once{}
|
||||||
|
jiebaInst = nil
|
||||||
|
t.Cleanup(func() {
|
||||||
|
jiebaOnce = sync.Once{}
|
||||||
|
jiebaInst = nil
|
||||||
|
})
|
||||||
|
|
||||||
|
if x := GetJieba(); x == nil {
|
||||||
|
t.Fatal("模块缓存不可见时 jieba 必须仍能初始化(词库应来自内嵌副本)")
|
||||||
|
}
|
||||||
|
|
||||||
|
words := TokenizeWords("今天天气很好,我们去公园散步")
|
||||||
|
if len(words) == 0 {
|
||||||
|
t.Fatal("分词结果为空:内嵌词库没有真正生效")
|
||||||
|
}
|
||||||
|
t.Logf("分词结果: %v", words)
|
||||||
|
|
||||||
|
// 内容词(名词/动词/形容词)——依赖词典里的词性列,顺便验证 Tag 路径可用。
|
||||||
|
content := TokenizeContentWords("北京是中国的首都,这里有很多历史建筑")
|
||||||
|
if len(content) == 0 {
|
||||||
|
t.Fatal("内容词为空:Tag(词性标注)路径失效")
|
||||||
|
}
|
||||||
|
t.Logf("内容词: %v", content)
|
||||||
|
|
||||||
|
if kw := ExtractKeywords("机器学习模型训练需要大量数据和算力"); len(kw) == 0 {
|
||||||
|
t.Fatal("关键词为空")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 落盘目录必须幂等:第二次调用不应重写文件(正常启动只做几次 stat)。
|
||||||
|
func TestMaterializeJiebaDictIsIdempotent(t *testing.T) {
|
||||||
|
dir, err := materializeJiebaDict()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("materializeJiebaDict: %v", err)
|
||||||
|
}
|
||||||
|
for _, name := range jiebaDictFiles {
|
||||||
|
p := filepath.Join(dir, name)
|
||||||
|
fi, err := os.Stat(p)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("缺少词库文件 %s: %v", name, err)
|
||||||
|
}
|
||||||
|
if fi.Size() == 0 {
|
||||||
|
t.Fatalf("词库文件 %s 为空", name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 记下 mtime,再调一次,必须完全没动过。
|
||||||
|
before, _ := os.Stat(filepath.Join(dir, "jieba.dict.utf8"))
|
||||||
|
dir2, err := materializeJiebaDict()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if dir2 != dir {
|
||||||
|
t.Fatalf("目录名不稳定:%s vs %s(会导致每次启动重建词库)", dir, dir2)
|
||||||
|
}
|
||||||
|
after, _ := os.Stat(filepath.Join(dir, "jieba.dict.utf8"))
|
||||||
|
if !before.ModTime().Equal(after.ModTime()) {
|
||||||
|
t.Fatal("已存在完整词库时不应重写文件")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 词库内容哈希必须稳定:否则目录名每次都变,等于每次启动都重建。
|
||||||
|
func TestJiebaDictDigestStable(t *testing.T) {
|
||||||
|
a, err := jiebaDictDigest()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
b, err := jiebaDictDigest()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if a != b {
|
||||||
|
t.Fatalf("哈希不稳定: %s vs %s", a, b)
|
||||||
|
}
|
||||||
|
if len(a) != 16 {
|
||||||
|
t.Fatalf("哈希长度异常: %q", a)
|
||||||
|
}
|
||||||
|
}
|
||||||
34
internal/memory/jiebadict/hmm_model.utf8
Normal file
34
internal/memory/jiebadict/hmm_model.utf8
Normal file
File diff suppressed because one or more lines are too long
258826
internal/memory/jiebadict/idf.utf8
Normal file
258826
internal/memory/jiebadict/idf.utf8
Normal file
File diff suppressed because it is too large
Load Diff
348982
internal/memory/jiebadict/jieba.dict.utf8
Normal file
348982
internal/memory/jiebadict/jieba.dict.utf8
Normal file
File diff suppressed because it is too large
Load Diff
1534
internal/memory/jiebadict/stop_words.utf8
Normal file
1534
internal/memory/jiebadict/stop_words.utf8
Normal file
File diff suppressed because it is too large
Load Diff
4
internal/memory/jiebadict/user.dict.utf8
Normal file
4
internal/memory/jiebadict/user.dict.utf8
Normal file
@ -0,0 +1,4 @@
|
|||||||
|
云计算
|
||||||
|
韩玉鉴赏
|
||||||
|
蓝翔 nz
|
||||||
|
区块链 10 nz
|
||||||
14
internal/memory/media/base64.go
Normal file
14
internal/memory/media/base64.go
Normal file
@ -0,0 +1,14 @@
|
|||||||
|
package media
|
||||||
|
|
||||||
|
import "encoding/base64"
|
||||||
|
|
||||||
|
// base64 编解码单独抽出来,让 media.go 的 import 块只留业务依赖。
|
||||||
|
// 用 StdEncoding:data URL 规范用的是标准表(含 + / =),不是 URL-safe 表。
|
||||||
|
|
||||||
|
func base64Decode(s string) ([]byte, error) {
|
||||||
|
return base64.StdEncoding.DecodeString(s)
|
||||||
|
}
|
||||||
|
|
||||||
|
func base64Encode(b []byte) string {
|
||||||
|
return base64.StdEncoding.EncodeToString(b)
|
||||||
|
}
|
||||||
662
internal/memory/media/media.go
Normal file
662
internal/memory/media/media.go
Normal file
@ -0,0 +1,662 @@
|
|||||||
|
// Package media 是记忆系统的内容寻址媒体存储(CAS)。
|
||||||
|
//
|
||||||
|
// 为何需要它:此前四层记忆全是纯文本载体——L0 `ContextEvent`、L1 `text.Event`、
|
||||||
|
// L2 `document.Doc`、L3 图库的 `sentences.text TEXT UNIQUE`——没有任何一层能
|
||||||
|
// 存二进制。multimodal 插件注入的图片在本轮对话内可见(走 message 数组,不经
|
||||||
|
// 记忆),下一轮起就只剩 `ToolResultItem.Output` 里那句
|
||||||
|
// "[已将图片注入后续对话] /tmp/x.png",即一条路径字符串。那个文件被删或被
|
||||||
|
// 覆盖之后连线索都断了。
|
||||||
|
//
|
||||||
|
// 为何是内容寻址而不是存路径:
|
||||||
|
// - 路径会失效。/tmp 下的探针图、下载缓存、其他进程的临时产物,记忆里留个
|
||||||
|
// 路径等于留个悬空指针。
|
||||||
|
// - 同一张图往往被反复注入(用户连问几轮同一张截图、see_video 相邻帧高度
|
||||||
|
// 相似)。按 sha256 寻址天然去重,同一份字节只存一遍。
|
||||||
|
// - 内容即身份,跟 L3 图库 `sentences.text UNIQUE` 的思路一致:文本节点用
|
||||||
|
// 文本本身做身份,媒体节点用内容摘要做身份。
|
||||||
|
package media
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/sha256"
|
||||||
|
"database/sql"
|
||||||
|
"encoding/hex"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"math"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
_ "github.com/mattn/go-sqlite3"
|
||||||
|
)
|
||||||
|
|
||||||
|
// digestHexLen 是 sha256 的十六进制串长度。
|
||||||
|
const digestHexLen = sha256.Size * 2
|
||||||
|
|
||||||
|
// Kind 是媒体大类。刻意只分三类而不细分具体格式:
|
||||||
|
// 记忆检索关心的是“这是张图还是段音频”,具体编码交给 MIME 字段。
|
||||||
|
type Kind string
|
||||||
|
|
||||||
|
const (
|
||||||
|
KindImage Kind = "image"
|
||||||
|
KindAudio Kind = "audio"
|
||||||
|
KindVideo Kind = "video"
|
||||||
|
KindOther Kind = "other"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Item 是一条媒体记录。
|
||||||
|
//
|
||||||
|
// Digest 既是主键也是磁盘文件名,所以没有单独的 Path 字段——路径可由
|
||||||
|
// Store 根据 Digest 推导,不落库(落了就又是个会失效的引用)。
|
||||||
|
type Item struct {
|
||||||
|
// Digest 是内容 sha256 的十六进制串(64 字符),媒体的唯一身份。
|
||||||
|
Digest string `json:"digest"`
|
||||||
|
// Kind 是大类,供检索时按模态筛选。
|
||||||
|
Kind Kind `json:"kind"`
|
||||||
|
// MIME 是原始 MIME 类型,如 image/png。
|
||||||
|
MIME string `json:"mime"`
|
||||||
|
// Size 是字节数。
|
||||||
|
Size int64 `json:"size"`
|
||||||
|
// Width/Height 是像素尺寸,未知或不适用时为 0。
|
||||||
|
Width int `json:"width,omitempty"`
|
||||||
|
Height int `json:"height,omitempty"`
|
||||||
|
// OriginPath 是首次入库时的来源路径,仅供人类溯源与调试。
|
||||||
|
// **不可用于读取内容**——它随时可能失效,这正是本包存在的理由。
|
||||||
|
OriginPath string `json:"origin_path,omitempty"`
|
||||||
|
// Tool 是注入这条媒体的工具名(如 multimodal_see_picture)。
|
||||||
|
Tool string `json:"tool,omitempty"`
|
||||||
|
// FirstSeen/LastSeen 是首末次入库时间。
|
||||||
|
FirstSeen time.Time `json:"first_seen"`
|
||||||
|
LastSeen time.Time `json:"last_seen"`
|
||||||
|
// --- 多模态嵌入(v1.2.0)---
|
||||||
|
// Vec 是视觉嵌入向量的序列化(JSON []float64),nil 表示未嵌入。
|
||||||
|
Vec []float64 `json:"vec,omitempty"`
|
||||||
|
// VecModel 是产生 Vec 的模型标识(如 "clip-vit-b32"),
|
||||||
|
// 用于模型切换后判断是否需要重算。
|
||||||
|
VecModel string `json:"vec_model,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Store 管理媒体的元数据(SQLite)与内容(磁盘 CAS 目录)。
|
||||||
|
//
|
||||||
|
// 元数据与内容分离而不是把 blob 塞进 SQLite:单张图动辄几 MB,塞进库会让
|
||||||
|
// 每次 VACUUM/备份都拖着几百 MB 走,也让 WAL 迅速膨胀。CAS 目录用两级
|
||||||
|
// 前缀分桶(ab/cdef...)避免单目录几万文件。
|
||||||
|
type Store struct {
|
||||||
|
mu sync.RWMutex
|
||||||
|
db *sql.DB
|
||||||
|
blobDir string
|
||||||
|
}
|
||||||
|
|
||||||
|
// New 打开(或初始化)媒体存储。
|
||||||
|
// dir 下会建 media.db 与 blobs/ 两个条目。
|
||||||
|
func New(dir string) (*Store, error) {
|
||||||
|
if err := os.MkdirAll(filepath.Join(dir, "blobs"), 0755); err != nil {
|
||||||
|
return nil, fmt.Errorf("media: create blob dir: %w", err)
|
||||||
|
}
|
||||||
|
dbPath := filepath.Join(dir, "media.db")
|
||||||
|
db, err := sql.Open("sqlite3", dbPath+"?_journal_mode=WAL&_busy_timeout=5000")
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("media: open db: %w", err)
|
||||||
|
}
|
||||||
|
s := &Store{db: db, blobDir: filepath.Join(dir, "blobs")}
|
||||||
|
if err := s.initSchema(); err != nil {
|
||||||
|
db.Close()
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return s, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Store) initSchema() error {
|
||||||
|
stmts := []string{
|
||||||
|
// digest 作主键:内容即身份,重复 Put 同一内容不重复落盘。
|
||||||
|
`CREATE TABLE IF NOT EXISTS media (
|
||||||
|
digest TEXT PRIMARY KEY,
|
||||||
|
kind TEXT NOT NULL,
|
||||||
|
mime TEXT NOT NULL,
|
||||||
|
size INTEGER NOT NULL,
|
||||||
|
width INTEGER DEFAULT 0,
|
||||||
|
height INTEGER DEFAULT 0,
|
||||||
|
origin_path TEXT,
|
||||||
|
tool TEXT,
|
||||||
|
first_seen TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
last_seen TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||||
|
)`,
|
||||||
|
`CREATE INDEX IF NOT EXISTS idx_media_kind ON media(kind)`,
|
||||||
|
`CREATE INDEX IF NOT EXISTS idx_media_last_seen ON media(last_seen)`,
|
||||||
|
}
|
||||||
|
for _, q := range stmts {
|
||||||
|
if _, err := s.db.Exec(q); err != nil {
|
||||||
|
return fmt.Errorf("media: schema %q: %w", truncate(q, 60), err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// v1.2.0 迁移:给 media 表加 vec(视觉嵌入向量 JSON)和 vec_model(模型标识)。
|
||||||
|
migrations := []string{
|
||||||
|
`ALTER TABLE media ADD COLUMN vec TEXT`,
|
||||||
|
`ALTER TABLE media ADD COLUMN vec_model TEXT`,
|
||||||
|
}
|
||||||
|
for _, q := range migrations {
|
||||||
|
_, _ = s.db.Exec(q) // 列已存在时返回 "duplicate column name",可忽略
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// blobPath 按两级前缀分桶推导内容路径。
|
||||||
|
func (s *Store) blobPath(digest string) string {
|
||||||
|
if len(digest) < 4 {
|
||||||
|
return filepath.Join(s.blobDir, digest)
|
||||||
|
}
|
||||||
|
return filepath.Join(s.blobDir, digest[:2], digest[2:])
|
||||||
|
}
|
||||||
|
|
||||||
|
// Put 落盘并登记一段媒体内容,返回其 digest。
|
||||||
|
//
|
||||||
|
// 幂等:同一内容重复 Put 不重复落盘,只更新 last_seen 与可选的新元数据
|
||||||
|
// (描述、尺寸等——后来者可能带着前一次没有的信息)。
|
||||||
|
func (s *Store) Put(data []byte, meta Item) (string, error) {
|
||||||
|
if len(data) == 0 {
|
||||||
|
return "", fmt.Errorf("media: empty content")
|
||||||
|
}
|
||||||
|
sum := sha256.Sum256(data)
|
||||||
|
digest := hex.EncodeToString(sum[:])
|
||||||
|
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
|
||||||
|
path := s.blobPath(digest)
|
||||||
|
if _, err := os.Stat(path); os.IsNotExist(err) {
|
||||||
|
if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
|
||||||
|
return "", fmt.Errorf("media: mkdir: %w", err)
|
||||||
|
}
|
||||||
|
// 先写临时文件再 rename:中途崩溃不会留下半个 blob 被后续
|
||||||
|
// 当成完整内容读走(digest 校验能发现,但那时已经把坏数据喂给模型了)。
|
||||||
|
tmp := path + ".tmp"
|
||||||
|
if err := os.WriteFile(tmp, data, 0644); err != nil {
|
||||||
|
return "", fmt.Errorf("media: write blob: %w", err)
|
||||||
|
}
|
||||||
|
if err := os.Rename(tmp, path); err != nil {
|
||||||
|
os.Remove(tmp)
|
||||||
|
return "", fmt.Errorf("media: commit blob: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
now := time.Now()
|
||||||
|
if meta.Kind == "" {
|
||||||
|
meta.Kind = KindFromMIME(meta.MIME)
|
||||||
|
}
|
||||||
|
_, err := s.db.Exec(`
|
||||||
|
INSERT INTO media (digest, kind, mime, size, width, height,
|
||||||
|
origin_path, tool, first_seen, last_seen)
|
||||||
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||||
|
ON CONFLICT(digest) DO UPDATE SET
|
||||||
|
last_seen = excluded.last_seen,
|
||||||
|
width = CASE WHEN media.width = 0 THEN excluded.width ELSE media.width END,
|
||||||
|
height = CASE WHEN media.height = 0 THEN excluded.height ELSE media.height END,
|
||||||
|
tool = CASE WHEN COALESCE(media.tool,'') = '' THEN excluded.tool ELSE media.tool END
|
||||||
|
`, digest, string(meta.Kind), meta.MIME, int64(len(data)), meta.Width, meta.Height,
|
||||||
|
meta.OriginPath, meta.Tool, now, now)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("media: upsert meta: %w", err)
|
||||||
|
}
|
||||||
|
return digest, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get 读取内容并校验 digest。
|
||||||
|
//
|
||||||
|
// 校验不是多余的:CAS 的全部保证建立在"文件名 == 内容摘要"上,磁盘位翻转
|
||||||
|
// 或外部误改会让这条保证失效,而把损坏的图喂给模型只会得到无从追溯的幻觉。
|
||||||
|
func (s *Store) Get(digest string) ([]byte, error) {
|
||||||
|
s.mu.RLock()
|
||||||
|
path := s.blobPath(digest)
|
||||||
|
s.mu.RUnlock()
|
||||||
|
|
||||||
|
data, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("media: read %s: %w", shortDigest(digest), err)
|
||||||
|
}
|
||||||
|
sum := sha256.Sum256(data)
|
||||||
|
if got := hex.EncodeToString(sum[:]); got != digest {
|
||||||
|
return nil, fmt.Errorf("media: digest mismatch for %s (content corrupted)", shortDigest(digest))
|
||||||
|
}
|
||||||
|
return data, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stat 返回元数据,不读内容。
|
||||||
|
func (s *Store) Stat(digest string) (*Item, error) {
|
||||||
|
s.mu.RLock()
|
||||||
|
defer s.mu.RUnlock()
|
||||||
|
return s.scanOne(s.db.QueryRow(`
|
||||||
|
SELECT digest, kind, mime, size, width, height, origin_path, tool,
|
||||||
|
first_seen, last_seen,
|
||||||
|
vec, vec_model
|
||||||
|
FROM media WHERE digest = ?`, digest))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stat 返回元数据,不读内容。
|
||||||
|
//
|
||||||
|
// 这不是 GC,也不看引用计数:调用方是记忆系统本身——当它把一个记忆块
|
||||||
|
// 永久地从三层记忆中删掉(而非在层间迁移)时,媒体作为块的内容一并删除。
|
||||||
|
// 文本块就是这么管理的:删除块即删除内容。
|
||||||
|
func (s *Store) Delete(digest string) error {
|
||||||
|
if digest == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
if err := os.Remove(s.blobPath(digest)); err != nil && !os.IsNotExist(err) {
|
||||||
|
return fmt.Errorf("media: remove blob %s: %w", shortDigest(digest), err)
|
||||||
|
}
|
||||||
|
if _, err := s.db.Exec(`DELETE FROM media WHERE digest = ?`, digest); err != nil {
|
||||||
|
return fmt.Errorf("media: delete meta %s: %w", shortDigest(digest), err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stats 返回条目统计,供 WebUI / healthcheck 展示。
|
||||||
|
func (s *Store) Stats() map[string]interface{} {
|
||||||
|
s.mu.RLock()
|
||||||
|
defer s.mu.RUnlock()
|
||||||
|
|
||||||
|
out := map[string]interface{}{"blob_dir": s.blobDir}
|
||||||
|
var count int
|
||||||
|
var total int64
|
||||||
|
s.db.QueryRow(`SELECT COUNT(*), COALESCE(SUM(size),0) FROM media`).Scan(&count, &total)
|
||||||
|
out["count"] = count
|
||||||
|
out["total_bytes"] = total
|
||||||
|
|
||||||
|
byKind := map[string]int{}
|
||||||
|
rows, err := s.db.Query(`SELECT kind, COUNT(*) FROM media GROUP BY kind`)
|
||||||
|
if err == nil {
|
||||||
|
defer rows.Close()
|
||||||
|
for rows.Next() {
|
||||||
|
var k string
|
||||||
|
var n int
|
||||||
|
if rows.Scan(&k, &n) == nil {
|
||||||
|
byKind[k] = n
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out["by_kind"] = byKind
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Store) Close() error {
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
return s.db.Close()
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- 多模态嵌入(v1.2.0) ----
|
||||||
|
|
||||||
|
// SetVec 给一条已入库的媒体设置视觉嵌入向量。
|
||||||
|
//
|
||||||
|
// 设计选择:vec 是 TEXT(JSON 序列化的 []float64)而非 BLOB,
|
||||||
|
// 因为 Go 的 json.Marshal/Unmarshal 对 []float64 是自然的,
|
||||||
|
// 而 SQLite 的 BLOB 是 []byte,序列化多一层反而复杂。
|
||||||
|
// 量级:一条 vec 最多 1536 维 × ~15 字节 ≈ 23KB,TEXT 合适。
|
||||||
|
func (s *Store) SetVec(digest string, vec []float64, model string) error {
|
||||||
|
vecJSON, err := json.Marshal(vec)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("media: marshal vec: %w", err)
|
||||||
|
}
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
_, err = s.db.Exec(`UPDATE media SET vec = ?, vec_model = ? WHERE digest = ?`,
|
||||||
|
string(vecJSON), model, digest)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// StaleVecDigests 返回所有需要重新嵌入的图片 digest:
|
||||||
|
// vec_model 不等于 currentModel(模型切换)或 vec_model 为空(从未嵌入)。
|
||||||
|
// 调用方使用返回的 digest 列表调用 Get/EmbedImage/SetVec 完成重算。
|
||||||
|
func (s *Store) StaleVecDigests(currentModel string) ([]string, error) {
|
||||||
|
return s.staleVecDigests(currentModel, "image")
|
||||||
|
}
|
||||||
|
|
||||||
|
// StaleVecDigestsAll 返回所有需要重新嵌入的媒体 digest(不限 kind),
|
||||||
|
// 供模型切换后全量迁移向量空间(image + audio + video 等)。
|
||||||
|
func (s *Store) StaleVecDigestsAll(currentModel string) ([]string, error) {
|
||||||
|
return s.staleVecDigests(currentModel, "")
|
||||||
|
}
|
||||||
|
|
||||||
|
// staleVecDigests 是 StaleVecDigests 的核心实现,kind=” 时不按 kind 过滤。
|
||||||
|
// 废弃了"只迁移图片"的限定:模型切换后所有模态都应迁移到新向量空间。
|
||||||
|
func (s *Store) staleVecDigests(currentModel string, kind string) ([]string, error) {
|
||||||
|
s.mu.RLock()
|
||||||
|
defer s.mu.RUnlock()
|
||||||
|
|
||||||
|
query := `
|
||||||
|
SELECT digest FROM media
|
||||||
|
WHERE (COALESCE(vec_model,'') = '' OR vec_model != ?)`
|
||||||
|
if kind != "" {
|
||||||
|
query += ` AND kind = ?`
|
||||||
|
}
|
||||||
|
query += ` ORDER BY last_seen`
|
||||||
|
|
||||||
|
var args []interface{}
|
||||||
|
args = append(args, currentModel)
|
||||||
|
if kind != "" {
|
||||||
|
args = append(args, kind)
|
||||||
|
}
|
||||||
|
|
||||||
|
rows, err := s.db.Query(query, args...)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
|
||||||
|
var digests []string
|
||||||
|
for rows.Next() {
|
||||||
|
var d string
|
||||||
|
if err := rows.Scan(&d); err == nil {
|
||||||
|
digests = append(digests, d)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return digests, rows.Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
// QueryMedia 用查询向量对所有已嵌入媒体做余弦相似度检索,返回 topK 个最相似的 Item。
|
||||||
|
//
|
||||||
|
// 这是跨模态检索的关键:查询可以是图片也可以是文本(经文本向量化后调用此方法),
|
||||||
|
// 被查的媒体库里的每个 item 也有一个视觉向量。两者在同一空间比对,
|
||||||
|
// 谁的相似度更高就召回谁——不再区分「这是一张图的查询」还是「这是一段文字的查询」,
|
||||||
|
// 由向量空间的相似度自动判断。
|
||||||
|
func (s *Store) QueryMedia(queryVec []float64, model string, topK int) ([]*Item, error) {
|
||||||
|
hits, err := s.QueryMediaScored(queryVec, model, topK)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if hits == nil {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
out := make([]*Item, len(hits))
|
||||||
|
for i, h := range hits {
|
||||||
|
out[i] = h.Item
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// MediaHit 是一条媒体相似度候选及其分数。
|
||||||
|
// 跨模态融合需要原始分数做归一化,仅返回 Item 会丢掉尺度信息。
|
||||||
|
type MediaHit struct {
|
||||||
|
Item *Item
|
||||||
|
Score float64
|
||||||
|
}
|
||||||
|
|
||||||
|
// QueryMediaScored 用查询向量对所有已嵌入媒体做余弦相似度检索,
|
||||||
|
// 返回 topK 个最相似的候选及其原始 cosine 分数(供跨模态归一化)。
|
||||||
|
//
|
||||||
|
// 分数只做排序,不在存储层设绝对阈值:多模态文本→图像的绝对 cosine 随模型、
|
||||||
|
// 语言与数据域漂移,真实标定中有效命中可以低至 0.015。相关性门控在融合器中
|
||||||
|
// 使用当前候选集合的相对分布完成。
|
||||||
|
func (s *Store) QueryMediaScored(queryVec []float64, model string, topK int) ([]MediaHit, error) {
|
||||||
|
return s.queryMediaScored(queryVec, model, topK)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Store) queryMediaScored(queryVec []float64, model string, topK int) ([]MediaHit, error) {
|
||||||
|
if topK <= 0 {
|
||||||
|
topK = 20
|
||||||
|
}
|
||||||
|
if len(queryVec) == 0 {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
s.mu.RLock()
|
||||||
|
defer s.mu.RUnlock()
|
||||||
|
|
||||||
|
query := `SELECT digest, kind, mime, size, width, height,
|
||||||
|
origin_path, tool, first_seen, last_seen,
|
||||||
|
vec, vec_model
|
||||||
|
FROM media WHERE vec IS NOT NULL AND vec != ''`
|
||||||
|
var args []interface{}
|
||||||
|
if model != "" {
|
||||||
|
query += ` AND vec_model = ?`
|
||||||
|
args = append(args, model)
|
||||||
|
}
|
||||||
|
rows, err := s.db.Query(query, args...)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
|
||||||
|
type scored struct {
|
||||||
|
item *Item
|
||||||
|
score float64
|
||||||
|
}
|
||||||
|
var candidates []scored
|
||||||
|
for rows.Next() {
|
||||||
|
var it Item
|
||||||
|
var kind string
|
||||||
|
var origin, tool, vecJSON, vecModel sql.NullString
|
||||||
|
if err := rows.Scan(&it.Digest, &kind, &it.MIME, &it.Size, &it.Width, &it.Height,
|
||||||
|
&origin, &tool, &it.FirstSeen, &it.LastSeen,
|
||||||
|
&vecJSON, &vecModel); err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
it.Kind = Kind(kind)
|
||||||
|
it.OriginPath = origin.String
|
||||||
|
it.Tool = tool.String
|
||||||
|
if !vecJSON.Valid || vecJSON.String == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
var itemVec []float64
|
||||||
|
if err := json.Unmarshal([]byte(vecJSON.String), &itemVec); err != nil || len(itemVec) == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if len(itemVec) != len(queryVec) {
|
||||||
|
continue // 维度不一致,跳过
|
||||||
|
}
|
||||||
|
score := cosineSimilaritySlice(queryVec, itemVec)
|
||||||
|
if score > 0.05 {
|
||||||
|
candidates = append(candidates, scored{&it, score})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := rows.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// 按分数降序排序
|
||||||
|
sort.Slice(candidates, func(i, j int) bool {
|
||||||
|
return candidates[i].score > candidates[j].score
|
||||||
|
})
|
||||||
|
if len(candidates) > topK {
|
||||||
|
candidates = candidates[:topK]
|
||||||
|
}
|
||||||
|
out := make([]MediaHit, len(candidates))
|
||||||
|
for i, c := range candidates {
|
||||||
|
out[i] = MediaHit{Item: c.item, Score: c.score}
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// cosineSimilaritySlice 计算两个 []float64 向量的余弦相似度。
|
||||||
|
func cosineSimilaritySlice(a, b []float64) float64 {
|
||||||
|
var dot, normA, normB float64
|
||||||
|
for i := range a {
|
||||||
|
dot += a[i] * b[i]
|
||||||
|
normA += a[i] * a[i]
|
||||||
|
normB += b[i] * b[i]
|
||||||
|
}
|
||||||
|
if normA == 0 || normB == 0 {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
return dot / (math.Sqrt(normA) * math.Sqrt(normB))
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- 扫描辅助 ----
|
||||||
|
|
||||||
|
type rowScanner interface {
|
||||||
|
Scan(dest ...interface{}) error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Store) scanOne(r rowScanner) (*Item, error) {
|
||||||
|
it, err := scanItem(r)
|
||||||
|
if err == sql.ErrNoRows {
|
||||||
|
return nil, fmt.Errorf("media: unknown digest")
|
||||||
|
}
|
||||||
|
return it, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Store) scanRows(r rowScanner) (*Item, error) { return scanItem(r) }
|
||||||
|
|
||||||
|
func scanItem(r rowScanner) (*Item, error) {
|
||||||
|
var it Item
|
||||||
|
var kind string
|
||||||
|
var origin, tool, vecJSON, vecModel sql.NullString
|
||||||
|
if err := r.Scan(&it.Digest, &kind, &it.MIME, &it.Size, &it.Width, &it.Height,
|
||||||
|
&origin, &tool, &it.FirstSeen, &it.LastSeen,
|
||||||
|
&vecJSON, &vecModel); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
it.Kind = Kind(kind)
|
||||||
|
it.OriginPath = origin.String
|
||||||
|
it.Tool = tool.String
|
||||||
|
if vecJSON.Valid && vecJSON.String != "" {
|
||||||
|
var v []float64
|
||||||
|
if err := json.Unmarshal([]byte(vecJSON.String), &v); err == nil {
|
||||||
|
it.Vec = v
|
||||||
|
}
|
||||||
|
}
|
||||||
|
it.VecModel = vecModel.String
|
||||||
|
return &it, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- 工具函数 ----
|
||||||
|
|
||||||
|
// KindFromMIME 把 MIME 归到大类。
|
||||||
|
func KindFromMIME(mime string) Kind {
|
||||||
|
m := strings.ToLower(strings.TrimSpace(mime))
|
||||||
|
switch {
|
||||||
|
case strings.HasPrefix(m, "image/"):
|
||||||
|
return KindImage
|
||||||
|
case strings.HasPrefix(m, "audio/"):
|
||||||
|
return KindAudio
|
||||||
|
case strings.HasPrefix(m, "video/"):
|
||||||
|
return KindVideo
|
||||||
|
default:
|
||||||
|
return KindOther
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ParseDataURL 从 data:<mime>;base64,<data> 提取 MIME 与原始字节。
|
||||||
|
//
|
||||||
|
// 与 agent/api 里的 parseAudioDataURL 分开实现:那个只认音频且只回 base64
|
||||||
|
// 串(它要把串塞回 OpenAI 的 input_audio 字段),这里要的是解码后的字节。
|
||||||
|
func ParseDataURL(url string) (mime string, data []byte, ok bool) {
|
||||||
|
const prefix = "data:"
|
||||||
|
if !strings.HasPrefix(url, prefix) {
|
||||||
|
return "", nil, false
|
||||||
|
}
|
||||||
|
rest := url[len(prefix):]
|
||||||
|
comma := strings.IndexByte(rest, ',')
|
||||||
|
if comma < 0 {
|
||||||
|
return "", nil, false
|
||||||
|
}
|
||||||
|
head := rest[:comma]
|
||||||
|
payload := rest[comma+1:]
|
||||||
|
if !strings.HasSuffix(strings.ToLower(head), ";base64") {
|
||||||
|
return "", nil, false
|
||||||
|
}
|
||||||
|
mime = head[:len(head)-len(";base64")]
|
||||||
|
if mime == "" || payload == "" {
|
||||||
|
return "", nil, false
|
||||||
|
}
|
||||||
|
decoded, err := base64Decode(payload)
|
||||||
|
if err != nil {
|
||||||
|
return "", nil, false
|
||||||
|
}
|
||||||
|
return mime, decoded, true
|
||||||
|
}
|
||||||
|
|
||||||
|
// DataURL 把内容编回 data URL,供重新注入模型对话。
|
||||||
|
func DataURL(mime string, data []byte) string {
|
||||||
|
return "data:" + mime + ";base64," + base64Encode(data)
|
||||||
|
}
|
||||||
|
|
||||||
|
// CopyFrom 从 reader 读全部内容后 Put,用于大文件不便一次性构造 []byte 的场合。
|
||||||
|
func (s *Store) CopyFrom(r io.Reader, meta Item) (string, error) {
|
||||||
|
data, err := io.ReadAll(r)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("media: read source: %w", err)
|
||||||
|
}
|
||||||
|
return s.Put(data, meta)
|
||||||
|
}
|
||||||
|
|
||||||
|
// MarshalItems 序列化条目列表,供工具返回给模型。
|
||||||
|
func MarshalItems(items []*Item) string {
|
||||||
|
b, err := json.Marshal(items)
|
||||||
|
if err != nil {
|
||||||
|
return "[]"
|
||||||
|
}
|
||||||
|
return string(b)
|
||||||
|
}
|
||||||
|
|
||||||
|
func shortDigest(d string) string {
|
||||||
|
if len(d) > 12 {
|
||||||
|
return d[:12]
|
||||||
|
}
|
||||||
|
return d
|
||||||
|
}
|
||||||
|
|
||||||
|
func truncate(s string, n int) string {
|
||||||
|
s = strings.Join(strings.Fields(s), " ")
|
||||||
|
if len(s) <= n {
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
return s[:n] + "..."
|
||||||
|
}
|
||||||
|
|
||||||
|
// ResolvePrefix 把 digest 前缀补全为完整 digest。
|
||||||
|
//
|
||||||
|
// 日志、事件摘要与图库句子里出现的都是 shortDigest(前 12 位),
|
||||||
|
// 因为完整的 64 位 sha256 会把一行文字撑爆、也无助于人眼辨认。
|
||||||
|
// 反查时需要这个补全,否则那些短标记只能看不能用。
|
||||||
|
//
|
||||||
|
// 前缀歧义视为错误而非"取第一个":挂错引用会让 GC 删掉仍被引用的内容,
|
||||||
|
// 宁可这次绑定失败。12 位十六进制的碰撞概率极低,真撞上说明该用更长前缀。
|
||||||
|
func (s *Store) ResolvePrefix(prefix string) (string, error) {
|
||||||
|
prefix = strings.ToLower(strings.TrimSpace(prefix))
|
||||||
|
if len(prefix) < 8 {
|
||||||
|
return "", fmt.Errorf("digest 前缀过短(至少 8 位): %q", prefix)
|
||||||
|
}
|
||||||
|
if len(prefix) == digestHexLen {
|
||||||
|
// 已是完整 digest:仍要确认存在,否则调用方会挂一条孤儿引用
|
||||||
|
if _, err := s.Stat(prefix); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return prefix, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
s.mu.RLock()
|
||||||
|
defer s.mu.RUnlock()
|
||||||
|
rows, err := s.db.Query(
|
||||||
|
`SELECT digest FROM media WHERE digest LIKE ? || '%' LIMIT 2`, prefix)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
|
||||||
|
var found []string
|
||||||
|
for rows.Next() {
|
||||||
|
var d string
|
||||||
|
if err := rows.Scan(&d); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
found = append(found, d)
|
||||||
|
}
|
||||||
|
if err := rows.Err(); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
switch len(found) {
|
||||||
|
case 0:
|
||||||
|
return "", fmt.Errorf("digest 前缀 %q 未匹配到媒体", prefix)
|
||||||
|
case 1:
|
||||||
|
return found[0], nil
|
||||||
|
default:
|
||||||
|
return "", fmt.Errorf("digest 前缀 %q 有歧义(至少匹配 %s 和 %s)",
|
||||||
|
prefix, found[0][:16], found[1][:16])
|
||||||
|
}
|
||||||
|
}
|
||||||
278
internal/memory/media/media_test.go
Normal file
278
internal/memory/media/media_test.go
Normal file
@ -0,0 +1,278 @@
|
|||||||
|
package media
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// newTestStore 建一个临时媒体存储。
|
||||||
|
// 参数保留只为兼容旧调用点;媒体不再有容量上限(生命周期由记忆块决定)。
|
||||||
|
func newTestStore(t *testing.T, _ ...int64) *Store {
|
||||||
|
t.Helper()
|
||||||
|
s, err := New(t.TempDir())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("New: %v", err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { s.Close() })
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPut_ContentAddressedDedup(t *testing.T) {
|
||||||
|
s := newTestStore(t, 0)
|
||||||
|
data := []byte("fake-png-bytes")
|
||||||
|
|
||||||
|
d1, err := s.Put(data, Item{MIME: "image/png", OriginPath: "/tmp/a.png"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
d2, err := s.Put(data, Item{MIME: "image/png", OriginPath: "/tmp/b.png"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if d1 != d2 {
|
||||||
|
t.Fatalf("同一内容应得同一 digest:%s vs %s", d1, d2)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 去重的意义:同一张图反复注入(连问几轮同一截图、see_video 相邻帧)
|
||||||
|
// 只占一份磁盘。
|
||||||
|
st := s.Stats()
|
||||||
|
if st["count"].(int) != 1 {
|
||||||
|
t.Fatalf("同一内容应只有 1 条记录,实际 %v", st["count"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPut_KindInferredFromMIME(t *testing.T) {
|
||||||
|
s := newTestStore(t, 0)
|
||||||
|
cases := map[string]Kind{
|
||||||
|
"image/png": KindImage,
|
||||||
|
"image/jpeg": KindImage,
|
||||||
|
"audio/wav": KindAudio,
|
||||||
|
"video/mp4": KindVideo,
|
||||||
|
"text/plain": KindOther,
|
||||||
|
}
|
||||||
|
for mime, want := range cases {
|
||||||
|
d, err := s.Put([]byte("payload-"+mime), Item{MIME: mime})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
it, err := s.Stat(d)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if it.Kind != want {
|
||||||
|
t.Fatalf("%s 应归为 %s,实际 %s", mime, want, it.Kind)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGet_DetectsCorruption(t *testing.T) {
|
||||||
|
// CAS 的全部保证建立在「文件名 == 内容摘要」上。外部误改或位翻转必须
|
||||||
|
// 被发现——把损坏的图喂给模型只会得到无从追溯的幻觉。
|
||||||
|
s := newTestStore(t, 0)
|
||||||
|
d, err := s.Put([]byte("original-content"), Item{MIME: "image/png"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := s.Get(d); err != nil {
|
||||||
|
t.Fatalf("正常读取应成功: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := os.WriteFile(s.blobPath(d), []byte("tampered!"), 0644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
_, err = s.Get(d)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("内容被改后应报 digest 不匹配,却读成功了")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "digest mismatch") {
|
||||||
|
t.Fatalf("错误应指明 digest 不匹配,得到: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPut_NoPartialBlobOnDisk(t *testing.T) {
|
||||||
|
// 先写 .tmp 再 rename:确认落地后目录里不留临时文件。
|
||||||
|
s := newTestStore(t, 0)
|
||||||
|
d, err := s.Put([]byte("some-bytes"), Item{MIME: "image/png"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
dir := filepath.Dir(s.blobPath(d))
|
||||||
|
entries, err := os.ReadDir(dir)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, e := range entries {
|
||||||
|
if strings.HasSuffix(e.Name(), ".tmp") {
|
||||||
|
t.Fatalf("落地后不该留临时文件: %s", e.Name())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDelete_RemovesContentAndMetadata(t *testing.T) {
|
||||||
|
// 删除块即删除内容:Delete 同时清掉 blob 与元数据。
|
||||||
|
// 这不是 GC,也不看引用计数——调用方是记忆系统本身。
|
||||||
|
s := newTestStore(t)
|
||||||
|
d, _ := s.Put([]byte("held"), Item{MIME: "image/png"})
|
||||||
|
other, _ := s.Put([]byte("orphaned"), Item{MIME: "image/png"})
|
||||||
|
|
||||||
|
if err := s.Delete(other); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := s.Stat(other); err == nil {
|
||||||
|
t.Fatal("删除后元数据应已移除")
|
||||||
|
}
|
||||||
|
if _, err := s.Get(other); err == nil {
|
||||||
|
t.Fatal("删除后内容应已移除")
|
||||||
|
}
|
||||||
|
// 未被删除的项不受影响
|
||||||
|
if _, err := s.Get(d); err != nil {
|
||||||
|
t.Fatalf("未删除的内容不该受影响: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDelete_UnknownDigestIsNoop(t *testing.T) {
|
||||||
|
s := newTestStore(t)
|
||||||
|
if err := s.Delete(""); err != nil {
|
||||||
|
t.Fatalf("空 digest 应为无操作: %v", err)
|
||||||
|
}
|
||||||
|
if err := s.Delete("ffffffffffffffff"); err != nil {
|
||||||
|
t.Fatalf("不存在的 digest 应为无操作: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDescribe_Removed(t *testing.T) {
|
||||||
|
// 媒体不再有文字描述:描述式索引是废弃的就机制。
|
||||||
|
// 这里只保留一个编译期断言,确保 API 不会静默回归。
|
||||||
|
s := newTestStore(t)
|
||||||
|
if _, err := s.Put([]byte("img"), Item{MIME: "image/png"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPut_BackfillsMissingDimensions(t *testing.T) {
|
||||||
|
// 后来者可能带着前一次没有的信息(尺寸、工具名)
|
||||||
|
s := newTestStore(t, 0)
|
||||||
|
data := []byte("img")
|
||||||
|
d, _ := s.Put(data, Item{MIME: "image/png"})
|
||||||
|
|
||||||
|
if _, err := s.Put(data, Item{MIME: "image/png", Width: 640, Height: 480, Tool: "multimodal_see_picture"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
it, _ := s.Stat(d)
|
||||||
|
if it.Width != 640 || it.Height != 480 {
|
||||||
|
t.Fatalf("尺寸应被补写: %dx%d", it.Width, it.Height)
|
||||||
|
}
|
||||||
|
if it.Tool != "multimodal_see_picture" {
|
||||||
|
t.Fatalf("工具名应被补写: %q", it.Tool)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStats_CountsByKind(t *testing.T) {
|
||||||
|
s := newTestStore(t)
|
||||||
|
s.Put([]byte("i1"), Item{MIME: "image/png"})
|
||||||
|
s.Put([]byte("i2"), Item{MIME: "image/jpeg"})
|
||||||
|
s.Put([]byte("a1"), Item{MIME: "audio/wav"})
|
||||||
|
|
||||||
|
st := s.Stats()
|
||||||
|
if st["count"].(int) != 3 {
|
||||||
|
t.Fatalf("count 应为 3,实际 %v", st["count"])
|
||||||
|
}
|
||||||
|
if _, ok := st["described"]; ok {
|
||||||
|
t.Fatal("媒体已不再有描述计数")
|
||||||
|
}
|
||||||
|
byKind := st["by_kind"].(map[string]int)
|
||||||
|
if byKind["image"] != 2 || byKind["audio"] != 1 {
|
||||||
|
t.Fatalf("by_kind 不对: %v", byKind)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseDataURL(t *testing.T) {
|
||||||
|
// 与 agent/api 的 parseAudioDataURL 不同:这里要解码后的字节。
|
||||||
|
raw := []byte{0x89, 'P', 'N', 'G'}
|
||||||
|
url := DataURL("image/png", raw)
|
||||||
|
|
||||||
|
mime, data, ok := ParseDataURL(url)
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("应解析成功")
|
||||||
|
}
|
||||||
|
if mime != "image/png" {
|
||||||
|
t.Fatalf("MIME 应为 image/png,得到 %q", mime)
|
||||||
|
}
|
||||||
|
if string(data) != string(raw) {
|
||||||
|
t.Fatalf("字节应还原,得到 %v", data)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, bad := range []string{
|
||||||
|
"http://example.com/x.png", // 不是 data URL
|
||||||
|
"data:image/png,notbase64", // 缺 ;base64
|
||||||
|
"data:;base64,", // 空 MIME 与空载荷
|
||||||
|
"data:image/png;base64,!!!", // 非法 base64
|
||||||
|
} {
|
||||||
|
if _, _, ok := ParseDataURL(bad); ok {
|
||||||
|
t.Fatalf("%q 应解析失败", bad)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPut_RejectsEmpty(t *testing.T) {
|
||||||
|
s := newTestStore(t)
|
||||||
|
if _, err := s.Put(nil, Item{MIME: "image/png"}); err == nil {
|
||||||
|
t.Fatal("空内容应报错")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestReopen_PersistsAcrossRestart(t *testing.T) {
|
||||||
|
// 记忆的意义就在于跨重启还在。
|
||||||
|
dir := t.TempDir()
|
||||||
|
s1, err := New(dir)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
d, _ := s1.Put([]byte("persistent-img"), Item{MIME: "image/png", OriginPath: "/tmp/x.png"})
|
||||||
|
s1.SetVec(d, []float64{0.1, 0.2}, "test-space")
|
||||||
|
s1.Close()
|
||||||
|
|
||||||
|
s2, err := New(dir)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer s2.Close()
|
||||||
|
|
||||||
|
it, err := s2.Stat(d)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("重开后应能查到: %v", err)
|
||||||
|
}
|
||||||
|
if it.OriginPath != "/tmp/x.png" || len(it.Vec) != 2 || it.VecModel != "test-space" {
|
||||||
|
t.Fatalf("元数据与向量应持久化: %+v", it)
|
||||||
|
}
|
||||||
|
data, err := s2.Get(d)
|
||||||
|
if err != nil || string(data) != "persistent-img" {
|
||||||
|
t.Fatalf("内容应持久化: %v / %q", err, data)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStaleVecDigests_TracksModelSwitch(t *testing.T) {
|
||||||
|
// 模型切换后旧向量必须被重算:StaleVecDigests 是启动迁移的入口。
|
||||||
|
s := newTestStore(t)
|
||||||
|
d1, _ := s.Put([]byte("a"), Item{MIME: "image/png"})
|
||||||
|
d2, _ := s.Put([]byte("b"), Item{MIME: "image/png"})
|
||||||
|
s.SetVec(d1, []float64{0.1}, "space-a")
|
||||||
|
|
||||||
|
stale, err := s.StaleVecDigestsAll("space-a")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(stale) != 1 || stale[0] != d2 {
|
||||||
|
t.Fatalf("只有未嵌入的 d2 需重算,实际 %v", stale)
|
||||||
|
}
|
||||||
|
|
||||||
|
stale, err = s.StaleVecDigestsAll("space-b")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(stale) != 2 {
|
||||||
|
t.Fatalf("换空间后两条都需重算,实际 %v", stale)
|
||||||
|
}
|
||||||
|
}
|
||||||
176
internal/memory/media/media_vec_test.go
Normal file
176
internal/memory/media/media_vec_test.go
Normal file
@ -0,0 +1,176 @@
|
|||||||
|
package media
|
||||||
|
|
||||||
|
import (
|
||||||
|
"math"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestQueryMedia_BasicSimilarity(t *testing.T) {
|
||||||
|
s := newTestStore(t, 0)
|
||||||
|
defer s.Close()
|
||||||
|
|
||||||
|
// 入库三张带向量的媒体:两张图、一段音频
|
||||||
|
d1, _ := s.Put([]byte("img1"), Item{MIME: "image/png"})
|
||||||
|
d2, _ := s.Put([]byte("img2"), Item{MIME: "image/jpeg"})
|
||||||
|
d3, _ := s.Put([]byte("aud1"), Item{MIME: "audio/wav"})
|
||||||
|
|
||||||
|
// 模拟视觉嵌入:img1 和 img2 向量接近,aud1 远离
|
||||||
|
vec1 := []float64{0.9, 0.1, 0.0, 0.0}
|
||||||
|
vec2 := []float64{0.8, 0.2, 0.0, 0.0} // 与 vec1 相似
|
||||||
|
vec3 := []float64{0.0, 0.0, 0.9, 0.1} // 与前两个完全不同
|
||||||
|
|
||||||
|
s.SetVec(d1, vec1, "test-clip")
|
||||||
|
s.SetVec(d2, vec2, "test-clip")
|
||||||
|
s.SetVec(d3, vec3, "test-clip")
|
||||||
|
|
||||||
|
// 用 vec1 作为查询:vec2 最相似,vec3 与 vec1 正交(相似度 0,被阈值过滤)
|
||||||
|
results, err := s.QueryMedia(vec1, "test-clip", 10)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// vec3 与 vec1 正交(余弦相似度 0),被 0.05 阈值正确剔除 → 只召回 2 个
|
||||||
|
if len(results) != 2 {
|
||||||
|
t.Fatalf("expected 2 results (正交的 aud1 被阈值过滤), got %d", len(results))
|
||||||
|
}
|
||||||
|
// 第一个应该是 img2(0.9 vs d1 的 1.0?不,这里算清楚)
|
||||||
|
// vec1·vec2 与 vec1·vec1 比较:
|
||||||
|
// sim(vec1,vec1) = 1.0(img1 与自身),sim(vec1,vec2) = 0.9*0.8+0.1*0.2 = 0.74
|
||||||
|
// 所以 img1(自相似 1.0)排第一,img2 排第二
|
||||||
|
if results[0].Digest != d1 {
|
||||||
|
t.Errorf("expected d1 (自相似 1.0) as first, got %s", results[0].Digest)
|
||||||
|
}
|
||||||
|
if results[1].Digest != d2 {
|
||||||
|
t.Errorf("expected d2 as second, got %s", results[1].Digest)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 验证分数:img1 与自身是 1.0
|
||||||
|
selfScore := cosineSimilaritySlice(vec1, vec1)
|
||||||
|
if math.Abs(selfScore-1.0) > 1e-10 {
|
||||||
|
t.Errorf("self-similarity should be 1.0, got %f", selfScore)
|
||||||
|
}
|
||||||
|
|
||||||
|
// img1 与 aud1 的相似度应该很低
|
||||||
|
crossScore := cosineSimilaritySlice(vec1, vec3)
|
||||||
|
if crossScore > 0.1 {
|
||||||
|
t.Errorf("cross-modality similarity should be low, got %f", crossScore)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestQueryMedia_EmptyVecSkipped(t *testing.T) {
|
||||||
|
s := newTestStore(t, 0)
|
||||||
|
defer s.Close()
|
||||||
|
|
||||||
|
d1, _ := s.Put([]byte("img1"), Item{MIME: "image/png"})
|
||||||
|
_, _ = s.Put([]byte("img2"), Item{MIME: "image/png"})
|
||||||
|
|
||||||
|
// d1 有向量,d2 没有
|
||||||
|
s.SetVec(d1, []float64{0.5, 0.5}, "test")
|
||||||
|
// d2 留空
|
||||||
|
|
||||||
|
results, err := s.QueryMedia([]float64{0.5, 0.5}, "test", 10)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(results) != 1 {
|
||||||
|
t.Fatalf("expected 1 result (d2 has no vec), got %d", len(results))
|
||||||
|
}
|
||||||
|
if results[0].Digest != d1 {
|
||||||
|
t.Errorf("expected d1, got %s", results[0].Digest)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestQueryMedia_DimensionMismatchSkipped(t *testing.T) {
|
||||||
|
s := newTestStore(t, 0)
|
||||||
|
defer s.Close()
|
||||||
|
|
||||||
|
d1, _ := s.Put([]byte("img1"), Item{MIME: "image/png"})
|
||||||
|
s.SetVec(d1, []float64{0.5, 0.5}, "model-A") // 2 维
|
||||||
|
|
||||||
|
// 查询用 3 维向量:维度不匹配,应该返回空
|
||||||
|
results, err := s.QueryMedia([]float64{0.3, 0.3, 0.3}, "model-A", 10)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(results) != 0 {
|
||||||
|
t.Fatalf("expected 0 results (dim mismatch), got %d", len(results))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestQueryMedia_EmptyQueryReturnsNil(t *testing.T) {
|
||||||
|
s := newTestStore(t, 0)
|
||||||
|
defer s.Close()
|
||||||
|
|
||||||
|
results, err := s.QueryMedia(nil, "", 10)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if results != nil {
|
||||||
|
t.Fatalf("expected nil, got %d results", len(results))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSetVec_PersistsCorrectly(t *testing.T) {
|
||||||
|
s := newTestStore(t, 0)
|
||||||
|
defer s.Close()
|
||||||
|
|
||||||
|
d, _ := s.Put([]byte("hello"), Item{MIME: "image/png"})
|
||||||
|
vec := []float64{0.1, 0.2, 0.3, 0.4}
|
||||||
|
s.SetVec(d, vec, "clip-vit-b32")
|
||||||
|
|
||||||
|
it, err := s.Stat(d)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if it.VecModel != "clip-vit-b32" {
|
||||||
|
t.Errorf("VecModel = %q, want clip-vit-b32", it.VecModel)
|
||||||
|
}
|
||||||
|
if len(it.Vec) != 4 {
|
||||||
|
t.Fatalf("Vec len = %d, want 4", len(it.Vec))
|
||||||
|
}
|
||||||
|
for i, v := range vec {
|
||||||
|
if math.Abs(it.Vec[i]-v) > 1e-10 {
|
||||||
|
t.Errorf("Vec[%d] = %f, want %f", i, it.Vec[i], v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStaleVecDigests(t *testing.T) {
|
||||||
|
s := newTestStore(t, 0)
|
||||||
|
defer s.Close()
|
||||||
|
|
||||||
|
// 有向量且 vec_model 匹配 → 非 stale
|
||||||
|
d1, _ := s.Put([]byte("img1"), Item{MIME: "image/png"})
|
||||||
|
s.SetVec(d1, []float64{0.1}, "space-a")
|
||||||
|
|
||||||
|
// 有向量但 vec_model 旧 → stale
|
||||||
|
d2, _ := s.Put([]byte("img2"), Item{MIME: "image/png"})
|
||||||
|
s.SetVec(d2, []float64{0.2}, "space-old")
|
||||||
|
|
||||||
|
// 从未嵌入(vec_model 空)→ stale
|
||||||
|
d3, _ := s.Put([]byte("img3"), Item{MIME: "image/png"})
|
||||||
|
|
||||||
|
// 与向量/描述无关的图片同样应被迁移:图片独立参与向量空间
|
||||||
|
d4, _ := s.Put([]byte("img4"), Item{MIME: "image/png"})
|
||||||
|
|
||||||
|
// 音频不参与图片迁移(StaleVecDigests 只查 kind='image')
|
||||||
|
s.Put([]byte("aud1"), Item{MIME: "audio/wav"})
|
||||||
|
|
||||||
|
stale, err := s.StaleVecDigests("space-a")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// d1 匹配模型 → 非 stale;d2 旧模型 + d3 未嵌入 + d4 无描述图片 = 3 stale;aud1 不算
|
||||||
|
if len(stale) != 3 {
|
||||||
|
t.Fatalf("expected 3 stale digests (d2 旧模型 + d3 未嵌入 + d4 无描述), got %d: %v", len(stale), stale)
|
||||||
|
}
|
||||||
|
got := map[string]bool{}
|
||||||
|
for _, d := range stale {
|
||||||
|
got[d] = true
|
||||||
|
}
|
||||||
|
if !got[d2] || !got[d3] || !got[d4] {
|
||||||
|
t.Errorf("expected d2, d3, d4 stale, got %v", stale)
|
||||||
|
}
|
||||||
|
if got[d1] {
|
||||||
|
t.Errorf("d1 (匹配模型) 不应 stale")
|
||||||
|
}
|
||||||
|
}
|
||||||
277
internal/memory/media/smoke_test.go
Normal file
277
internal/memory/media/smoke_test.go
Normal file
@ -0,0 +1,277 @@
|
|||||||
|
package media
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"compress/zlib"
|
||||||
|
"encoding/binary"
|
||||||
|
"fmt"
|
||||||
|
"hash/crc32"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// 冒烟测试:走真实数据路径的端到端场景,而非孤立的 API 单测。
|
||||||
|
//
|
||||||
|
// 媒体存储现在只做内容寻址(CAS):字节 + 元数据 + 向量。
|
||||||
|
// “哪些字节还活着”由三层记忆持有的一等记忆块决定,调用方把该集合传给
|
||||||
|
// GC/检索,本层不维护 media_refs/ref_count 这类平行账本。
|
||||||
|
|
||||||
|
// makePNG 生成一张 w×h 的条带 PNG,用真 PNG 而不是随机字节,
|
||||||
|
// 让入库/回读/digest 走的是与生产一致的数据形态。
|
||||||
|
func makePNG(w, h, variant int) []byte {
|
||||||
|
palette := [][3]byte{
|
||||||
|
{255, 0, 0}, {0, 192, 0}, {0, 0, 255}, {255, 220, 0}, {160, 0, 200},
|
||||||
|
}
|
||||||
|
var raw bytes.Buffer
|
||||||
|
for y := 0; y < h; y++ {
|
||||||
|
raw.WriteByte(0) // 每行的滤波器字节
|
||||||
|
c := palette[(variant+y*3/h)%len(palette)]
|
||||||
|
for x := 0; x < w; x++ {
|
||||||
|
raw.Write(c[:])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// 把 variant 写进首行头几个像素,确保不同 variant 字节必然不同。
|
||||||
|
b := raw.Bytes()
|
||||||
|
if len(b) > 8 {
|
||||||
|
b[1] = byte(variant)
|
||||||
|
b[2] = byte(variant >> 8)
|
||||||
|
}
|
||||||
|
|
||||||
|
var z bytes.Buffer
|
||||||
|
zw := zlib.NewWriter(&z)
|
||||||
|
zw.Write(b)
|
||||||
|
zw.Close()
|
||||||
|
|
||||||
|
chunk := func(typ string, data []byte) []byte {
|
||||||
|
var out bytes.Buffer
|
||||||
|
binary.Write(&out, binary.BigEndian, uint32(len(data)))
|
||||||
|
out.WriteString(typ)
|
||||||
|
out.Write(data)
|
||||||
|
binary.Write(&out, binary.BigEndian, crc32.ChecksumIEEE(append([]byte(typ), data...)))
|
||||||
|
return out.Bytes()
|
||||||
|
}
|
||||||
|
var ihdr bytes.Buffer
|
||||||
|
binary.Write(&ihdr, binary.BigEndian, uint32(w))
|
||||||
|
binary.Write(&ihdr, binary.BigEndian, uint32(h))
|
||||||
|
ihdr.Write([]byte{8, 2, 0, 0, 0}) // 8bit 深度、truecolor
|
||||||
|
|
||||||
|
var out bytes.Buffer
|
||||||
|
out.Write([]byte{0x89, 'P', 'N', 'G', '\r', '\n', 0x1a, '\n'})
|
||||||
|
out.Write(chunk("IHDR", ihdr.Bytes()))
|
||||||
|
out.Write(chunk("IDAT", z.Bytes()))
|
||||||
|
out.Write(chunk("IEND", nil))
|
||||||
|
return out.Bytes()
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSmoke_SamePictureAcrossTurns(t *testing.T) {
|
||||||
|
// 场景:用户连问几轮同一张截图。multimodal 每轮都会重新注入,
|
||||||
|
// 内容寻址天然去重,磁盘上只应有一份。
|
||||||
|
s := newTestStore(t, 50*1024*1024)
|
||||||
|
png := makePNG(400, 400, 0)
|
||||||
|
|
||||||
|
var d0 string
|
||||||
|
for turn := 1; turn <= 5; turn++ {
|
||||||
|
// 走 data URL:这是 SetToolBlocks 实际给出的形态
|
||||||
|
url := DataURL("image/png", png)
|
||||||
|
mime, data, ok := ParseDataURL(url)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("第 %d 轮 data URL 解析失败", turn)
|
||||||
|
}
|
||||||
|
d, err := s.Put(data, Item{
|
||||||
|
MIME: mime, Width: 400, Height: 400,
|
||||||
|
OriginPath: fmt.Sprintf("/tmp/probe_%d.png", turn),
|
||||||
|
Tool: "multimodal_see_picture",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("第 %d 轮 Put: %v", turn, err)
|
||||||
|
}
|
||||||
|
if d0 == "" {
|
||||||
|
d0 = d
|
||||||
|
} else if d != d0 {
|
||||||
|
t.Fatalf("同一张图第 %d 轮 digest 变了", turn)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
st := s.Stats()
|
||||||
|
if st["count"].(int) != 1 {
|
||||||
|
t.Fatalf("5 轮同图应只存 1 份,实际 %v 条", st["count"])
|
||||||
|
}
|
||||||
|
if total := st["total_bytes"].(int64); total != int64(len(png)) {
|
||||||
|
t.Fatalf("字节数应等于单张原图 %d,实际 %d", len(png), total)
|
||||||
|
}
|
||||||
|
// 三层记忆持有它;内容应仍可读
|
||||||
|
if _, err := s.Get(d0); err != nil {
|
||||||
|
t.Fatalf("内容应仍可读: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSmoke_VideoFramesDistinct(t *testing.T) {
|
||||||
|
// 场景:see_video 抽 6 帧,帧间内容不同,应各存一份。
|
||||||
|
s := newTestStore(t, 50*1024*1024)
|
||||||
|
keep := map[string]bool{}
|
||||||
|
for i := 0; i < 6; i++ {
|
||||||
|
d, err := s.Put(makePNG(320, 240, i), Item{
|
||||||
|
MIME: "image/jpeg", Width: 320, Height: 240, Tool: "multimodal_see_video",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("第 %d 帧: %v", i, err)
|
||||||
|
}
|
||||||
|
keep[d] = true
|
||||||
|
}
|
||||||
|
|
||||||
|
if st := s.Stats(); st["count"].(int) != 6 {
|
||||||
|
t.Fatalf("6 帧应各存一份,实际 %v 条", st["count"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSmoke_NearestNeighborVectorRetrieve(t *testing.T) {
|
||||||
|
// 场景 C:图片只按自己的原生向量被检索。
|
||||||
|
// 没有描述文本参与——描述式索引是废弃的就机制。
|
||||||
|
s := newTestStore(t)
|
||||||
|
|
||||||
|
pic, _ := s.Put(makePNG(400, 400, 0), Item{MIME: "image/png", Tool: "multimodal_see_picture"})
|
||||||
|
s.SetVec(pic, []float64{1, 0, 0, 0}, "space")
|
||||||
|
var frames []string
|
||||||
|
for i := 0; i < 6; i++ {
|
||||||
|
d, _ := s.Put(makePNG(320, 240, i), Item{MIME: "image/jpeg", Tool: "multimodal_see_video"})
|
||||||
|
s.SetVec(d, []float64{1, 1, float64(i) / 10, 0}, "space")
|
||||||
|
frames = append(frames, d)
|
||||||
|
}
|
||||||
|
|
||||||
|
hits, err := s.QueryMediaScored([]float64{1, 0, 0, 0}, "space", 10)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(hits) != 7 {
|
||||||
|
t.Fatalf("7 份媒体都有同空间向量,应全部可召,实际 %d", len(hits))
|
||||||
|
}
|
||||||
|
if hits[0].Item.Digest != pic {
|
||||||
|
t.Fatalf("与查询同向的应是第一命中,实际 %s", shortDigest(hits[0].Item.Digest))
|
||||||
|
}
|
||||||
|
|
||||||
|
// 不同向量空间/模型的条目不得参与:坐标系不同,余弦无意义。
|
||||||
|
foreign := frames[0]
|
||||||
|
if err := s.SetVec(foreign, []float64{1, 0, 0, 0}, "other-space"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
hits, err = s.QueryMediaScored([]float64{1, 0, 0, 0}, "space", 10)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, h := range hits {
|
||||||
|
if h.Item.Digest == foreign {
|
||||||
|
t.Fatal("另一套空间(other-space)的向量不该被 space 查询召回")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSmoke_ContentSurvivesLayerMigration(t *testing.T) {
|
||||||
|
// 场景:同一份媒体随记忆块从 Context 迁移到 Document 再到 Graph。
|
||||||
|
// 迁移的是块本身,digest 不变,因此内容在整条链路上始终可读。
|
||||||
|
s := newTestStore(t, 50*1024*1024)
|
||||||
|
png := makePNG(400, 400, 0)
|
||||||
|
d, _ := s.Put(png, Item{MIME: "image/png", Tool: "multimodal_see_picture"})
|
||||||
|
|
||||||
|
// 迁移过程中该 digest 始终可读
|
||||||
|
for _, layer := range []string{"context", "document", "graph"} {
|
||||||
|
if got, err := s.Get(d); err != nil || !bytes.Equal(got, png) {
|
||||||
|
t.Fatalf("迁移到 %s 时内容应完好: %v", layer, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSmoke_DeleteRemovesOnlyThatContent(t *testing.T) {
|
||||||
|
// 场景:某个工具产出的一次性图片所在的记忆块被删除时,
|
||||||
|
// 只有它自己的内容被删;其他块的内容一个都不能少。
|
||||||
|
s := newTestStore(t)
|
||||||
|
|
||||||
|
held, _ := s.Put(makePNG(400, 400, 0), Item{MIME: "image/png"})
|
||||||
|
var frames []string
|
||||||
|
for i := 0; i < 6; i++ {
|
||||||
|
d, _ := s.Put(makePNG(320, 240, i), Item{MIME: "image/jpeg"})
|
||||||
|
frames = append(frames, d)
|
||||||
|
}
|
||||||
|
var ephemeral []string
|
||||||
|
for i := 0; i < 20; i++ {
|
||||||
|
d, _ := s.Put(makePNG(100, 100, 1000+i), Item{MIME: "image/png", Tool: "cmd_run"})
|
||||||
|
ephemeral = append(ephemeral, d)
|
||||||
|
}
|
||||||
|
|
||||||
|
before := s.Stats()["count"].(int)
|
||||||
|
for _, d := range ephemeral {
|
||||||
|
if err := s.Delete(d); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
after := s.Stats()["count"].(int)
|
||||||
|
if after != before-20 {
|
||||||
|
t.Fatalf("条目数应从 %d 降到 %d,实际 %d", before, before-20, after)
|
||||||
|
}
|
||||||
|
if _, err := s.Get(held); err != nil {
|
||||||
|
t.Fatalf("被保留的内容被误删: %v", err)
|
||||||
|
}
|
||||||
|
for i, f := range frames {
|
||||||
|
if _, err := s.Get(f); err != nil {
|
||||||
|
t.Fatalf("第 %d 帧被误删: %v", i, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSmoke_FullLifecycleAcrossRestart(t *testing.T) {
|
||||||
|
// 端到端:入库 → 嵌入 → 删除一些内容 → 重启 → 向量检索,
|
||||||
|
// 并确认磁盘与元数据不出现双向孤儿。记忆的意义就在于跨重启还在。
|
||||||
|
dir := t.TempDir()
|
||||||
|
s, err := New(dir)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
png := makePNG(400, 400, 0)
|
||||||
|
pic, _ := s.Put(png, Item{MIME: "image/png", Width: 400, Height: 400, Tool: "multimodal_see_picture"})
|
||||||
|
s.SetVec(pic, []float64{1, 0, 0}, "space")
|
||||||
|
for i := 0; i < 6; i++ {
|
||||||
|
d, _ := s.Put(makePNG(320, 240, i), Item{MIME: "image/jpeg", Tool: "multimodal_see_video"})
|
||||||
|
s.SetVec(d, []float64{0, 1, float64(i)}, "space")
|
||||||
|
}
|
||||||
|
for i := 0; i < 10; i++ {
|
||||||
|
d, _ := s.Put(makePNG(64, 64, 2000+i), Item{MIME: "image/png", Tool: "cmd_run"})
|
||||||
|
if err := s.Delete(d); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
beforeCount := s.Stats()["count"].(int)
|
||||||
|
s.Close()
|
||||||
|
|
||||||
|
s2, err := New(dir)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("重开失败: %v", err)
|
||||||
|
}
|
||||||
|
defer s2.Close()
|
||||||
|
|
||||||
|
if got := s2.Stats()["count"].(int); got != beforeCount {
|
||||||
|
t.Fatalf("重开后条目数变了: %d → %d", beforeCount, got)
|
||||||
|
}
|
||||||
|
it, err := s2.Stat(pic)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("重开后查不到: %v", err)
|
||||||
|
}
|
||||||
|
if len(it.Vec) != 3 || it.VecModel != "space" {
|
||||||
|
t.Fatalf("向量未持久化: %+v", it)
|
||||||
|
}
|
||||||
|
data, err := s2.Get(pic)
|
||||||
|
if err != nil || !bytes.Equal(data, png) {
|
||||||
|
t.Fatalf("重开后内容不一致: %v", err)
|
||||||
|
}
|
||||||
|
hits, err := s2.QueryMediaScored([]float64{1, 0, 0}, "space", 10)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(hits) == 0 || hits[0].Item.Digest != pic {
|
||||||
|
t.Fatal("重开后向量检索应仍能命中")
|
||||||
|
}
|
||||||
|
|
||||||
|
// 磁盘文件数 == 元数据条数:无「元数据在文件没了」也无「文件在元数据没了」
|
||||||
|
if n := blobFileCount(t, s2); n != beforeCount {
|
||||||
|
t.Fatalf("磁盘 blob=%d 与元数据=%d 不一致", n, beforeCount)
|
||||||
|
}
|
||||||
|
t.Logf("跨重启:%d 条目、向量与内容全部完好", beforeCount)
|
||||||
|
}
|
||||||
166
internal/memory/media/soak_test.go
Normal file
166
internal/memory/media/soak_test.go
Normal file
@ -0,0 +1,166 @@
|
|||||||
|
package media
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"crypto/rand"
|
||||||
|
"fmt"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestSoak_SustainedMixedLoad 长稳测试:持续混合负载下不变量不破。
|
||||||
|
// 用 -run TestSoak -timeout 300s 单独跑,默认 short 模式跳过。
|
||||||
|
//
|
||||||
|
// 媒体没有独立生命周期管理:blob 是记忆块的内容,块被删除时内容随之删除。
|
||||||
|
func TestSoak_SustainedMixedLoad(t *testing.T) {
|
||||||
|
if testing.Short() {
|
||||||
|
t.Skip("long soak test; run with -run TestSoak")
|
||||||
|
}
|
||||||
|
dur := 60 * time.Second
|
||||||
|
s := newTestStore(t)
|
||||||
|
|
||||||
|
// 常驻受保护区:全程被记忆块持有,模拟 Graph L3 中的块
|
||||||
|
const keepN = 20
|
||||||
|
keep := make([]string, keepN)
|
||||||
|
keepData := make([][]byte, keepN)
|
||||||
|
for i := range keep {
|
||||||
|
d := make([]byte, 4096)
|
||||||
|
rand.Read(d)
|
||||||
|
d = append([]byte(fmt.Sprintf("keep-%d-", i)), d...)
|
||||||
|
dg, err := s.Put(d, Item{MIME: "image/png"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
keep[i] = dg
|
||||||
|
keepData[i] = d
|
||||||
|
}
|
||||||
|
|
||||||
|
stop := make(chan struct{})
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
var puts, gets, deletes, embeds, searches atomic.Int64
|
||||||
|
var fatal atomic.Int64
|
||||||
|
|
||||||
|
worker := func(name string, fn func(iter int) error) {
|
||||||
|
wg.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
for i := 0; ; i++ {
|
||||||
|
select {
|
||||||
|
case <-stop:
|
||||||
|
return
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
if err := fn(i); err != nil {
|
||||||
|
fatal.Add(1)
|
||||||
|
t.Errorf("%s 第 %d 次失败: %v", name, i, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
|
||||||
|
// 写入者 ×3:持续写入一次性内容(无人持有)
|
||||||
|
for w := 0; w < 3; w++ {
|
||||||
|
wid := w
|
||||||
|
worker(fmt.Sprintf("put-%d", wid), func(i int) error {
|
||||||
|
b := make([]byte, 2048)
|
||||||
|
rand.Read(b)
|
||||||
|
b = append([]byte(fmt.Sprintf("eph-%d-%d-", wid, i)), b...)
|
||||||
|
if _, err := s.Put(b, Item{MIME: "image/png", Tool: "cmd_run"}); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
puts.Add(1)
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// 读取者 ×3:受保护集必须始终完好
|
||||||
|
for r := 0; r < 3; r++ {
|
||||||
|
worker("get", func(i int) error {
|
||||||
|
idx := i % keepN
|
||||||
|
got, err := s.Get(keep[idx])
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !bytes.Equal(got, keepData[idx]) {
|
||||||
|
return fmt.Errorf("内容被改 %s", shortDigest(keep[idx]))
|
||||||
|
}
|
||||||
|
gets.Add(1)
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// 删除者:持续删除一次性内容(模拟块创建后又被遗忘)
|
||||||
|
worker("delete", func(i int) error {
|
||||||
|
b := make([]byte, 2048)
|
||||||
|
rand.Read(b)
|
||||||
|
b = append([]byte(fmt.Sprintf("del-%d-", i)), b...)
|
||||||
|
d, err := s.Put(b, Item{MIME: "image/png", Tool: "cmd_run"})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := s.Delete(d); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
deletes.Add(1)
|
||||||
|
time.Sleep(time.Millisecond)
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
|
||||||
|
// 向量写入者:持续给新内容嵌入并删除(模拟启动迁移/短命媒体)
|
||||||
|
worker("embed", func(i int) error {
|
||||||
|
b := make([]byte, 1024)
|
||||||
|
rand.Read(b)
|
||||||
|
b = append([]byte(fmt.Sprintf("emb-%d-", i)), b...)
|
||||||
|
d, err := s.Put(b, Item{MIME: "image/png", Tool: "cmd_run"})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := s.SetVec(d, []float64{1, float64(i % 7)}, "soak-space"); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
embeds.Add(1)
|
||||||
|
time.Sleep(time.Millisecond)
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
|
||||||
|
// 检索者
|
||||||
|
worker("search", func(i int) error {
|
||||||
|
if _, err := s.QueryMediaScored([]float64{1, 0}, "soak-space", 20); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if _, err := s.Stat(keep[i%keepN]); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
searches.Add(1)
|
||||||
|
time.Sleep(2 * time.Millisecond)
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
|
||||||
|
time.Sleep(dur)
|
||||||
|
close(stop)
|
||||||
|
wg.Wait()
|
||||||
|
|
||||||
|
if n := fatal.Load(); n > 0 {
|
||||||
|
t.Fatalf("%d 个 worker 报致命错误", n)
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Logf("%v 内: put=%d get=%d delete=%d embed=%d search=%d",
|
||||||
|
dur, puts.Load(), gets.Load(), deletes.Load(), embeds.Load(), searches.Load())
|
||||||
|
|
||||||
|
// 收尾断言
|
||||||
|
for i, d := range keep {
|
||||||
|
got, err := s.Get(d)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("受保护项丢失 %s: %v", shortDigest(d), err)
|
||||||
|
}
|
||||||
|
if !bytes.Equal(got, keepData[i]) {
|
||||||
|
t.Fatalf("受保护项内容变了 %s", shortDigest(d))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
st := s.Stats()
|
||||||
|
t.Logf("收尾: 条目=%v 字节=%v 类型=%v", st["count"], st["total_bytes"], st["by_kind"])
|
||||||
|
}
|
||||||
527
internal/memory/media/stress_test.go
Normal file
527
internal/memory/media/stress_test.go
Normal file
@ -0,0 +1,527 @@
|
|||||||
|
package media
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"crypto/rand"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// 压力测试与冒烟测试。
|
||||||
|
//
|
||||||
|
// 关注点不是吞吐数字,而是并发下的不变量是否被破坏:
|
||||||
|
// 1. GC 与读写并发时,被记忆块持有的内容绝不能被删
|
||||||
|
// 2. 同内容并发 Put 只落一份磁盘、digest 一致
|
||||||
|
// 3. SQLite 在多 goroutine 下不出现 "database is locked"
|
||||||
|
//
|
||||||
|
// 存活判定不再依赖 media_refs/ref_count:调用方把「三层记忆当前持有的
|
||||||
|
// digest 集合」传给 GC,本层只做 CAS。
|
||||||
|
|
||||||
|
func randBytes(t *testing.T, n int) []byte {
|
||||||
|
t.Helper()
|
||||||
|
b := make([]byte, n)
|
||||||
|
if _, err := rand.Read(b); err != nil {
|
||||||
|
t.Fatalf("rand: %v", err)
|
||||||
|
}
|
||||||
|
return b
|
||||||
|
}
|
||||||
|
|
||||||
|
// blobFileCount 统计 CAS 目录下的实际文件数(不含 .tmp)。
|
||||||
|
func blobFileCount(t *testing.T, s *Store) int {
|
||||||
|
t.Helper()
|
||||||
|
n := 0
|
||||||
|
filepath.Walk(s.blobDir, func(path string, info os.FileInfo, err error) error {
|
||||||
|
if err != nil || info == nil || info.IsDir() {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if strings.HasSuffix(path, ".tmp") {
|
||||||
|
t.Errorf("残留临时文件: %s", path)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
n++
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
return n
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStress_ConcurrentPutSameContent(t *testing.T) {
|
||||||
|
// 同一内容被 N 个 goroutine 同时 Put:digest 必须一致,磁盘只一份。
|
||||||
|
// 现实对应:see_video 抽出的相邻帧、用户连发同一张图。
|
||||||
|
s := newTestStore(t, 0)
|
||||||
|
data := randBytes(t, 64*1024)
|
||||||
|
|
||||||
|
const workers = 32
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
digests := make([]string, workers)
|
||||||
|
errs := make([]error, workers)
|
||||||
|
for i := 0; i < workers; i++ {
|
||||||
|
wg.Add(1)
|
||||||
|
go func(idx int) {
|
||||||
|
defer wg.Done()
|
||||||
|
d, err := s.Put(data, Item{MIME: "image/png", OriginPath: fmt.Sprintf("/tmp/%d.png", idx)})
|
||||||
|
digests[idx] = d
|
||||||
|
errs[idx] = err
|
||||||
|
}(i)
|
||||||
|
}
|
||||||
|
wg.Wait()
|
||||||
|
|
||||||
|
for i, err := range errs {
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("worker %d Put 失败: %v", i, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
first := digests[0]
|
||||||
|
for i, d := range digests {
|
||||||
|
if d != first {
|
||||||
|
t.Fatalf("worker %d digest 不一致: %s vs %s", i, d, first)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if n := blobFileCount(t, s); n != 1 {
|
||||||
|
t.Fatalf("同一内容应只落一份 blob,实际 %d 个文件", n)
|
||||||
|
}
|
||||||
|
if got, err := s.Get(first); err != nil || !bytes.Equal(got, data) {
|
||||||
|
t.Fatalf("内容应可完整读回: err=%v len=%d", err, len(got))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStress_ConcurrentPutDistinctContent(t *testing.T) {
|
||||||
|
// 大量不同内容并发入库:不丢条目、不串内容。
|
||||||
|
s := newTestStore(t, 0)
|
||||||
|
const workers = 16
|
||||||
|
const perWorker = 25
|
||||||
|
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
var failed atomic.Int64
|
||||||
|
type rec struct {
|
||||||
|
digest string
|
||||||
|
data []byte
|
||||||
|
}
|
||||||
|
recCh := make(chan rec, workers*perWorker)
|
||||||
|
|
||||||
|
for w := 0; w < workers; w++ {
|
||||||
|
wg.Add(1)
|
||||||
|
go func(wid int) {
|
||||||
|
defer wg.Done()
|
||||||
|
for i := 0; i < perWorker; i++ {
|
||||||
|
data := []byte(fmt.Sprintf("w%d-i%d-", wid, i))
|
||||||
|
data = append(data, randBytes(t, 512)...)
|
||||||
|
d, err := s.Put(data, Item{MIME: "image/png"})
|
||||||
|
if err != nil {
|
||||||
|
failed.Add(1)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
recCh <- rec{digest: d, data: data}
|
||||||
|
}
|
||||||
|
}(w)
|
||||||
|
}
|
||||||
|
wg.Wait()
|
||||||
|
close(recCh)
|
||||||
|
|
||||||
|
if n := failed.Load(); n > 0 {
|
||||||
|
t.Fatalf("%d 次 Put 失败", n)
|
||||||
|
}
|
||||||
|
|
||||||
|
var records []rec
|
||||||
|
for r := range recCh {
|
||||||
|
records = append(records, r)
|
||||||
|
}
|
||||||
|
if len(records) != workers*perWorker {
|
||||||
|
t.Fatalf("应有 %d 条记录,实际 %d", workers*perWorker, len(records))
|
||||||
|
}
|
||||||
|
|
||||||
|
// 逐条回读校验内容没串
|
||||||
|
for _, r := range records {
|
||||||
|
got, err := s.Get(r.digest)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("读 %s 失败: %v", shortDigest(r.digest), err)
|
||||||
|
}
|
||||||
|
if !bytes.Equal(got, r.data) {
|
||||||
|
t.Fatalf("内容串了: %s", shortDigest(r.digest))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
st := s.Stats()
|
||||||
|
if st["count"].(int) != len(records) {
|
||||||
|
t.Fatalf("库内条目应为 %d,实际 %v", len(records), st["count"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStress_ConcurrentDeleteAndPut(t *testing.T) {
|
||||||
|
// 删除与写入并发:核心断言是被保留的内容永远可读,
|
||||||
|
// 删除只影响目标 digest,不误伤其他内容。
|
||||||
|
s := newTestStore(t)
|
||||||
|
|
||||||
|
const heldCount = 8
|
||||||
|
held := make([]string, heldCount)
|
||||||
|
for i := range held {
|
||||||
|
d, err := s.Put([]byte(fmt.Sprintf("payload-%d", i)), Item{MIME: "image/png"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
held[i] = d
|
||||||
|
}
|
||||||
|
|
||||||
|
const workers = 16
|
||||||
|
const rounds = 30
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
for w := 0; w < workers; w++ {
|
||||||
|
wg.Add(1)
|
||||||
|
go func(wid int) {
|
||||||
|
defer wg.Done()
|
||||||
|
for r := 0; r < rounds; r++ {
|
||||||
|
d, err := s.Put([]byte(fmt.Sprintf("tmp-%d-%d", wid, r)), Item{MIME: "image/png"})
|
||||||
|
if err != nil {
|
||||||
|
t.Errorf("Put: %v", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := s.Delete(d); err != nil {
|
||||||
|
t.Errorf("Delete: %v", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}(w)
|
||||||
|
}
|
||||||
|
// 并发读取被保留内容
|
||||||
|
for rdr := 0; rdr < 4; rdr++ {
|
||||||
|
wg.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
for r := 0; r < rounds; r++ {
|
||||||
|
for i, d := range held {
|
||||||
|
if _, err := s.Get(d); err != nil {
|
||||||
|
t.Errorf("内容 %d 被误删: %v", i, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
wg.Wait()
|
||||||
|
|
||||||
|
for i, d := range held {
|
||||||
|
if _, err := s.Get(d); err != nil {
|
||||||
|
t.Fatalf("仍被保留的第 %d 项不可读: %v", i, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStress_DeleteConcurrentWithReads(t *testing.T) {
|
||||||
|
// 删除与读取并发。最重要的断言:被保留的内容在整个过程中始终可读。
|
||||||
|
s := newTestStore(t)
|
||||||
|
|
||||||
|
const protectedCount = 10
|
||||||
|
protected := make([]string, protectedCount)
|
||||||
|
protectedData := make([][]byte, protectedCount)
|
||||||
|
for i := range protected {
|
||||||
|
data := append([]byte(fmt.Sprintf("protected-%d-", i)), randBytes(t, 256)...)
|
||||||
|
d, err := s.Put(data, Item{MIME: "image/png"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
protected[i] = d
|
||||||
|
protectedData[i] = data
|
||||||
|
}
|
||||||
|
|
||||||
|
stop := make(chan struct{})
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
var readErr atomic.Int64
|
||||||
|
var deleteCount atomic.Int64
|
||||||
|
var putCount atomic.Int64
|
||||||
|
|
||||||
|
// 写入者:持续 Put 一次性内容再删除(模拟块创建后又被遗忘)
|
||||||
|
for w := 0; w < 4; w++ {
|
||||||
|
wg.Add(1)
|
||||||
|
go func(wid int) {
|
||||||
|
defer wg.Done()
|
||||||
|
i := 0
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-stop:
|
||||||
|
return
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
data := append([]byte(fmt.Sprintf("ephemeral-%d-%d-", wid, i)), randBytes(t, 128)...)
|
||||||
|
d, err := s.Put(data, Item{MIME: "image/png"})
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
putCount.Add(1)
|
||||||
|
if err := s.Delete(d); err == nil {
|
||||||
|
deleteCount.Add(1)
|
||||||
|
}
|
||||||
|
i++
|
||||||
|
}
|
||||||
|
}(w)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 读取者:反复读受保护内容,任何一次失败都是致命的
|
||||||
|
for r := 0; r < 4; r++ {
|
||||||
|
wg.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-stop:
|
||||||
|
return
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
for i, d := range protected {
|
||||||
|
got, err := s.Get(d)
|
||||||
|
if err != nil {
|
||||||
|
readErr.Add(1)
|
||||||
|
t.Errorf("受保护内容读失败 %s: %v", shortDigest(d), err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if !bytes.Equal(got, protectedData[i]) {
|
||||||
|
readErr.Add(1)
|
||||||
|
t.Errorf("受保护内容被改 %s", shortDigest(d))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
|
||||||
|
time.Sleep(1500 * time.Millisecond)
|
||||||
|
close(stop)
|
||||||
|
wg.Wait()
|
||||||
|
|
||||||
|
if n := readErr.Load(); n > 0 {
|
||||||
|
t.Fatalf("受保护内容读取失败 %d 次", n)
|
||||||
|
}
|
||||||
|
t.Logf("并发窗口内: Put=%d Delete=%d", putCount.Load(), deleteCount.Load())
|
||||||
|
|
||||||
|
// 收尾确认:受保护的一个都没少
|
||||||
|
for i, d := range protected {
|
||||||
|
got, err := s.Get(d)
|
||||||
|
if err != nil || !bytes.Equal(got, protectedData[i]) {
|
||||||
|
t.Fatalf("收尾检查失败 %s: %v", shortDigest(d), err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStress_SetVecConcurrentWithQuery(t *testing.T) {
|
||||||
|
// 嵌入写入与向量检索并发(启动时的向量迁移就会长期这样跑)。
|
||||||
|
s := newTestStore(t)
|
||||||
|
const n = 60
|
||||||
|
digests := make([]string, n)
|
||||||
|
for i := range digests {
|
||||||
|
d, err := s.Put([]byte(fmt.Sprintf("img-%d", i)), Item{MIME: "image/png"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
digests[i] = d
|
||||||
|
}
|
||||||
|
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
var writeErr, queryErr atomic.Int64
|
||||||
|
|
||||||
|
// 向量写入者
|
||||||
|
for w := 0; w < 4; w++ {
|
||||||
|
wg.Add(1)
|
||||||
|
go func(wid int) {
|
||||||
|
defer wg.Done()
|
||||||
|
for i := wid; i < n; i += 4 {
|
||||||
|
vec := []float64{1, float64(i) / 100, 0}
|
||||||
|
if err := s.SetVec(digests[i], vec, "space"); err != nil {
|
||||||
|
writeErr.Add(1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}(w)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 检索者
|
||||||
|
for r := 0; r < 3; r++ {
|
||||||
|
wg.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
for i := 0; i < 50; i++ {
|
||||||
|
if _, err := s.QueryMediaScored([]float64{1, 0, 0}, "space", 20); err != nil {
|
||||||
|
queryErr.Add(1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
wg.Wait()
|
||||||
|
|
||||||
|
if v := writeErr.Load(); v > 0 {
|
||||||
|
t.Fatalf("SetVec 失败 %d 次", v)
|
||||||
|
}
|
||||||
|
if v := queryErr.Load(); v > 0 {
|
||||||
|
t.Fatalf("QueryMediaScored 失败 %d 次", v)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 全部应已嵌入,且都在同一空间
|
||||||
|
stale, err := s.StaleVecDigestsAll("space")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(stale) != 0 {
|
||||||
|
t.Fatalf("应全部已嵌入,仍有 %d 条未嵌入", len(stale))
|
||||||
|
}
|
||||||
|
got, err := s.QueryMediaScored([]float64{1, 0, 0}, "space", 1000)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(got) != n {
|
||||||
|
t.Fatalf("应检索到 %d 条,实际 %d", n, len(got))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStress_DeleteUnderLoad(t *testing.T) {
|
||||||
|
// 持续写入 + 删除下,被保留的项必须始终可读。
|
||||||
|
s := newTestStore(t)
|
||||||
|
|
||||||
|
const keepN = 3
|
||||||
|
keepList := make([]string, keepN)
|
||||||
|
for i := range keepList {
|
||||||
|
data := append([]byte(fmt.Sprintf("keep-%d-", i)), randBytes(t, 32*1024)...)
|
||||||
|
d, err := s.Put(data, Item{MIME: "image/png"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
keepList[i] = d
|
||||||
|
}
|
||||||
|
|
||||||
|
for round := 0; round < 30; round++ {
|
||||||
|
for i := 0; i < 3; i++ {
|
||||||
|
data := append([]byte(fmt.Sprintf("tmp-%d-%d-", round, i)), randBytes(t, 16*1024)...)
|
||||||
|
d, err := s.Put(data, Item{MIME: "image/png"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("round %d Put: %v", round, err)
|
||||||
|
}
|
||||||
|
if err := s.Delete(d); err != nil {
|
||||||
|
t.Fatalf("round %d Delete: %v", round, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 被保留的项必须都在
|
||||||
|
for _, d := range keepList {
|
||||||
|
if _, err := s.Get(d); err != nil {
|
||||||
|
t.Fatalf("被保留项被误删 %s: %v", shortDigest(d), err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStress_ReopenAfterHeavyChurn(t *testing.T) {
|
||||||
|
// 大量写入 + 删除之后重开:元数据与磁盘不该出现互相不认的孤儿。
|
||||||
|
dir := t.TempDir()
|
||||||
|
s1, err := New(dir)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var kept []string
|
||||||
|
for i := 0; i < 100; i++ {
|
||||||
|
data := append([]byte(fmt.Sprintf("churn-%d-", i)), randBytes(t, 256)...)
|
||||||
|
d, err := s1.Put(data, Item{MIME: "image/png"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if i%5 == 0 {
|
||||||
|
kept = append(kept, d)
|
||||||
|
} else if err := s1.Delete(d); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
beforeStats := s1.Stats()
|
||||||
|
s1.Close()
|
||||||
|
|
||||||
|
s2, err := New(dir)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("重开失败: %v", err)
|
||||||
|
}
|
||||||
|
defer s2.Close()
|
||||||
|
|
||||||
|
afterStats := s2.Stats()
|
||||||
|
if beforeStats["count"] != afterStats["count"] {
|
||||||
|
t.Fatalf("重开后条目数变了: %v → %v", beforeStats["count"], afterStats["count"])
|
||||||
|
}
|
||||||
|
|
||||||
|
// 每条元数据都应有对应磁盘文件(无「元数据在文件没了」的孤儿)
|
||||||
|
rows, err := s2.db.Query(`SELECT digest FROM media`)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var missing int
|
||||||
|
for rows.Next() {
|
||||||
|
var d string
|
||||||
|
if rows.Scan(&d) != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(s2.blobPath(d)); err != nil {
|
||||||
|
missing++
|
||||||
|
if missing <= 3 {
|
||||||
|
t.Errorf("元数据存在但 blob 缺失: %s", shortDigest(d))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
rows.Close()
|
||||||
|
if missing > 0 {
|
||||||
|
t.Fatalf("%d 条元数据没有对应文件", missing)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 磁盘文件数应等于元数据条数(无「文件在元数据没了」的孤儿)
|
||||||
|
if n := blobFileCount(t, s2); n != afterStats["count"].(int) {
|
||||||
|
t.Fatalf("磁盘文件 %d 与元数据 %v 不一致", n, afterStats["count"])
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, d := range kept {
|
||||||
|
if _, err := s2.Get(d); err != nil {
|
||||||
|
t.Fatalf("被持有项重开后读不到 %s: %v", shortDigest(d), err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStress_LargeBlob(t *testing.T) {
|
||||||
|
// 单个大文件:see_video 10 帧 × 2MB 是现实上限附近。
|
||||||
|
s := newTestStore(t, 0)
|
||||||
|
data := randBytes(t, 4*1024*1024) // 4MB
|
||||||
|
|
||||||
|
d, err := s.Put(data, Item{MIME: "image/jpeg", Width: 1920, Height: 1080})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("4MB Put 失败: %v", err)
|
||||||
|
}
|
||||||
|
got, err := s.Get(d)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("4MB Get 失败: %v", err)
|
||||||
|
}
|
||||||
|
if !bytes.Equal(got, data) {
|
||||||
|
t.Fatal("4MB 内容回读不一致")
|
||||||
|
}
|
||||||
|
it, _ := s.Stat(d)
|
||||||
|
if it.Size != int64(len(data)) {
|
||||||
|
t.Fatalf("Size 记录错: %d vs %d", it.Size, len(data))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStress_DataURLRoundTripAtScale(t *testing.T) {
|
||||||
|
// data URL 往返是插件注入的实际路径(SetToolBlocks 给的就是 data URL)。
|
||||||
|
s := newTestStore(t, 0)
|
||||||
|
for i := 0; i < 50; i++ {
|
||||||
|
raw := randBytes(t, 2048)
|
||||||
|
url := DataURL("image/png", raw)
|
||||||
|
mime, decoded, ok := ParseDataURL(url)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("第 %d 次解析失败", i)
|
||||||
|
}
|
||||||
|
if mime != "image/png" || !bytes.Equal(decoded, raw) {
|
||||||
|
t.Fatalf("第 %d 次往返不一致", i)
|
||||||
|
}
|
||||||
|
d, err := s.Put(decoded, Item{MIME: mime})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
back, err := s.Get(d)
|
||||||
|
if err != nil || !bytes.Equal(back, raw) {
|
||||||
|
t.Fatalf("第 %d 次入库回读不一致: %v", i, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
193
internal/memory/migrate.go
Normal file
193
internal/memory/migrate.go
Normal file
@ -0,0 +1,193 @@
|
|||||||
|
package memory
|
||||||
|
|
||||||
|
import (
|
||||||
|
"database/sql"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"regexp"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// 旧媒体实体迁移。
|
||||||
|
//
|
||||||
|
// 历史上媒体进 L3 的方式是把正文 marker 反解成普通实体三元组:
|
||||||
|
//
|
||||||
|
// [image/png a1b2c3d4e5f6] 一张紫蓝红三色带图
|
||||||
|
// → 实体「图片 a1b2c3d4e5f6」(type=Media) -内容-> 「一张紫蓝红三色带图」
|
||||||
|
//
|
||||||
|
// 这条路径把「媒体」伪装成实体 + 用生成的描述文本当语义索引,正是要废弃的
|
||||||
|
// 将就机制。迁移做的事:把每条这类实体还原成原生记忆块,用
|
||||||
|
// sentence --contains--> block 结构边挂到它当时所属的句子上,
|
||||||
|
// 然后删掉旧实体与它的描述关系。块只按自己的向量被检索。
|
||||||
|
//
|
||||||
|
// 迁移是幂等的:实体处理完即删除,重复运行不会重复建块。
|
||||||
|
|
||||||
|
// legacyMediaDigestPattern 从旧媒体实体名尾部取出短 digest。
|
||||||
|
// 名字形如「图片 a1b2c3d4e5f6」——旧实现刻意为每种模态加中文前缀。
|
||||||
|
var legacyMediaDigestPattern = regexp.MustCompile(`([0-9a-f]{8,64})$`)
|
||||||
|
|
||||||
|
// LegacyMediaEntityDigest 从旧媒体实体名里取出短 digest,取不到返回空串。
|
||||||
|
func LegacyMediaEntityDigest(name string) string {
|
||||||
|
name = strings.TrimSpace(name)
|
||||||
|
if !strings.Contains(name, " ") {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
m := legacyMediaDigestPattern.FindStringSubmatch(name)
|
||||||
|
if m == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return m[1]
|
||||||
|
}
|
||||||
|
|
||||||
|
// LegacyMediaResolver 把一个短 digest 解析成可用于 L3 的一等记忆块。
|
||||||
|
// 解析失败(内容已不存在)返回 false,该实体将被直接删除而不建块。
|
||||||
|
type LegacyMediaResolver func(shortDigest string) (MemoryBlock, bool)
|
||||||
|
|
||||||
|
// MigrateLegacyMediaEntities 把 marker 反解出来的旧媒体实体迁移成原生块。
|
||||||
|
//
|
||||||
|
// 返回迁移的块数与删除的旧实体数。任何一步失败都会回滚整个迁移,
|
||||||
|
// 因为半途中断会留下既没有块也没有实体的句子——信息静默消失。
|
||||||
|
func (g *GraphDB) MigrateLegacyMediaEntities(resolve LegacyMediaResolver) (blocks, entities int, err error) {
|
||||||
|
if resolve == nil {
|
||||||
|
return 0, 0, nil
|
||||||
|
}
|
||||||
|
g.mu.Lock()
|
||||||
|
defer g.mu.Unlock()
|
||||||
|
|
||||||
|
rows, err := g.db.Query(`SELECT id, name FROM entities WHERE type = 'Media'`)
|
||||||
|
if err != nil {
|
||||||
|
return 0, 0, err
|
||||||
|
}
|
||||||
|
type legacyEntity struct {
|
||||||
|
id int64
|
||||||
|
name string
|
||||||
|
}
|
||||||
|
var legacy []legacyEntity
|
||||||
|
for rows.Next() {
|
||||||
|
var e legacyEntity
|
||||||
|
if err := rows.Scan(&e.id, &e.name); err != nil {
|
||||||
|
rows.Close()
|
||||||
|
return 0, 0, err
|
||||||
|
}
|
||||||
|
legacy = append(legacy, e)
|
||||||
|
}
|
||||||
|
if err := rows.Err(); err != nil {
|
||||||
|
rows.Close()
|
||||||
|
return 0, 0, err
|
||||||
|
}
|
||||||
|
rows.Close()
|
||||||
|
if len(legacy) == 0 {
|
||||||
|
return 0, 0, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
tx, err := g.db.Begin()
|
||||||
|
if err != nil {
|
||||||
|
return 0, 0, err
|
||||||
|
}
|
||||||
|
defer tx.Rollback()
|
||||||
|
|
||||||
|
// 同一份字节可能被多个旧实体引用(重复注入的同一张图),
|
||||||
|
// 迁移后应指向同一个块:块的身份是内容,不是实体行。
|
||||||
|
blockIDForDigest := make(map[string]string)
|
||||||
|
|
||||||
|
for _, e := range legacy {
|
||||||
|
short := LegacyMediaEntityDigest(e.name)
|
||||||
|
if short != "" {
|
||||||
|
if block, ok := resolve(short); ok && block.PayloadDigest != "" {
|
||||||
|
id, seen := blockIDForDigest[block.PayloadDigest]
|
||||||
|
if !seen {
|
||||||
|
if err := insertMigratedBlock(tx, block); err != nil {
|
||||||
|
return 0, 0, fmt.Errorf("migrate legacy media %s: %w", short, err)
|
||||||
|
}
|
||||||
|
blockIDForDigest[block.PayloadDigest] = block.ID
|
||||||
|
id = block.ID
|
||||||
|
blocks++
|
||||||
|
}
|
||||||
|
n, err := attachBlockToLegacySentences(tx, e.id, id)
|
||||||
|
if err != nil {
|
||||||
|
return 0, 0, err
|
||||||
|
}
|
||||||
|
_ = n
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// 无论能否解析出内容,旧实体与它的描述关系都必须删除:
|
||||||
|
// 留着就等于继续用描述文本当媒体索引。
|
||||||
|
if _, err := tx.Exec(`DELETE FROM relations WHERE source_id = ? OR target_id = ?`, e.id, e.id); err != nil {
|
||||||
|
return 0, 0, err
|
||||||
|
}
|
||||||
|
if _, err := tx.Exec(`DELETE FROM entities WHERE id = ?`, e.id); err != nil {
|
||||||
|
return 0, 0, err
|
||||||
|
}
|
||||||
|
entities++
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := tx.Commit(); err != nil {
|
||||||
|
return 0, 0, err
|
||||||
|
}
|
||||||
|
return blocks, entities, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// insertMigratedBlock 写一条迁移来的块(不经过 PutMemoryBlocks,避免重入锁)。
|
||||||
|
func insertMigratedBlock(tx *sql.Tx, block MemoryBlock) error {
|
||||||
|
if block.ID == "" {
|
||||||
|
return fmt.Errorf("migrated block id is required")
|
||||||
|
}
|
||||||
|
if !validBlockModality(block.Modality) {
|
||||||
|
return fmt.Errorf("migrated block %s has invalid modality %q", block.ID, block.Modality)
|
||||||
|
}
|
||||||
|
vectorJSON, err := json.Marshal(block.Vector)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
created := block.CreatedAt
|
||||||
|
if created.IsZero() {
|
||||||
|
created = time.Now()
|
||||||
|
}
|
||||||
|
_, err = tx.Exec(`INSERT INTO memory_blocks (
|
||||||
|
id, modality, text_content, payload_digest, mime, size, width, height,
|
||||||
|
vector, fingerprint, source, tool, created_at, updated_at
|
||||||
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||||
|
ON CONFLICT(id) DO NOTHING`,
|
||||||
|
block.ID, block.Modality, block.Text, block.PayloadDigest, block.MIME,
|
||||||
|
block.Size, block.Width, block.Height, string(vectorJSON), block.Fingerprint,
|
||||||
|
block.Source, block.Tool, created, time.Now())
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// attachBlockToLegacySentences 把迁移出的块挂到该旧实体当时所属的句子上,
|
||||||
|
// 并保留那些句子(它们可能只有媒体关系,删实体后就再无关系引用)。
|
||||||
|
func attachBlockToLegacySentences(tx *sql.Tx, entityID int64, blockID string) (int, error) {
|
||||||
|
rows, err := tx.Query(`SELECT DISTINCT s.id FROM sentences s
|
||||||
|
JOIN relations r ON r.sentence_id = s.id
|
||||||
|
WHERE r.source_id = ? OR r.target_id = ?`, entityID, entityID)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
var sids []int64
|
||||||
|
for rows.Next() {
|
||||||
|
var sid int64
|
||||||
|
if err := rows.Scan(&sid); err != nil {
|
||||||
|
rows.Close()
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
sids = append(sids, sid)
|
||||||
|
}
|
||||||
|
if err := rows.Err(); err != nil {
|
||||||
|
rows.Close()
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
rows.Close()
|
||||||
|
|
||||||
|
n := 0
|
||||||
|
for _, sid := range sids {
|
||||||
|
if _, err := tx.Exec(`INSERT OR IGNORE INTO memory_block_edges
|
||||||
|
(source_kind, source_id, target_kind, target_id, edge_type)
|
||||||
|
VALUES ('sentence', ?, 'block', ?, 'contains')`,
|
||||||
|
fmt.Sprintf("%d", sid), blockID); err != nil {
|
||||||
|
return n, err
|
||||||
|
}
|
||||||
|
n++
|
||||||
|
}
|
||||||
|
return n, nil
|
||||||
|
}
|
||||||
@ -14,8 +14,8 @@ import (
|
|||||||
"sync"
|
"sync"
|
||||||
"unicode/utf8"
|
"unicode/utf8"
|
||||||
|
|
||||||
"github.com/yanyiwu/gojieba"
|
|
||||||
"gitcode.com/JianFeeeee/HomeAgent/internal/memory/vector"
|
"gitcode.com/JianFeeeee/HomeAgent/internal/memory/vector"
|
||||||
|
"github.com/yanyiwu/gojieba"
|
||||||
)
|
)
|
||||||
|
|
||||||
const downloadMaxWords = 200000
|
const downloadMaxWords = 200000
|
||||||
@ -391,6 +391,13 @@ func (e *StaticEmbedder) Vectorize(text string) vector.Vector {
|
|||||||
return vec
|
return vec
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// EmbedImage 返回 ErrNotSupported:fastText 是纯文本词向量模型,
|
||||||
|
// 没有视觉编码器。要用图像嵌入需要外部视觉模型(如 CLIP/MobileCLIP),
|
||||||
|
// 那个由 config 里的 EmbeddingModelPath 指定的视觉模型负责。
|
||||||
|
func (e *StaticEmbedder) EmbedImage(img []byte, mime string) (vector.Vector, error) {
|
||||||
|
return nil, vector.ErrNotSupported
|
||||||
|
}
|
||||||
|
|
||||||
func (e *StaticEmbedder) Dim() int {
|
func (e *StaticEmbedder) Dim() int {
|
||||||
e.mu.RLock()
|
e.mu.RLock()
|
||||||
defer e.mu.RUnlock()
|
defer e.mu.RUnlock()
|
||||||
|
|||||||
@ -108,7 +108,7 @@ func TestStaticEmbedderSemanticSimilarity(t *testing.T) {
|
|||||||
e := newSynthEmbedder(t, 300)
|
e := newSynthEmbedder(t, 300)
|
||||||
|
|
||||||
pairs := []struct {
|
pairs := []struct {
|
||||||
a, b string
|
a, b string
|
||||||
related bool
|
related bool
|
||||||
}{
|
}{
|
||||||
{"今天天气怎么样", "明天会不会下雨", true},
|
{"今天天气怎么样", "明天会不会下雨", true},
|
||||||
|
|||||||
51
internal/memory/vector/fuse.go
Normal file
51
internal/memory/vector/fuse.go
Normal file
@ -0,0 +1,51 @@
|
|||||||
|
package vector
|
||||||
|
|
||||||
|
import "math"
|
||||||
|
|
||||||
|
// FuseVectors 把同一统一空间里的多个向量融合为一个向量:
|
||||||
|
// 逐维求和后重新 L2 归一化。
|
||||||
|
//
|
||||||
|
// 用途:文档/上下文事件既带文本、又带若干一等记忆块(图片/视频),
|
||||||
|
// 二者的向量来自同一模型、同一 fingerprint、同一维度。融合后,
|
||||||
|
// 一篇文档既能按文字、也能按它携带的图片内容被召回——
|
||||||
|
// 图片由自己的向量参与检索,不依赖任何生成的描述文本。
|
||||||
|
//
|
||||||
|
// 约定:调用方传入的向量应已是 L2 归一化的同空间向量。长度不一致的
|
||||||
|
// 向量会被跳过(不同模型/维度的残留);全空或全零返回 nil。
|
||||||
|
func FuseVectors(vectors ...[]float64) []float64 {
|
||||||
|
dim := 0
|
||||||
|
for _, v := range vectors {
|
||||||
|
if len(v) > dim {
|
||||||
|
dim = len(v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if dim == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
out := make([]float64, dim)
|
||||||
|
used := 0
|
||||||
|
for _, v := range vectors {
|
||||||
|
if len(v) != dim {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for i, x := range v {
|
||||||
|
out[i] += x
|
||||||
|
}
|
||||||
|
used++
|
||||||
|
}
|
||||||
|
if used == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
var norm float64
|
||||||
|
for _, x := range out {
|
||||||
|
norm += x * x
|
||||||
|
}
|
||||||
|
if norm == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
norm = math.Sqrt(norm)
|
||||||
|
for i := range out {
|
||||||
|
out[i] /= norm
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
194
internal/memory/vector/http_embedder.go
Normal file
194
internal/memory/vector/http_embedder.go
Normal file
@ -0,0 +1,194 @@
|
|||||||
|
package vector
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/base64"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitcode.com/JianFeeeee/HomeAgent/pkg/embedding"
|
||||||
|
)
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
// http 也是一个普通 provider:核心只按名字打开它,不知道它背后是云 API、
|
||||||
|
// 自建服务还是别的语言写的模型。
|
||||||
|
embedding.Register("http", func(cfg embedding.Config) (embedding.Provider, error) {
|
||||||
|
return NewHTTPEmbedder(HTTPEmbedderConfig{
|
||||||
|
Endpoint: cfg.Options["endpoint"],
|
||||||
|
APIKey: cfg.Options["api_key"],
|
||||||
|
Model: cfg.Options["model"],
|
||||||
|
Fingerprint: cfg.Options["fingerprint"],
|
||||||
|
Dimension: atoiOrZero(cfg.Options["dimension"]),
|
||||||
|
Timeout: durationOrZero(cfg.Options["timeout"]),
|
||||||
|
})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func atoiOrZero(s string) int {
|
||||||
|
n := 0
|
||||||
|
for _, r := range strings.TrimSpace(s) {
|
||||||
|
if r < '0' || r > '9' {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
n = n*10 + int(r-'0')
|
||||||
|
}
|
||||||
|
return n
|
||||||
|
}
|
||||||
|
|
||||||
|
func durationOrZero(s string) time.Duration {
|
||||||
|
d, err := time.ParseDuration(strings.TrimSpace(s))
|
||||||
|
if err != nil {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
return d
|
||||||
|
}
|
||||||
|
|
||||||
|
// HTTPEmbedderConfig 配置一个外部多模态向量服务。
|
||||||
|
// 服务契约刻意很小:POST Endpoint,输入 modality/data/mime/side,返回 embedding。
|
||||||
|
// 任何云 API 或自建服务只需适配这一个协议,即可复用内核全部向量存储与检索链路。
|
||||||
|
type HTTPEmbedderConfig struct {
|
||||||
|
Endpoint string
|
||||||
|
APIKey string
|
||||||
|
Model string
|
||||||
|
Dimension int
|
||||||
|
Timeout time.Duration
|
||||||
|
Fingerprint string
|
||||||
|
}
|
||||||
|
|
||||||
|
// HTTPEmbedder 是 MultimodalEmbedder 的外部 API 实现。
|
||||||
|
type HTTPEmbedder struct {
|
||||||
|
cfg HTTPEmbedderConfig
|
||||||
|
client *http.Client
|
||||||
|
mu sync.Mutex
|
||||||
|
closed bool
|
||||||
|
}
|
||||||
|
|
||||||
|
type httpEmbedRequest struct {
|
||||||
|
Model string `json:"model,omitempty"`
|
||||||
|
Modality string `json:"modality"`
|
||||||
|
Side string `json:"side"`
|
||||||
|
Text string `json:"text,omitempty"`
|
||||||
|
Data string `json:"data,omitempty"`
|
||||||
|
MIME string `json:"mime,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type httpEmbedResponse struct {
|
||||||
|
Embedding []float64 `json:"embedding"`
|
||||||
|
Data []struct {
|
||||||
|
Embedding []float64 `json:"embedding"`
|
||||||
|
} `json:"data,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewHTTPEmbedder(cfg HTTPEmbedderConfig) (*HTTPEmbedder, error) {
|
||||||
|
if strings.TrimSpace(cfg.Endpoint) == "" {
|
||||||
|
return nil, fmt.Errorf("vector: empty HTTP embedding endpoint")
|
||||||
|
}
|
||||||
|
if cfg.Dimension <= 0 {
|
||||||
|
return nil, fmt.Errorf("vector: invalid HTTP embedding dimension %d", cfg.Dimension)
|
||||||
|
}
|
||||||
|
if cfg.Timeout <= 0 {
|
||||||
|
cfg.Timeout = 30 * time.Second
|
||||||
|
}
|
||||||
|
if cfg.Fingerprint == "" {
|
||||||
|
cfg.Fingerprint = "http:" + cfg.Model + fmt.Sprintf(":%d", cfg.Dimension)
|
||||||
|
}
|
||||||
|
return &HTTPEmbedder{cfg: cfg, client: &http.Client{Timeout: cfg.Timeout}}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *HTTPEmbedder) VectorizeDense(text string) ([]float64, error) {
|
||||||
|
return e.embed(context.Background(), httpEmbedRequest{Model: e.cfg.Model, Modality: string(ModalityText), Side: "query", Text: text})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *HTTPEmbedder) EmbedImageDense(img []byte, mime string) ([]float64, error) {
|
||||||
|
return e.embed(context.Background(), httpEmbedRequest{Model: e.cfg.Model, Modality: string(ModalityImage), Side: "document", Data: base64.StdEncoding.EncodeToString(img), MIME: mime})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *HTTPEmbedder) embed(ctx context.Context, payload httpEmbedRequest) ([]float64, error) {
|
||||||
|
e.mu.Lock()
|
||||||
|
closed := e.closed
|
||||||
|
e.mu.Unlock()
|
||||||
|
if closed {
|
||||||
|
return nil, fmt.Errorf("vector: HTTP embedder closed")
|
||||||
|
}
|
||||||
|
body, err := json.Marshal(payload)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
req, err := http.NewRequestWithContext(ctx, http.MethodPost, e.cfg.Endpoint, bytes.NewReader(body))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
req.Header.Set("Content-Type", "application/json")
|
||||||
|
if e.cfg.APIKey != "" {
|
||||||
|
req.Header.Set("Authorization", "Bearer "+e.cfg.APIKey)
|
||||||
|
}
|
||||||
|
resp, err := e.client.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("vector: HTTP embedding request: %w", err)
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
b, err := io.ReadAll(io.LimitReader(resp.Body, 4<<20))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||||
|
return nil, fmt.Errorf("vector: HTTP embedding status %d: %s", resp.StatusCode, strings.TrimSpace(string(b)))
|
||||||
|
}
|
||||||
|
var out httpEmbedResponse
|
||||||
|
if err := json.Unmarshal(b, &out); err != nil {
|
||||||
|
return nil, fmt.Errorf("vector: decode HTTP embedding: %w", err)
|
||||||
|
}
|
||||||
|
v := out.Embedding
|
||||||
|
if len(v) == 0 && len(out.Data) > 0 {
|
||||||
|
v = out.Data[0].Embedding
|
||||||
|
}
|
||||||
|
if len(v) != e.cfg.Dimension {
|
||||||
|
return nil, fmt.Errorf("vector: HTTP embedding dimension %d, want %d", len(v), e.cfg.Dimension)
|
||||||
|
}
|
||||||
|
return v, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *HTTPEmbedder) Fingerprint() string { return e.cfg.Fingerprint }
|
||||||
|
func (e *HTTPEmbedder) Dim() int { return e.cfg.Dimension }
|
||||||
|
|
||||||
|
// Embed 实现公共 provider 契约:核心只传模态与不透明字节,本实现负责把它
|
||||||
|
// 翻译成外部服务的协议。
|
||||||
|
func (e *HTTPEmbedder) Embed(ctx context.Context, in embedding.Input) ([]float64, error) {
|
||||||
|
req := httpEmbedRequest{
|
||||||
|
Model: e.cfg.Model,
|
||||||
|
Modality: string(in.Modality),
|
||||||
|
Side: string(in.Purpose),
|
||||||
|
Text: in.Text,
|
||||||
|
MIME: in.MIME,
|
||||||
|
}
|
||||||
|
if in.Modality != embedding.ModalityText {
|
||||||
|
req.Data = base64.StdEncoding.EncodeToString(in.Data)
|
||||||
|
}
|
||||||
|
return e.embed(ctx, req)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Info 声明本 provider 的向量空间身份。外部服务的支持模态无法在本地探测,
|
||||||
|
// 因此只声明 text/image 这两条内核真正会走到的路径。
|
||||||
|
func (e *HTTPEmbedder) Info() embedding.Info {
|
||||||
|
return embedding.Info{
|
||||||
|
Dimension: e.cfg.Dimension,
|
||||||
|
Fingerprint: e.cfg.Fingerprint,
|
||||||
|
Modalities: []embedding.Modality{embedding.ModalityText, embedding.ModalityImage},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
func (e *HTTPEmbedder) Loaded() bool {
|
||||||
|
e.mu.Lock()
|
||||||
|
defer e.mu.Unlock()
|
||||||
|
return !e.closed
|
||||||
|
}
|
||||||
|
func (e *HTTPEmbedder) Close() {
|
||||||
|
e.mu.Lock()
|
||||||
|
e.closed = true
|
||||||
|
e.mu.Unlock()
|
||||||
|
}
|
||||||
178
internal/memory/vector/http_embedder_test.go
Normal file
178
internal/memory/vector/http_embedder_test.go
Normal file
@ -0,0 +1,178 @@
|
|||||||
|
package vector
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestHTTPEmbedder_RequiresEndpoint(t *testing.T) {
|
||||||
|
_, err := NewHTTPEmbedder(HTTPEmbedderConfig{Dimension: 512})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("应拒绝空 endpoint")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHTTPEmbedder_RequiresDimension(t *testing.T) {
|
||||||
|
_, err := NewHTTPEmbedder(HTTPEmbedderConfig{Endpoint: "http://localhost"})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("应拒绝 dimension<=0")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHTTPEmbedder_TextEmbedding(t *testing.T) {
|
||||||
|
// 模拟返回 4 维向量的外部服务
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if r.Method != http.MethodPost {
|
||||||
|
t.Errorf("期望 POST,实际 %s", r.Method)
|
||||||
|
}
|
||||||
|
var req httpEmbedRequest
|
||||||
|
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if req.Modality != "text" {
|
||||||
|
t.Errorf("期望 modality=text,实际 %s", req.Modality)
|
||||||
|
}
|
||||||
|
if req.Text == "" {
|
||||||
|
t.Fatal("text 不应为空")
|
||||||
|
}
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
w.Write([]byte(`{"embedding":[0.1,0.2,0.3,0.4]}`))
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
e, err := NewHTTPEmbedder(HTTPEmbedderConfig{
|
||||||
|
Endpoint: srv.URL,
|
||||||
|
Dimension: 4,
|
||||||
|
Model: "test-model",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer e.Close()
|
||||||
|
|
||||||
|
if !e.Loaded() {
|
||||||
|
t.Fatal("应处于 loaded 状态")
|
||||||
|
}
|
||||||
|
|
||||||
|
vec, err := e.VectorizeDense("hello world")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(vec) != 4 || vec[0] != 0.1 || vec[3] != 0.4 {
|
||||||
|
t.Errorf("向量不符合预期: %v", vec)
|
||||||
|
}
|
||||||
|
if e.Fingerprint() != "http:test-model:4" {
|
||||||
|
t.Errorf("指纹不符合预期: %s", e.Fingerprint())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHTTPEmbedder_ImageEmbedding(t *testing.T) {
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
var req httpEmbedRequest
|
||||||
|
json.NewDecoder(r.Body).Decode(&req)
|
||||||
|
if req.Modality != "image" {
|
||||||
|
t.Errorf("期望 modality=image,实际 %s", req.Modality)
|
||||||
|
}
|
||||||
|
if req.MIME != "image/png" {
|
||||||
|
t.Errorf("期望 mime=image/png,实际 %s", req.MIME)
|
||||||
|
}
|
||||||
|
w.Write([]byte(`{"embedding":[0.5,0.5,0.5]}`))
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
e, err := NewHTTPEmbedder(HTTPEmbedderConfig{
|
||||||
|
Endpoint: srv.URL,
|
||||||
|
Dimension: 3,
|
||||||
|
Model: "img-model",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer e.Close()
|
||||||
|
|
||||||
|
vec, err := e.EmbedImageDense([]byte("fake-png-data"), "image/png")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(vec) != 3 {
|
||||||
|
t.Errorf("期望 3 维,实际 %d", len(vec))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHTTPEmbedder_CustomFingerprint(t *testing.T) {
|
||||||
|
e, err := NewHTTPEmbedder(HTTPEmbedderConfig{
|
||||||
|
Endpoint: "http://localhost:1234",
|
||||||
|
Dimension: 512,
|
||||||
|
Fingerprint: "jina-v5-omni-nano:2026",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer e.Close()
|
||||||
|
if e.Fingerprint() != "jina-v5-omni-nano:2026" {
|
||||||
|
t.Errorf("自定义指纹未生效: %s", e.Fingerprint())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHTTPEmbedder_DimensionMismatchReturnsError(t *testing.T) {
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.Write([]byte(`{"embedding":[1,2]}`)) // 返回 2 维,配置期望 4
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
e, err := NewHTTPEmbedder(HTTPEmbedderConfig{
|
||||||
|
Endpoint: srv.URL,
|
||||||
|
Dimension: 4,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer e.Close()
|
||||||
|
|
||||||
|
_, err = e.VectorizeDense("test")
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("维度不匹配时应返回错误")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHTTPEmbedder_ServerErrorReturnsError(t *testing.T) {
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.WriteHeader(http.StatusBadGateway)
|
||||||
|
w.Write([]byte("gateway down"))
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
e, err := NewHTTPEmbedder(HTTPEmbedderConfig{
|
||||||
|
Endpoint: srv.URL,
|
||||||
|
Dimension: 4,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer e.Close()
|
||||||
|
|
||||||
|
_, err = e.VectorizeDense("test")
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("服务端错误时应返回错误")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHTTPEmbedder_ClosePreventsFurtherCalls(t *testing.T) {
|
||||||
|
e, err := NewHTTPEmbedder(HTTPEmbedderConfig{
|
||||||
|
Endpoint: "http://localhost:1234",
|
||||||
|
Dimension: 4,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
e.Close()
|
||||||
|
if e.Loaded() {
|
||||||
|
t.Fatal("关闭后 Loaded() 应返回 false")
|
||||||
|
}
|
||||||
|
_, err = e.VectorizeDense("test")
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("关闭后应返回错误")
|
||||||
|
}
|
||||||
|
}
|
||||||
83
internal/memory/vector/provider_adapter.go
Normal file
83
internal/memory/vector/provider_adapter.go
Normal file
@ -0,0 +1,83 @@
|
|||||||
|
package vector
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"sync"
|
||||||
|
|
||||||
|
"gitcode.com/JianFeeeee/HomeAgent/pkg/embedding"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ProviderAdapter translates the public model-neutral embedding.Provider SPI
|
||||||
|
// to the small internal interface used by the existing memory consumers.
|
||||||
|
// Model selection, media decoding, preprocessing, and runtime details remain
|
||||||
|
// entirely inside the selected provider.
|
||||||
|
type ProviderAdapter struct {
|
||||||
|
provider embedding.Provider
|
||||||
|
info embedding.Info
|
||||||
|
|
||||||
|
mu sync.RWMutex
|
||||||
|
closed bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// AdaptProvider validates and wraps a public provider for internal memory use.
|
||||||
|
func AdaptProvider(provider embedding.Provider) (*ProviderAdapter, error) {
|
||||||
|
info := provider.Info()
|
||||||
|
if err := embedding.ValidateInfo(info); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return &ProviderAdapter{provider: provider, info: info}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *ProviderAdapter) VectorizeDense(text string) ([]float64, error) {
|
||||||
|
return a.embed(embedding.Input{
|
||||||
|
Modality: embedding.ModalityText,
|
||||||
|
Purpose: embedding.PurposeQuery,
|
||||||
|
Text: text,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *ProviderAdapter) EmbedImageDense(data []byte, mime string) ([]float64, error) {
|
||||||
|
return a.embed(embedding.Input{
|
||||||
|
Modality: embedding.ModalityImage,
|
||||||
|
Purpose: embedding.PurposeDocument,
|
||||||
|
Data: data,
|
||||||
|
MIME: mime,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *ProviderAdapter) embed(input embedding.Input) ([]float64, error) {
|
||||||
|
a.mu.RLock()
|
||||||
|
closed := a.closed
|
||||||
|
a.mu.RUnlock()
|
||||||
|
if closed {
|
||||||
|
return nil, context.Canceled
|
||||||
|
}
|
||||||
|
vec, err := a.provider.Embed(context.Background(), input)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if err := embedding.ValidateVector(vec, a.info.Dimension); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return vec, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *ProviderAdapter) Fingerprint() string { return a.info.Fingerprint }
|
||||||
|
func (a *ProviderAdapter) Dim() int { return a.info.Dimension }
|
||||||
|
|
||||||
|
func (a *ProviderAdapter) Loaded() bool {
|
||||||
|
a.mu.RLock()
|
||||||
|
defer a.mu.RUnlock()
|
||||||
|
return !a.closed
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *ProviderAdapter) Close() {
|
||||||
|
a.mu.Lock()
|
||||||
|
if a.closed {
|
||||||
|
a.mu.Unlock()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
a.closed = true
|
||||||
|
a.mu.Unlock()
|
||||||
|
a.provider.Close()
|
||||||
|
}
|
||||||
115
internal/memory/vector/provider_adapter_test.go
Normal file
115
internal/memory/vector/provider_adapter_test.go
Normal file
@ -0,0 +1,115 @@
|
|||||||
|
package vector
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitcode.com/JianFeeeee/HomeAgent/pkg/embedding"
|
||||||
|
)
|
||||||
|
|
||||||
|
// recordingProvider 记录核心传给 provider 的原始请求,用来断言
|
||||||
|
// 「核心不解释内容、只搬字节」这一契约。
|
||||||
|
type recordingProvider struct {
|
||||||
|
got []embedding.Input
|
||||||
|
dim int
|
||||||
|
closed bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *recordingProvider) Embed(_ context.Context, in embedding.Input) ([]float64, error) {
|
||||||
|
p.got = append(p.got, in)
|
||||||
|
return make([]float64, p.dim), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *recordingProvider) Info() embedding.Info {
|
||||||
|
return embedding.Info{Dimension: p.dim, Fingerprint: "recording:1"}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *recordingProvider) Close() { p.closed = true }
|
||||||
|
|
||||||
|
func TestProviderAdapterPassesOpaqueDataUnchanged(t *testing.T) {
|
||||||
|
inner := &recordingProvider{dim: 3}
|
||||||
|
adapted, err := AdaptProvider(inner)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer adapted.Close()
|
||||||
|
|
||||||
|
// 核心把媒体当作不透明字节搬运:既不解码也不改字节。
|
||||||
|
raw := []byte{0x89, 'P', 'N', 'G', 0x00, 0xff}
|
||||||
|
if _, err := adapted.EmbedImageDense(raw, "image/png"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
got := inner.got[0]
|
||||||
|
if string(got.Data) != string(raw) {
|
||||||
|
t.Fatalf("provider 收到的字节被改动: %v", got.Data)
|
||||||
|
}
|
||||||
|
if got.Modality != embedding.ModalityImage || got.MIME != "image/png" {
|
||||||
|
t.Fatalf("模态/MIME 未原样传递: %+v", got)
|
||||||
|
}
|
||||||
|
if got.Purpose != embedding.PurposeDocument {
|
||||||
|
t.Fatalf("用途应为 document: %q", got.Purpose)
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := adapted.VectorizeDense("hello"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if inner.got[1].Modality != embedding.ModalityText || inner.got[1].Text != "hello" {
|
||||||
|
t.Fatalf("文本请求不正确: %+v", inner.got[1])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProviderAdapterRejectsWrongDimensionFromProvider(t *testing.T) {
|
||||||
|
// provider 声明 3 维却返回 2 维:必须在进入存储前被拦下,
|
||||||
|
// 否则一个维度错的向量会污染整个余弦检索。
|
||||||
|
bad := &badDimProvider{}
|
||||||
|
adapted, err := AdaptProvider(bad)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer adapted.Close()
|
||||||
|
if _, err := adapted.VectorizeDense("x"); err == nil {
|
||||||
|
t.Fatal("维度不符时应返回错误")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type badDimProvider struct{}
|
||||||
|
|
||||||
|
func (badDimProvider) Embed(context.Context, embedding.Input) ([]float64, error) {
|
||||||
|
return []float64{1, 2}, nil
|
||||||
|
}
|
||||||
|
func (badDimProvider) Info() embedding.Info {
|
||||||
|
return embedding.Info{Dimension: 3, Fingerprint: "bad:1"}
|
||||||
|
}
|
||||||
|
func (badDimProvider) Close() {}
|
||||||
|
|
||||||
|
func TestProviderAdapterCloseIsIdempotentAndStopsUse(t *testing.T) {
|
||||||
|
inner := &recordingProvider{dim: 2}
|
||||||
|
adapted, err := AdaptProvider(inner)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
adapted.Close()
|
||||||
|
adapted.Close() // 重复关闭不应 panic 或二次 Close provider
|
||||||
|
if !inner.closed {
|
||||||
|
t.Fatal("Close 未传递到 provider")
|
||||||
|
}
|
||||||
|
if _, err := adapted.VectorizeDense("x"); err == nil {
|
||||||
|
t.Fatal("关闭后应拒绝调用")
|
||||||
|
}
|
||||||
|
if adapted.Loaded() {
|
||||||
|
t.Fatal("关闭后 Loaded() 应为 false")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestModalityUnsupportedSentinelIsShared(t *testing.T) {
|
||||||
|
// 内核侧的哨兵与公共契约的哨兵必须是同一个:provider 返回公共哨兵时,
|
||||||
|
// 内核仍能用自己原有的名字识别。
|
||||||
|
if !errors.Is(ErrModalityUnsupported, embedding.ErrUnsupportedModality) {
|
||||||
|
t.Fatal("vector.ErrModalityUnsupported 与 embedding.ErrUnsupportedModality 未打通")
|
||||||
|
}
|
||||||
|
wrapped := errors.Join(embedding.ErrUnsupportedModality, errors.New("audio/wav"))
|
||||||
|
if !errors.Is(wrapped, ErrModalityUnsupported) {
|
||||||
|
t.Fatal("包装后的错误无法用内核哨兵识别")
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -1,17 +1,73 @@
|
|||||||
package vector
|
package vector
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"fmt"
|
||||||
"math"
|
"math"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
|
|
||||||
|
"gitcode.com/JianFeeeee/HomeAgent/pkg/embedding"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Vectorizer 接口:将文本转为向量
|
// Vectorizer 接口:将文本转为向量
|
||||||
|
//
|
||||||
|
// 多模态嵌入新增可选的 EmbedImage:支持视觉嵌入的实现者覆写此方法,
|
||||||
|
// 不支持的(TF-IDF 等)在默认实现里返回 ErrNotSupported。
|
||||||
type Vectorizer interface {
|
type Vectorizer interface {
|
||||||
Vectorize(text string) Vector
|
Vectorize(text string) Vector
|
||||||
|
EmbedImage(img []byte, mime string) (Vector, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// MultimodalEmbedder 是稠密多模态编码器的接口。
|
||||||
|
//
|
||||||
|
// 与 Vectorizer(稀疏词向量,供 TF-IDF/倒排检索)刻意区分:多模态模型产出的
|
||||||
|
// 是共享稠密空间,直接用于 media.Store 的稠密余弦检索,
|
||||||
|
// **不得**塞进文档/知识层的稀疏 vector.Store(会破坏倒排剪枝与 TF-IDF 语义)。
|
||||||
|
//
|
||||||
|
// 实现不限:可以是内嵌 ONNX,也可以是外部 HTTP 向量服务——
|
||||||
|
// 内核只依赖本接口,两条路径共享同一套检索/存储基础设施。Fingerprint 是模型
|
||||||
|
// 空间标识(如模型文件指纹),作为 vec_model 持久化用于切换后重算。
|
||||||
|
type MultimodalEmbedder interface {
|
||||||
|
VectorizeDense(text string) ([]float64, error)
|
||||||
|
EmbedImageDense(img []byte, mime string) ([]float64, error)
|
||||||
|
Fingerprint() string
|
||||||
|
Dim() int
|
||||||
|
Loaded() bool
|
||||||
|
Close()
|
||||||
|
}
|
||||||
|
|
||||||
|
// MultimodalModality 是统一向量空间支持的输入模态。
|
||||||
|
// 现内核只消费 text/image;外部 API 路径可能扩展 audio/video,
|
||||||
|
// 通过类型断言在接口外按需扩展,不破坏现有契约。
|
||||||
|
type MultimodalModality string
|
||||||
|
|
||||||
|
const (
|
||||||
|
ModalityText MultimodalModality = "text"
|
||||||
|
ModalityImage MultimodalModality = "image"
|
||||||
|
ModalityAudio MultimodalModality = "audio"
|
||||||
|
ModalityVideo MultimodalModality = "video"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ErrNotSupported 表示 Vectorizer 不支持该原生模态;调用方不得以描述文本冒充其向量。
|
||||||
|
var ErrNotSupported = fmt.Errorf("vectorizer does not support image embedding")
|
||||||
|
|
||||||
|
// ErrModalityUnsupported 表示该模态不在本统一向量空间的原生覆盖范围内。
|
||||||
|
//
|
||||||
|
// 它与普通错误语义不同:调用方应把它当作「这条媒体本空间永远不会有向量」
|
||||||
|
// 而不是「这次失败了、下次重试」。绝不能拿另一个模型的向量顶替——那会把
|
||||||
|
// 两套坐标系混进同一空间,检索出来的相似度没有任何意义。
|
||||||
|
//
|
||||||
|
// 它是公共 provider 契约里那个哨兵值的别名,两者 errors.Is 互通:
|
||||||
|
// provider 在自己的包内返回 embedding.ErrUnsupportedModality 即可,
|
||||||
|
// 内核侧的判断无需改变。
|
||||||
|
var ErrModalityUnsupported = embedding.ErrUnsupportedModality
|
||||||
|
|
||||||
|
// 注:曾经这里还有一个可选的 VideoEmbedder 接口(用类型断言探测视频能力)。
|
||||||
|
// 已删除:那让核心为每一个新模态长出一套模型专属方法,正是“核心适配模型”的
|
||||||
|
// 坏味道。模态能力现在是数据(embedding.Info.Modalities),输入是不透明的
|
||||||
|
// Data+MIME(见 pkg/embedding)。
|
||||||
|
|
||||||
// Vector 是带权特征映射:feature → weight
|
// Vector 是带权特征映射:feature → weight
|
||||||
type Vector map[string]float64
|
type Vector map[string]float64
|
||||||
|
|
||||||
@ -61,6 +117,26 @@ func (s *Store) Remove(id string) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (s *Store) Search(query Vector, topK int) []DocVector {
|
func (s *Store) Search(query Vector, topK int) []DocVector {
|
||||||
|
hits := s.SearchScored(query, topK)
|
||||||
|
if len(hits) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
out := make([]DocVector, len(hits))
|
||||||
|
for i, h := range hits {
|
||||||
|
out[i] = h.Doc
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// DocVectorHit 是一篇文档的相似度候选及其原始 cosine 分数。
|
||||||
|
// 跨模态融合需要分数做归一化;纯排序的 Search 不暴露它。
|
||||||
|
type DocVectorHit struct {
|
||||||
|
Doc DocVector
|
||||||
|
Score float64
|
||||||
|
}
|
||||||
|
|
||||||
|
// SearchScored 与 Search 同语义,但返回带原始 cosine 分数的候选。
|
||||||
|
func (s *Store) SearchScored(query Vector, topK int) []DocVectorHit {
|
||||||
s.mu.RLock()
|
s.mu.RLock()
|
||||||
defer s.mu.RUnlock()
|
defer s.mu.RUnlock()
|
||||||
|
|
||||||
@ -100,9 +176,9 @@ func (s *Store) Search(query Vector, topK int) []DocVector {
|
|||||||
results = results[:topK]
|
results = results[:topK]
|
||||||
}
|
}
|
||||||
|
|
||||||
out := make([]DocVector, len(results))
|
out := make([]DocVectorHit, len(results))
|
||||||
for i, r := range results {
|
for i, r := range results {
|
||||||
out[i] = r.doc
|
out[i] = DocVectorHit{Doc: r.doc, Score: r.score}
|
||||||
}
|
}
|
||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
@ -246,9 +322,24 @@ func CosineSimilarity(a, b Vector) float64 {
|
|||||||
return dot / (math.Sqrt(normA) * math.Sqrt(normB))
|
return dot / (math.Sqrt(normA) * math.Sqrt(normB))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// DenseCosine 计算两个 []float64 稠密向量的余弦相似度。
|
||||||
|
// 与 CosineSimilarity(稀疏 map)数学等价,但面向稠密多模态向量。
|
||||||
|
func DenseCosine(a, b []float64) float64 {
|
||||||
|
var dot, na, nb float64
|
||||||
|
for i := range a {
|
||||||
|
dot += a[i] * b[i]
|
||||||
|
na += a[i] * a[i]
|
||||||
|
nb += b[i] * b[i]
|
||||||
|
}
|
||||||
|
if na == 0 || nb == 0 {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
return dot / math.Sqrt(na*nb)
|
||||||
|
}
|
||||||
|
|
||||||
// InvertedIndex 倒排索引,加速向量搜索
|
// InvertedIndex 倒排索引,加速向量搜索
|
||||||
type InvertedIndex struct {
|
type InvertedIndex struct {
|
||||||
mu sync.RWMutex
|
mu sync.RWMutex
|
||||||
postings map[string]map[string]float64 // feature → {docID: weight}
|
postings map[string]map[string]float64 // feature → {docID: weight}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -10,7 +10,21 @@ var (
|
|||||||
// 1.0.0:外部插件从 C ABI 动态库迁到子进程 + 共享内存。
|
// 1.0.0:外部插件从 C ABI 动态库迁到子进程 + 共享内存。
|
||||||
// 这是首个不再加载 `.so`/`.dll` 的版本,与 0.9.x 不兼容(存量插件必须
|
// 这是首个不再加载 `.so`/`.dll` 的版本,与 0.9.x 不兼容(存量插件必须
|
||||||
// 用新版 plugindev 重编),故跃到主版本号。
|
// 用新版 plugindev 重编),故跃到主版本号。
|
||||||
Version = "1.0.0"
|
// 1.1.0:记忆系统支持二进制多媒体节点——CAS 媒体存储 + L0/L2/L3 贯通。
|
||||||
|
// 1.1.1:多模态贯通**插件边界**。内核实现公开 SDK 1.1.0 新增的媒体接口
|
||||||
|
// (doc.insertWithMedia、io.injectMedia / injectMediaSync /
|
||||||
|
// injectInterruptMedia),并把 text/image/audio 三条输入路径归一成
|
||||||
|
// 一条 processInput 主干。
|
||||||
|
// 1.2.0:模型中立的多模态 provider SPI(pkg/embedding)——内核不再适配任何
|
||||||
|
// 具体模型,Qwen 实现移到 providers/qwen3vl;插件运行协议升到 2
|
||||||
|
// (统一共享内存区,fd3 布局改变,不支持滚动升级);并实现 SDK 1.2.0
|
||||||
|
// 新增的注入行为标志位(InjectOptions:no_memory / context_policy)
|
||||||
|
// 与 ChannelDef.ContextPolicy,使输入/排队注入/中断注入/同步注入都能
|
||||||
|
// 声明「是否记入记忆」与「是否据此裁剪上下文」(默认都是否)。
|
||||||
|
//
|
||||||
|
// ❗main 上此值始终是**下一个未发布中版本**,不随 patch 发布变动
|
||||||
|
//(见 docs/git-branching.md §2.1);已发布的版本号看对应的 release/vX.Y.x 与 tag。
|
||||||
|
Version = "1.2.0"
|
||||||
|
|
||||||
// Commit 是构建时的 Git commit hash。
|
// Commit 是构建时的 Git commit hash。
|
||||||
Commit = "unknown"
|
Commit = "unknown"
|
||||||
@ -22,7 +36,26 @@ var (
|
|||||||
KernelName = "HomeAgent"
|
KernelName = "HomeAgent"
|
||||||
|
|
||||||
// SDKCompatibleVersion 是此内核可兼容的最高 SDK 版本(semver)。
|
// SDKCompatibleVersion 是此内核可兼容的最高 SDK 版本(semver)。
|
||||||
SDKCompatibleVersion = "1.0.0"
|
//
|
||||||
|
// 1.1.0:本内核实现了 SDK 1.1.0 的全部新增方法。
|
||||||
|
// 1.2.0:本内核实现了 SDK 1.2.0 的全部新增方法(IOInjector 的六个 *Opts
|
||||||
|
// 注入变体、InjectOptions、ChannelDef.ContextPolicy),因此声明为
|
||||||
|
// 1.2.0。用 SDK 1.0.0/1.1.0 编的存量插件照旧可用——新增方法由
|
||||||
|
// **插件调用、内核实现**,不调就不受影响,无需重编。
|
||||||
|
SDKCompatibleVersion = "1.2.0"
|
||||||
|
|
||||||
|
// SourceURL 是本内核构建所对应的源码地址。
|
||||||
|
//
|
||||||
|
// AGPL-3.0 §13(Remote Network Interaction)要求:当你把修改过的版本
|
||||||
|
// 作为网络服务提供出去时,必须给使用者提供取得 Corresponding Source 的机会。
|
||||||
|
// WebUI 的状态页会把这个值渲染成可见链接,所以:
|
||||||
|
//
|
||||||
|
// ❗**修改后对外部署的分支必须把它改指向自己的源码仓库**,否则链接指向的
|
||||||
|
// 不是你实际运行的那份代码,§13 的提供义务并未履行。
|
||||||
|
//
|
||||||
|
// 构建时可用 -ldflags 覆盖,无需改源码:
|
||||||
|
// -X gitcode.com/JianFeeeee/HomeAgent/internal/meta.SourceURL=<你的仓库>
|
||||||
|
SourceURL = "https://gitcode.com/JianFeeeee/HomeAgent"
|
||||||
)
|
)
|
||||||
|
|
||||||
// FullVersion 返回完整的版本字符串。
|
// FullVersion 返回完整的版本字符串。
|
||||||
|
|||||||
@ -49,9 +49,20 @@ func NewONNXParser(cfg ONNXConfig) (*ONNXParser, error) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
ort.SetSharedLibraryPath(libPath())
|
// ORT 环境是**进程级单例**,同一进程里可能有多个消费者(多模态向量
|
||||||
if err := ort.InitializeEnvironment(); err != nil {
|
// provider、本依存解析器)。onnxruntime_go 的行为是:第二次
|
||||||
return nil, fmt.Errorf("init onnx env: %w", err)
|
// InitializeEnvironment 报「already been initialized」,而
|
||||||
|
// DestroyEnvironment 会把别人正在用的环境一起拆掉——先初始化的 provider
|
||||||
|
// 会因此拿到失效的会话。所以这里只在未初始化时初始化,并且**永不销毁**
|
||||||
|
// (与 providers/chineseclip、providers/qwen3vl 的约定一致):环境随进程存活。
|
||||||
|
//
|
||||||
|
// 这个缺陷是在「发行版默认带 onnxruntime 标签」后才暴露的:不带标签时
|
||||||
|
// 两个消费者不会同时存在,重复初始化与误销毁都无法发生。
|
||||||
|
if !ort.IsInitialized() {
|
||||||
|
ort.SetSharedLibraryPath(libPath())
|
||||||
|
if err := ort.InitializeEnvironment(); err != nil {
|
||||||
|
return nil, fmt.Errorf("init onnx env: %w", err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
inputNames := []string{"input_ids"}
|
inputNames := []string{"input_ids"}
|
||||||
@ -59,7 +70,7 @@ func NewONNXParser(cfg ONNXConfig) (*ONNXParser, error) {
|
|||||||
|
|
||||||
session, err := ort.NewDynamicAdvancedSession(modelPath, inputNames, outputNames, nil)
|
session, err := ort.NewDynamicAdvancedSession(modelPath, inputNames, outputNames, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
ort.DestroyEnvironment()
|
// 不在这里 DestroyEnvironment:环境是进程级的,可能正被多模态 provider 使用。
|
||||||
return nil, fmt.Errorf("create session: %w", err)
|
return nil, fmt.Errorf("create session: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -73,7 +84,7 @@ func NewONNXParser(cfg ONNXConfig) (*ONNXParser, error) {
|
|||||||
func (p *ONNXParser) Close() error {
|
func (p *ONNXParser) Close() error {
|
||||||
p.close.Do(func() {
|
p.close.Do(func() {
|
||||||
p.rt.Destroy()
|
p.rt.Destroy()
|
||||||
ort.DestroyEnvironment()
|
// 不销毁进程级 ORT 环境:多模态 provider 可能仍在使用(见 NewONNXParser)。
|
||||||
})
|
})
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
@ -34,6 +34,9 @@ func TestEventRing_BasicWriteAndConsume(t *testing.T) {
|
|||||||
},
|
},
|
||||||
)
|
)
|
||||||
go consumer.Run()
|
go consumer.Run()
|
||||||
|
// LIFO:先 Stop(打断阻塞的 Read)再 Wait(等 Run 退出),
|
||||||
|
// 两者都必须在 host.Close(munmap 整个区域)之前完成。
|
||||||
|
defer consumer.Wait()
|
||||||
defer consumer.Stop()
|
defer consumer.Stop()
|
||||||
|
|
||||||
// 订阅 agent_output 事件
|
// 订阅 agent_output 事件
|
||||||
@ -85,11 +88,18 @@ func TestEventRing_OverflowStillDelivers(t *testing.T) {
|
|||||||
host.EvtfdReadFile(),
|
host.EvtfdReadFile(),
|
||||||
0,
|
0,
|
||||||
func(evt *pubsdk.Event) error {
|
func(evt *pubsdk.Event) error {
|
||||||
received <- evt
|
// 非阻塞投递:本用例写入了 8292 条事件,若这里阻塞在
|
||||||
|
// channel 上,drainEvents 会卡在 handler 里,Stop 就无法
|
||||||
|
// 让 Run 退出。
|
||||||
|
select {
|
||||||
|
case received <- evt:
|
||||||
|
default:
|
||||||
|
}
|
||||||
return nil
|
return nil
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
go consumer.Run()
|
go consumer.Run()
|
||||||
|
defer consumer.Wait()
|
||||||
defer consumer.Stop()
|
defer consumer.Stop()
|
||||||
|
|
||||||
select {
|
select {
|
||||||
@ -125,6 +135,7 @@ func TestEventRing_TypeMaskFiltering(t *testing.T) {
|
|||||||
},
|
},
|
||||||
)
|
)
|
||||||
go consumer.Run()
|
go consumer.Run()
|
||||||
|
defer consumer.Wait()
|
||||||
defer consumer.Stop()
|
defer consumer.Stop()
|
||||||
|
|
||||||
unsub := er.Subscribe(pubsdk.EventToolCall)
|
unsub := er.Subscribe(pubsdk.EventToolCall)
|
||||||
|
|||||||
538
internal/plugin/proc/arena.go
Normal file
538
internal/plugin/proc/arena.go
Normal file
@ -0,0 +1,538 @@
|
|||||||
|
package proc
|
||||||
|
|
||||||
|
// Exchange Arena:**内核独占管理**的跨进程共享内存块分配器(§13.2 重设计)。
|
||||||
|
//
|
||||||
|
// ── 所有权模型(架构约束)────────────────────────────────────────────
|
||||||
|
//
|
||||||
|
// 共享内存由内核全权管理。插件需要使用共享内存时,通过 syscall 风格的
|
||||||
|
// RPC 向内核申请,内核返回偏移与大小;使用完毕后插件再通知内核回收。
|
||||||
|
//
|
||||||
|
// 因此分配器只存在于**内核进程内**,用一把普通 sync.Mutex 保护即可:
|
||||||
|
// 不需要任何跨进程原子操作,也不存在"共享游标被两个进程各自更新"这一
|
||||||
|
// 类竞态——这正是前几版(bump 游标 / CAS 位图跨进程分配)失败的根本原因。
|
||||||
|
//
|
||||||
|
// 共享内存是**内部实现**,不对插件开发者暴露:插件的公开 API 仍是
|
||||||
|
// 普通字符串/Map(见 SDK 的 IOInjector / ToolHandler)。模板运行时在
|
||||||
|
// 传输层完成全部搬运,开发者无感。
|
||||||
|
//
|
||||||
|
// ── 为什么是变长块而不是定长槽 ──────────────────────────────────────
|
||||||
|
//
|
||||||
|
// 定长槽唯一的理由是"跨进程无法安全地做变长分配"。分配器收回内核后这个
|
||||||
|
// 约束消失,于是可以采用真正的变长块分配(first-fit + 邻块合并):
|
||||||
|
//
|
||||||
|
// - 内核可以按需**标定**每块大小(funccall 帧模型)而不是一律给固定槽
|
||||||
|
// - 大 payload(文件内容、长工具输出)不再受 16KB 槽容量限制
|
||||||
|
// - 块用尽时插件才请求扩容,而不是事先把池铺满
|
||||||
|
//
|
||||||
|
// ── 布局 ──────────────────────────────────────────────────────────
|
||||||
|
//
|
||||||
|
// ┌──────────────────────────────────────────────┐
|
||||||
|
// │ Arena Header 64B │
|
||||||
|
// │ magic / version / capacity / blockBase │
|
||||||
|
// ├──────────────────────────────────────────────┤
|
||||||
|
// │ Block 0: [size | state | owner | prevSize] │
|
||||||
|
// │ [data ...] │
|
||||||
|
// │ Block 1: ... │
|
||||||
|
// │ Block N-1: ...(最后一块的 size 到 arena 末尾)│
|
||||||
|
// └──────────────────────────────────────────────┘
|
||||||
|
//
|
||||||
|
// 块头 16B,size 含头并按 8 字节对齐,因此所有数据起点都是 8 字节对齐的。
|
||||||
|
// prevSize 让 Free 能 O(1) 找到前驱做向后合并(经典分离/合并做法)。
|
||||||
|
//
|
||||||
|
// ── 安全边界 ──────────────────────────────────────────────────────
|
||||||
|
//
|
||||||
|
// 插件归还/读取时提交的是 SharedRef(offset 由插件转述)。内核必须把它
|
||||||
|
// 当成不可信输入:Free 会重新走一遍块链确认 offset 确实是一个已分配块的
|
||||||
|
// 数据起点、owner 匹配,才改动分配器状态;否则伪造的 offset 会直接破坏
|
||||||
|
// 块链。Read 同理。
|
||||||
|
//
|
||||||
|
// ── 惰性物理内存 ──────────────────────────────────────────────────
|
||||||
|
//
|
||||||
|
// arena 很大(MB 级)但底层是 memfd:**未触碰的页不占物理内存**。所以
|
||||||
|
// 把容量开大不会带来常驻内存开销,只是虚拟地址空间。
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"sync"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
arenaMagic uint32 = 0x41524E41 // "ARNA"
|
||||||
|
arenaVersion uint32 = 2 // v2:定长槽 → 变长块
|
||||||
|
|
||||||
|
arenaHeaderSize = 64
|
||||||
|
|
||||||
|
// Arena Header 字段偏移(相对 arena 起始)。
|
||||||
|
arOffMagic = 0
|
||||||
|
arOffVersion = 4
|
||||||
|
arOffCapacity = 8 // arena 可用字节数
|
||||||
|
arOffBlockBase = 12 // 首个块相对 arena 起始的偏移
|
||||||
|
arOffBlockUsed = 16 // 已分配的数据字节数(诊断)
|
||||||
|
arOffReserved = 20
|
||||||
|
|
||||||
|
// 块头 16B:size 含头,按 8 字节对齐。
|
||||||
|
blockHeaderSize = 16
|
||||||
|
blockOffSize = 0 // uint32 总大小(含头)
|
||||||
|
blockOffState = 4 // uint32 0=free 1=used
|
||||||
|
blockOffOwner = 8 // uint32 所有者
|
||||||
|
blockOffPrevSize = 12 // uint32 前一块总大小(0=无前块)
|
||||||
|
|
||||||
|
blockFree uint32 = 0
|
||||||
|
blockUsed uint32 = 1
|
||||||
|
|
||||||
|
// minBlockSize 是拆分后允许的最小块(含头)。低于它就不再拆,
|
||||||
|
// 避免产生一堆无法再利用的碎片。
|
||||||
|
minBlockSize = 32
|
||||||
|
|
||||||
|
// SharedRef.Flags 语义位。
|
||||||
|
sharedRefFlagJSON = 1 << 0 // 载荷是 JSON(工具参数/结果)
|
||||||
|
sharedRefFlagExpand = 1 << 1 // 引用指向插件申请的扩容块(内核需单独归还)
|
||||||
|
|
||||||
|
// OwnerHost 标记由内核分配的块。插件用 Host.NextOwnerID 分配的值。
|
||||||
|
OwnerHost uint32 = 0
|
||||||
|
)
|
||||||
|
|
||||||
|
// arenaDefaultCapacity 是统一区域里预留给 arena 的字节数。
|
||||||
|
//
|
||||||
|
// 取 4MB:memfd 惰性分配,未触碰的页不占物理内存,所以开大无成本;
|
||||||
|
// 但它让单个工具调用可以承载 MB 级 payload,不必再退回内联 RPC。
|
||||||
|
const arenaDefaultCapacity = 4 * 1024 * 1024
|
||||||
|
|
||||||
|
// arenaPublishSize 是统一区域里预留给 arena 的字节数。
|
||||||
|
var arenaPublishSize = uint32(arenaDefaultCapacity)
|
||||||
|
|
||||||
|
// arenaRegion 是块分配器视图。
|
||||||
|
//
|
||||||
|
// region 始终是**完整**统一区域 mmap:SharedRef.Offset 是相对区域起始的
|
||||||
|
// 绝对偏移,因此读写都直接落在 region 上。
|
||||||
|
//
|
||||||
|
// mu 只在**内核进程内**使用——分配器完全由内核持有(见文件头所有权模型)。
|
||||||
|
type arenaRegion struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
|
||||||
|
region []byte
|
||||||
|
base uint32 // arena 在 region 内的起始偏移
|
||||||
|
cap uint32 // arena 可用字节数
|
||||||
|
}
|
||||||
|
|
||||||
|
// arenaMinSize 返回块分配器能工作的最小字节数。
|
||||||
|
func arenaMinSize() uint32 {
|
||||||
|
return arenaHeaderSize + blockHeaderSize + minBlockSize
|
||||||
|
}
|
||||||
|
|
||||||
|
// align8 把 n 向上取整到 8 的倍数(块大小与数据起点都必须 8 字节对齐)。
|
||||||
|
func align8(n uint32) uint32 { return (n + 7) &^ 7 }
|
||||||
|
|
||||||
|
// initArena 在统一区域的 arena 段上初始化块分配器。
|
||||||
|
func initArena(region []byte, base, size uint32) (*arenaRegion, error) {
|
||||||
|
if size < arenaMinSize() {
|
||||||
|
return nil, fmt.Errorf("arena: 段太小(%d 字节,至少需要 %d)", size, arenaMinSize())
|
||||||
|
}
|
||||||
|
if uint64(base)+uint64(size) > uint64(len(region)) {
|
||||||
|
return nil, fmt.Errorf("arena: 越出映射(base=%d size=%d total=%d)", base, size, len(region))
|
||||||
|
}
|
||||||
|
|
||||||
|
a := &arenaRegion{region: region, base: base, cap: size}
|
||||||
|
|
||||||
|
blockBase := align8(arenaHeaderSize)
|
||||||
|
if blockBase+blockHeaderSize+minBlockSize > size {
|
||||||
|
return nil, fmt.Errorf("arena: 头部过大(blockBase=%d size=%d)", blockBase, size)
|
||||||
|
}
|
||||||
|
|
||||||
|
ap := region[base : base+size]
|
||||||
|
putU32(ap[arOffMagic:], arenaMagic)
|
||||||
|
putU32(ap[arOffVersion:], arenaVersion)
|
||||||
|
putU32(ap[arOffCapacity:], size)
|
||||||
|
putU32(ap[arOffBlockBase:], blockBase)
|
||||||
|
putU32(ap[arOffBlockUsed:], 0)
|
||||||
|
|
||||||
|
// 初始状态:一整块 free 覆盖剩余空间。
|
||||||
|
first := blockBase
|
||||||
|
putU32(ap[first+blockOffSize:], size-first)
|
||||||
|
putU32(ap[first+blockOffState:], blockFree)
|
||||||
|
putU32(ap[first+blockOffOwner:], OwnerHost)
|
||||||
|
putU32(ap[first+blockOffPrevSize:], 0)
|
||||||
|
|
||||||
|
return a, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// attachArena 从已初始化的区域解析分配器(诊断用)。
|
||||||
|
func attachArena(region []byte, base, size uint32) (*arenaRegion, error) {
|
||||||
|
if size < arenaHeaderSize {
|
||||||
|
return nil, fmt.Errorf("arena: 段太小(%d)", size)
|
||||||
|
}
|
||||||
|
if uint64(base)+uint64(arenaHeaderSize) > uint64(len(region)) {
|
||||||
|
return nil, fmt.Errorf("arena: 头部越界(base=%d total=%d)", base, len(region))
|
||||||
|
}
|
||||||
|
ap := region[base : base+size]
|
||||||
|
|
||||||
|
if got := getU32(ap[arOffMagic:]); got != arenaMagic {
|
||||||
|
return nil, fmt.Errorf("arena: 魔数不匹配(0x%x)", got)
|
||||||
|
}
|
||||||
|
if got := getU32(ap[arOffVersion:]); got != arenaVersion {
|
||||||
|
return nil, fmt.Errorf("arena: 版本不匹配(%d,期望 %d)", got, arenaVersion)
|
||||||
|
}
|
||||||
|
capacity := getU32(ap[arOffCapacity:])
|
||||||
|
blockBase := getU32(ap[arOffBlockBase:])
|
||||||
|
if capacity != size {
|
||||||
|
return nil, fmt.Errorf("arena: capacity 不匹配(header=%d mapped=%d)", capacity, size)
|
||||||
|
}
|
||||||
|
if blockBase+blockHeaderSize > size {
|
||||||
|
return nil, fmt.Errorf("arena: blockBase 越界(%d,size=%d)", blockBase, size)
|
||||||
|
}
|
||||||
|
return &arenaRegion{region: region, base: base, cap: size}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// MaxPayload 返回单次分配可承载的最大 payload 字节数(上界)。
|
||||||
|
func (a *arenaRegion) MaxPayload() int {
|
||||||
|
return int(a.cap) - arenaHeaderSize - blockHeaderSize
|
||||||
|
}
|
||||||
|
|
||||||
|
// Capacity 返回 arena 总字节数。
|
||||||
|
func (a *arenaRegion) Capacity() uint32 { return a.cap }
|
||||||
|
|
||||||
|
// ---- 块头访问(相对 region 的绝对偏移)----
|
||||||
|
|
||||||
|
func (a *arenaRegion) blockBase() uint32 {
|
||||||
|
return a.base + getU32(a.region[a.base+arOffBlockBase:])
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *arenaRegion) blockEnd() uint32 { return a.base + a.cap }
|
||||||
|
|
||||||
|
func (a *arenaRegion) blockSize(off uint32) uint32 {
|
||||||
|
return getU32(a.region[off+blockOffSize:])
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *arenaRegion) setBlockSize(off, v uint32) {
|
||||||
|
putU32(a.region[off+blockOffSize:], v)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *arenaRegion) blockState(off uint32) uint32 {
|
||||||
|
return getU32(a.region[off+blockOffState:])
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *arenaRegion) setBlockState(off, v uint32) {
|
||||||
|
putU32(a.region[off+blockOffState:], v)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *arenaRegion) blockOwner(off uint32) uint32 {
|
||||||
|
return getU32(a.region[off+blockOffOwner:])
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *arenaRegion) setBlockOwner(off, v uint32) {
|
||||||
|
putU32(a.region[off+blockOffOwner:], v)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *arenaRegion) blockPrevSize(off uint32) uint32 {
|
||||||
|
return getU32(a.region[off+blockOffPrevSize:])
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *arenaRegion) setBlockPrevSize(off, v uint32) {
|
||||||
|
putU32(a.region[off+blockOffPrevSize:], v)
|
||||||
|
}
|
||||||
|
|
||||||
|
// blockDataOff 返回块数据区起点(SharedRef.Offset 即此值)。
|
||||||
|
func (a *arenaRegion) blockDataOff(off uint32) uint32 { return off + blockHeaderSize }
|
||||||
|
|
||||||
|
// nextBlock 返回下一块偏移;到末尾返回 blockEnd()。
|
||||||
|
func (a *arenaRegion) nextBlock(off uint32) uint32 {
|
||||||
|
sz := a.blockSize(off)
|
||||||
|
if sz < blockHeaderSize {
|
||||||
|
return a.blockEnd() // 块链损坏:交给 walk 的步数上限兜底
|
||||||
|
}
|
||||||
|
next := off + sz
|
||||||
|
if next > a.blockEnd() {
|
||||||
|
return a.blockEnd()
|
||||||
|
}
|
||||||
|
return next
|
||||||
|
}
|
||||||
|
|
||||||
|
// DataOffsetOf 返回块偏移对应的数据起点(诊断/测试用)。
|
||||||
|
func (a *arenaRegion) DataOffsetOf(off uint32) uint32 { return a.blockDataOff(off) }
|
||||||
|
|
||||||
|
// Alloc 分配一块至少 n 字节的块,owner 写入块头。
|
||||||
|
//
|
||||||
|
// first-fit:从首个块开始找第一个容量足够的空闲块。分配器由内核独占,
|
||||||
|
// 因此这里的线性扫描不需要任何跨进程同步。
|
||||||
|
func (a *arenaRegion) Alloc(owner uint32, n int, gen uint64) (SharedRef, error) {
|
||||||
|
if n < 0 {
|
||||||
|
return SharedRef{}, fmt.Errorf("arena: 非法长度 %d", n)
|
||||||
|
}
|
||||||
|
need := align8(uint32(n) + blockHeaderSize)
|
||||||
|
if need > a.cap-blockHeaderSize {
|
||||||
|
return SharedRef{}, fmt.Errorf("arena: 请求 %d 字节超出 arena 容量 %d", n, a.MaxPayload())
|
||||||
|
}
|
||||||
|
|
||||||
|
a.mu.Lock()
|
||||||
|
defer a.mu.Unlock()
|
||||||
|
|
||||||
|
for off := a.blockBase(); off < a.blockEnd(); {
|
||||||
|
sz := a.blockSize(off)
|
||||||
|
if sz < blockHeaderSize {
|
||||||
|
return SharedRef{}, fmt.Errorf("arena: 块链损坏(off=%d size=%d)", off, sz)
|
||||||
|
}
|
||||||
|
if a.blockState(off) == blockFree && sz >= need {
|
||||||
|
a.carveLocked(off, need)
|
||||||
|
a.setBlockState(off, blockUsed)
|
||||||
|
a.setBlockOwner(off, owner)
|
||||||
|
a.accountUsed(int32(align8(uint32(n))))
|
||||||
|
return a.refOf(off, n, gen), nil
|
||||||
|
}
|
||||||
|
off = a.nextBlock(off)
|
||||||
|
}
|
||||||
|
return SharedRef{}, fmt.Errorf("arena: 空间不足(请求 %d 字节,容量 %d)", n, a.MaxPayload())
|
||||||
|
}
|
||||||
|
|
||||||
|
// carveLocked 把 off 处的空闲块劈成「已用 need + 剩余空闲」,并把剩余块
|
||||||
|
// 的头与后继块的 prevSize 维护好。
|
||||||
|
func (a *arenaRegion) carveLocked(off, need uint32) {
|
||||||
|
sz := a.blockSize(off)
|
||||||
|
if sz-need < minBlockSize {
|
||||||
|
return // 剩余太小,整块给出去,不拆
|
||||||
|
}
|
||||||
|
rest := off + need
|
||||||
|
restSize := sz - need
|
||||||
|
a.setBlockSize(rest, restSize)
|
||||||
|
a.setBlockState(rest, blockFree)
|
||||||
|
a.setBlockOwner(rest, OwnerHost)
|
||||||
|
a.setBlockPrevSize(rest, need)
|
||||||
|
a.setBlockSize(off, need)
|
||||||
|
|
||||||
|
// rest 的后继块现在以 rest 为前驱
|
||||||
|
if nxt := rest + restSize; nxt < a.blockEnd() {
|
||||||
|
a.setBlockPrevSize(nxt, restSize)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// refOf 由块偏移构造引用。
|
||||||
|
func (a *arenaRegion) refOf(off uint32, n int, gen uint64) SharedRef {
|
||||||
|
return SharedRef{
|
||||||
|
Offset: a.blockDataOff(off),
|
||||||
|
Length: uint32(n),
|
||||||
|
Generation: uint32(gen),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Put 分配并写入 payload。
|
||||||
|
func (a *arenaRegion) Put(owner uint32, payload []byte, gen uint64) (SharedRef, error) {
|
||||||
|
ref, err := a.Alloc(owner, len(payload), gen)
|
||||||
|
if err != nil {
|
||||||
|
return SharedRef{}, err
|
||||||
|
}
|
||||||
|
copy(a.region[ref.Offset:ref.Offset+ref.Length], payload)
|
||||||
|
return ref, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// findBlockByData 在块链上找到数据起点等于 dataOff 的块。
|
||||||
|
//
|
||||||
|
// 这是对**插件提交的不可信 offset** 的校验入口:只有真的走完块链确认
|
||||||
|
// 该 offset 是一个块的数据起点,才允许后续改动分配器状态。
|
||||||
|
func (a *arenaRegion) findBlockByData(dataOff uint32) (uint32, bool) {
|
||||||
|
steps := 0
|
||||||
|
limit := int(a.cap/minBlockSize) + 4
|
||||||
|
for off := a.blockBase(); off < a.blockEnd(); {
|
||||||
|
if steps++; steps > limit {
|
||||||
|
return 0, false // 链损坏,防死循环
|
||||||
|
}
|
||||||
|
sz := a.blockSize(off)
|
||||||
|
if sz < blockHeaderSize || off+sz > a.blockEnd() {
|
||||||
|
return 0, false
|
||||||
|
}
|
||||||
|
if a.blockDataOff(off) == dataOff {
|
||||||
|
return off, true
|
||||||
|
}
|
||||||
|
off += sz
|
||||||
|
}
|
||||||
|
return 0, false
|
||||||
|
}
|
||||||
|
|
||||||
|
// findBlockContaining 找到数据区包含 off 的块。
|
||||||
|
//
|
||||||
|
// 与 findBlockByData 的区别:后者要求 off 恰好是块数据起点(用于 Free),
|
||||||
|
// 前者允许 off 落在块数据区的任意位置(用于 Read:调用帧内的结果区就
|
||||||
|
// 位于帧块中间,而不是块起点)。
|
||||||
|
func (a *arenaRegion) findBlockContaining(off uint32) (uint32, bool) {
|
||||||
|
steps := 0
|
||||||
|
limit := int(a.cap/minBlockSize) + 4
|
||||||
|
for bo := a.blockBase(); bo < a.blockEnd(); {
|
||||||
|
if steps++; steps > limit {
|
||||||
|
return 0, false
|
||||||
|
}
|
||||||
|
sz := a.blockSize(bo)
|
||||||
|
if sz < blockHeaderSize || bo+sz > a.blockEnd() {
|
||||||
|
return 0, false
|
||||||
|
}
|
||||||
|
if off >= a.blockDataOff(bo) && off < bo+sz {
|
||||||
|
return bo, true
|
||||||
|
}
|
||||||
|
bo += sz
|
||||||
|
}
|
||||||
|
return 0, false
|
||||||
|
}
|
||||||
|
|
||||||
|
// Read 按 SharedRef 读取数据。
|
||||||
|
//
|
||||||
|
// 校验:generation 与当前区域一致、offset 落在某个**已分配**块的数据区内、
|
||||||
|
// 引用不跨越块边界。任一不满足都返回 error(而不是 nil——早期版本返回
|
||||||
|
// nil 让调用方分不清“空数据”和“非法引用”)。
|
||||||
|
//
|
||||||
|
// 允许 offset 不必是块起点:调用帧的结果区就在帧块内部。
|
||||||
|
func (a *arenaRegion) Read(ref SharedRef, gen uint64) ([]byte, error) {
|
||||||
|
if ref.IsZero() {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
if ref.Generation != uint32(gen) {
|
||||||
|
return nil, fmt.Errorf("arena: 引用 generation 过期(ref=%d now=%d)", ref.Generation, gen)
|
||||||
|
}
|
||||||
|
if uint64(ref.Offset)+uint64(ref.Length) > uint64(len(a.region)) {
|
||||||
|
return nil, fmt.Errorf("arena: 引用越界(offset=%d len=%d total=%d)",
|
||||||
|
ref.Offset, ref.Length, len(a.region))
|
||||||
|
}
|
||||||
|
|
||||||
|
a.mu.Lock()
|
||||||
|
defer a.mu.Unlock()
|
||||||
|
|
||||||
|
bo, ok := a.findBlockContaining(ref.Offset)
|
||||||
|
if !ok {
|
||||||
|
return nil, fmt.Errorf("arena: 非法引用(offset=%d 不在任何块的数据区)", ref.Offset)
|
||||||
|
}
|
||||||
|
if a.blockState(bo) != blockUsed {
|
||||||
|
return nil, fmt.Errorf("arena: 块(offset=%d)已释放,引用失效", ref.Offset)
|
||||||
|
}
|
||||||
|
if uint64(ref.Offset)+uint64(ref.Length) > uint64(bo+a.blockSize(bo)) {
|
||||||
|
return nil, fmt.Errorf("arena: 引用跨越块边界(offset=%d len=%d blockEnd=%d)",
|
||||||
|
ref.Offset, ref.Length, bo+a.blockSize(bo))
|
||||||
|
}
|
||||||
|
return a.region[ref.Offset : ref.Offset+ref.Length], nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Free 归还 owner 名下的块,并与相邻空闲块合并。
|
||||||
|
//
|
||||||
|
// 会走块链校验 offset 与 owner:伪造引用不能改动分配器状态。
|
||||||
|
func (a *arenaRegion) Free(owner uint32, ref SharedRef) error {
|
||||||
|
if ref.IsZero() {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
a.mu.Lock()
|
||||||
|
defer a.mu.Unlock()
|
||||||
|
|
||||||
|
off, ok := a.findBlockByData(ref.Offset)
|
||||||
|
if !ok {
|
||||||
|
return fmt.Errorf("arena: 非法引用(offset=%d 不是块数据起点)", ref.Offset)
|
||||||
|
}
|
||||||
|
if a.blockState(off) != blockUsed {
|
||||||
|
return fmt.Errorf("arena: 块(offset=%d)未分配(重复归还?)", ref.Offset)
|
||||||
|
}
|
||||||
|
if got := a.blockOwner(off); got != owner {
|
||||||
|
return fmt.Errorf("arena: 块(offset=%d)不属于调用者(owner=%d caller=%d)", ref.Offset, got, owner)
|
||||||
|
}
|
||||||
|
|
||||||
|
a.setBlockState(off, blockFree)
|
||||||
|
a.setBlockOwner(off, OwnerHost)
|
||||||
|
a.accountUsed(-int32(align8(ref.Length)))
|
||||||
|
a.coalesceLocked(off)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// coalesceLocked 向后、向前合并相邻空闲块,并修正后继块的 prevSize。
|
||||||
|
func (a *arenaRegion) coalesceLocked(off uint32) {
|
||||||
|
// 向后合并
|
||||||
|
for {
|
||||||
|
nxt := a.nextBlock(off)
|
||||||
|
if nxt >= a.blockEnd() || a.blockState(nxt) != blockFree {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
a.setBlockSize(off, a.blockSize(off)+a.blockSize(nxt))
|
||||||
|
}
|
||||||
|
// 向前合并
|
||||||
|
if ps := a.blockPrevSize(off); ps > 0 && off > a.blockBase() {
|
||||||
|
prev := off - ps
|
||||||
|
if prev >= a.blockBase() && a.blockState(prev) == blockFree {
|
||||||
|
a.setBlockSize(prev, a.blockSize(prev)+a.blockSize(off))
|
||||||
|
off = prev
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// 后继块的 prevSize 现在应等于合并后本块的大小
|
||||||
|
if nxt := a.nextBlock(off); nxt < a.blockEnd() {
|
||||||
|
a.setBlockPrevSize(nxt, a.blockSize(off))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ReclaimOwner 归还 owner 名下所有块,返回回收块数。
|
||||||
|
//
|
||||||
|
// 用于插件进程退出:崩溃的插件无法归还自己申请的块,若不管会把 arena
|
||||||
|
// 慢慢耗尽,最终让所有走共享内存的调用退化成内联 RPC。
|
||||||
|
func (a *arenaRegion) ReclaimOwner(owner uint32) int {
|
||||||
|
if owner == OwnerHost {
|
||||||
|
return 0 // 内核自己的块由正常路径归还,不在此回收
|
||||||
|
}
|
||||||
|
a.mu.Lock()
|
||||||
|
defer a.mu.Unlock()
|
||||||
|
|
||||||
|
// 先收集偏移再逐个归还:归还过程中会做合并,边遍历边改块链不可靠。
|
||||||
|
var targets []uint32
|
||||||
|
var reclaimedBytes uint32
|
||||||
|
limit := int(a.cap/minBlockSize) + 4
|
||||||
|
steps := 0
|
||||||
|
for off := a.blockBase(); off < a.blockEnd(); {
|
||||||
|
if steps++; steps > limit {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
sz := a.blockSize(off)
|
||||||
|
if sz < blockHeaderSize || off+sz > a.blockEnd() {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if a.blockState(off) == blockUsed && a.blockOwner(off) == owner {
|
||||||
|
targets = append(targets, off)
|
||||||
|
}
|
||||||
|
off += sz
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, off := range targets {
|
||||||
|
if a.blockState(off) != blockUsed {
|
||||||
|
continue // 已被前面的合并吸收
|
||||||
|
}
|
||||||
|
if a.blockOwner(off) != owner {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
reclaimedBytes += a.blockSize(off) - blockHeaderSize
|
||||||
|
a.setBlockState(off, blockFree)
|
||||||
|
a.setBlockOwner(off, OwnerHost)
|
||||||
|
a.coalesceLocked(off)
|
||||||
|
}
|
||||||
|
if reclaimedBytes > 0 {
|
||||||
|
a.accountUsed(-int32(align8(reclaimedBytes)))
|
||||||
|
}
|
||||||
|
return len(targets)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stats 返回 (已用数据字节, 总字节)。已用字节按 8 字节对齐记账。
|
||||||
|
func (a *arenaRegion) Stats() (used, total uint32) {
|
||||||
|
return getU32(a.region[a.base+arOffBlockUsed:]), a.cap
|
||||||
|
}
|
||||||
|
|
||||||
|
// accountUsed 累加/扣减已用字节(相对量,正数表示分配)。
|
||||||
|
func (a *arenaRegion) accountUsed(delta int32) {
|
||||||
|
off := a.base + arOffBlockUsed
|
||||||
|
cur := int32(getU32(a.region[off:]))
|
||||||
|
next := cur + delta
|
||||||
|
if next < 0 {
|
||||||
|
next = 0
|
||||||
|
}
|
||||||
|
putU32(a.region[off:], uint32(next))
|
||||||
|
}
|
||||||
|
|
||||||
|
// BlockCount 返回块链上的块数(诊断/测试用)。
|
||||||
|
func (a *arenaRegion) BlockCount() int {
|
||||||
|
a.mu.Lock()
|
||||||
|
defer a.mu.Unlock()
|
||||||
|
n, limit := 0, int(a.cap/minBlockSize)+4
|
||||||
|
for off := a.blockBase(); off < a.blockEnd() && n < limit; n++ {
|
||||||
|
sz := a.blockSize(off)
|
||||||
|
if sz < blockHeaderSize {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
off += sz
|
||||||
|
}
|
||||||
|
return n
|
||||||
|
}
|
||||||
307
internal/plugin/proc/arena_test.go
Normal file
307
internal/plugin/proc/arena_test.go
Normal file
@ -0,0 +1,307 @@
|
|||||||
|
package proc
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// newTestArena 构造一个独立块分配器(不经 Host),保持单测快速。
|
||||||
|
func newTestArena(t *testing.T, size uint32) *arenaRegion {
|
||||||
|
t.Helper()
|
||||||
|
const base = 64 // 8 字节对齐
|
||||||
|
region := make([]byte, base+size)
|
||||||
|
a, err := initArena(region, base, size)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("initArena: %v", err)
|
||||||
|
}
|
||||||
|
return a
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestArena_AllocFreeRoundTrip(t *testing.T) {
|
||||||
|
a := newTestArena(t, 8*1024)
|
||||||
|
if used, total := a.Stats(); used != 0 || total != 8*1024 {
|
||||||
|
t.Fatalf("初始 Stats: got (%d,%d), want (0,%d)", used, total, 8*1024)
|
||||||
|
}
|
||||||
|
|
||||||
|
ref, err := a.Put(OwnerHost, []byte("hello"), 0)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Put: %v", err)
|
||||||
|
}
|
||||||
|
if used, _ := a.Stats(); used == 0 {
|
||||||
|
t.Fatal("Put 后 used 应大于 0")
|
||||||
|
}
|
||||||
|
got, err := a.Read(ref, 0)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Read: %v", err)
|
||||||
|
}
|
||||||
|
if string(got) != "hello" {
|
||||||
|
t.Fatalf("Read=%q, want hello", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := a.Free(OwnerHost, ref); err != nil {
|
||||||
|
t.Fatalf("Free: %v", err)
|
||||||
|
}
|
||||||
|
if used, _ := a.Stats(); used != 0 {
|
||||||
|
t.Fatalf("Free 后 used=%d, want 0", used)
|
||||||
|
}
|
||||||
|
// 归还后再读必须失败(块已回收)
|
||||||
|
if _, err := a.Read(ref, 0); err == nil {
|
||||||
|
t.Fatal("已释放块的引用应读取失败")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestArena_ConcurrentAllocUnique(t *testing.T) {
|
||||||
|
const workers = 64
|
||||||
|
a := newTestArena(t, 256*1024)
|
||||||
|
|
||||||
|
refs := make(chan SharedRef, workers)
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
for i := 0; i < workers; i++ {
|
||||||
|
wg.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
ref, err := a.Alloc(OwnerHost, 1024, 0)
|
||||||
|
if err != nil {
|
||||||
|
t.Errorf("Alloc: %v", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
refs <- ref
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
wg.Wait()
|
||||||
|
close(refs)
|
||||||
|
|
||||||
|
seen := make(map[uint32]bool, workers)
|
||||||
|
for ref := range refs {
|
||||||
|
if seen[ref.Offset] {
|
||||||
|
t.Fatalf("并发分配拿到重复 offset=%d", ref.Offset)
|
||||||
|
}
|
||||||
|
seen[ref.Offset] = true
|
||||||
|
}
|
||||||
|
if len(seen) != workers {
|
||||||
|
t.Fatalf("唯一块数=%d, want %d", len(seen), workers)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestArena_ExhaustionReturnsError(t *testing.T) {
|
||||||
|
a := newTestArena(t, 4*1024)
|
||||||
|
// 第一次分配吃掉几乎整块
|
||||||
|
if _, err := a.Alloc(OwnerHost, 3*1024, 0); err != nil {
|
||||||
|
t.Fatalf("首次 Alloc: %v", err)
|
||||||
|
}
|
||||||
|
// 再申请一大块必然失败
|
||||||
|
if _, err := a.Alloc(OwnerHost, 3*1024, 0); err == nil {
|
||||||
|
t.Fatal("空间不足时 Alloc 应返回错误")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestArena_RejectsOversizePayload(t *testing.T) {
|
||||||
|
a := newTestArena(t, 4*1024)
|
||||||
|
if _, err := a.Alloc(OwnerHost, a.MaxPayload()+1, 0); err == nil {
|
||||||
|
t.Fatal("超出 arena 容量的请求应被拒绝")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestArena_FreeEnforcesOwnership(t *testing.T) {
|
||||||
|
a := newTestArena(t, 8*1024)
|
||||||
|
const ownerA, ownerB = 7, 9
|
||||||
|
|
||||||
|
refA, err := a.Alloc(ownerA, 128, 0)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// 另一个插件不能释放 A 的块
|
||||||
|
if err := a.Free(ownerB, refA); err == nil {
|
||||||
|
t.Fatal("跨 owner 归还应被拒绝")
|
||||||
|
}
|
||||||
|
if used, _ := a.Stats(); used == 0 {
|
||||||
|
t.Fatal("拒绝归还后块应仍然占用")
|
||||||
|
}
|
||||||
|
// 正确的 owner 可以归还
|
||||||
|
if err := a.Free(ownerA, refA); err != nil {
|
||||||
|
t.Fatalf("同 owner 归还: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestArena_FreeRejectsForgedOffset(t *testing.T) {
|
||||||
|
a := newTestArena(t, 8*1024)
|
||||||
|
ref, err := a.Alloc(OwnerHost, 256, 0)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 未对齐到块数据起点的 offset 必须被拒绝——否则会直接破坏块链。
|
||||||
|
forged := ref
|
||||||
|
forged.Offset += 8
|
||||||
|
if err := a.Free(OwnerHost, forged); err == nil {
|
||||||
|
t.Fatal("伪造 offset 应被拒绝")
|
||||||
|
}
|
||||||
|
// 合法引用仍能正常归还(分配器状态未被破坏)
|
||||||
|
if err := a.Free(OwnerHost, ref); err != nil {
|
||||||
|
t.Fatalf("合法引用不应受影响: %v", err)
|
||||||
|
}
|
||||||
|
if used, _ := a.Stats(); used != 0 {
|
||||||
|
t.Fatalf("归还后 used=%d, want 0", used)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestArena_ReclaimOwnerReleasesOnlyItsBlocks(t *testing.T) {
|
||||||
|
a := newTestArena(t, 64*1024)
|
||||||
|
const ownerA, ownerB = 7, 9
|
||||||
|
|
||||||
|
for i := 0; i < 3; i++ {
|
||||||
|
if _, err := a.Alloc(ownerA, 1024, 0); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for i := 0; i < 2; i++ {
|
||||||
|
if _, err := a.Alloc(ownerB, 1024, 0); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if n := a.ReclaimOwner(ownerA); n != 3 {
|
||||||
|
t.Fatalf("ReclaimOwner(A)=%d, want 3", n)
|
||||||
|
}
|
||||||
|
// B 的两块必须保留:仍能读回
|
||||||
|
if n := a.ReclaimOwner(ownerB); n != 2 {
|
||||||
|
t.Fatalf("ReclaimOwner(B)=%d, want 2(A 的回收不能误伤 B)", n)
|
||||||
|
}
|
||||||
|
if used, _ := a.Stats(); used != 0 {
|
||||||
|
t.Fatalf("全部回收后 used=%d, want 0", used)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestArena_ReclaimHostIsNoop(t *testing.T) {
|
||||||
|
a := newTestArena(t, 4*1024)
|
||||||
|
if _, err := a.Alloc(OwnerHost, 128, 0); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if n := a.ReclaimOwner(OwnerHost); n != 0 {
|
||||||
|
t.Fatalf("内核块不应被 ReclaimOwner 回收,实际回收 %d", n)
|
||||||
|
}
|
||||||
|
if used, _ := a.Stats(); used == 0 {
|
||||||
|
t.Fatal("used 应仍大于 0")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestArena_ReadRejectsInvalidRefs(t *testing.T) {
|
||||||
|
a := newTestArena(t, 8*1024)
|
||||||
|
ref, err := a.Put(OwnerHost, []byte("payload"), 0)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// generation 过期
|
||||||
|
stale := ref
|
||||||
|
stale.Generation = 99
|
||||||
|
if _, err := a.Read(stale, 0); err == nil {
|
||||||
|
t.Fatal("generation 不匹配应被拒绝")
|
||||||
|
}
|
||||||
|
|
||||||
|
// offset 不是块数据起点
|
||||||
|
badOffset := ref
|
||||||
|
badOffset.Offset += 8
|
||||||
|
if _, err := a.Read(badOffset, 0); err == nil {
|
||||||
|
t.Fatal("offset 不是块数据起点应被拒绝")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Length 超出块容量
|
||||||
|
tooLong := ref
|
||||||
|
tooLong.Length = 1 << 20
|
||||||
|
if _, err := a.Read(tooLong, 0); err == nil {
|
||||||
|
t.Fatal("Length 超容量应被拒绝")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestArena_PutReadLargePayload(t *testing.T) {
|
||||||
|
a := newTestArena(t, 512*1024)
|
||||||
|
payload := bytes.Repeat([]byte("abcdefgh"), 8192) // 64KB
|
||||||
|
ref, err := a.Put(OwnerHost, payload, 3)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Put: %v", err)
|
||||||
|
}
|
||||||
|
got, err := a.Read(ref, 3)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Read: %v", err)
|
||||||
|
}
|
||||||
|
if !bytes.Equal(got, payload) {
|
||||||
|
t.Fatalf("读回数据不一致:len(got)=%d len(want)=%d", len(got), len(payload))
|
||||||
|
}
|
||||||
|
// 大 payload 不能被 16KB 定长槽时代的容量假设卡住
|
||||||
|
if len(got) <= 16*1024 {
|
||||||
|
t.Fatalf("本用例应验证超过旧 16KB 槽容量的 payload,实际 %d", len(got))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestArena_FreeCoalescesAdjacentBlocks(t *testing.T) {
|
||||||
|
a := newTestArena(t, 64*1024)
|
||||||
|
|
||||||
|
refs := make([]SharedRef, 0, 8)
|
||||||
|
for i := 0; i < 8; i++ {
|
||||||
|
r, err := a.Alloc(OwnerHost, 4*1024, 0)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("第 %d 次 Alloc: %v", i, err)
|
||||||
|
}
|
||||||
|
refs = append(refs, r)
|
||||||
|
}
|
||||||
|
for _, r := range refs {
|
||||||
|
if err := a.Free(OwnerHost, r); err != nil {
|
||||||
|
t.Fatalf("Free: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 全部归还并合并后,应能再分配一个接近整块的大块。
|
||||||
|
big, err := a.Alloc(OwnerHost, 48*1024, 0)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("合并后应能分配大块: %v", err)
|
||||||
|
}
|
||||||
|
if err := a.Free(OwnerHost, big); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if n := a.BlockCount(); n != 1 {
|
||||||
|
t.Fatalf("全部归还并合并后应只剩 1 块,实际 %d", n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestArena_AttachRejectsCorruptLayout(t *testing.T) {
|
||||||
|
a := newTestArena(t, 4*1024)
|
||||||
|
|
||||||
|
putU32(a.region[a.base+arOffMagic:], 0xDEADBEEF)
|
||||||
|
if _, err := attachArena(a.region, a.base, a.cap); err == nil {
|
||||||
|
t.Fatal("魔数错误应被拒绝")
|
||||||
|
}
|
||||||
|
putU32(a.region[a.base+arOffMagic:], arenaMagic)
|
||||||
|
|
||||||
|
putU32(a.region[a.base+arOffVersion:], 99)
|
||||||
|
if _, err := attachArena(a.region, a.base, a.cap); err == nil {
|
||||||
|
t.Fatal("版本不匹配应被拒绝")
|
||||||
|
}
|
||||||
|
putU32(a.region[a.base+arOffVersion:], arenaVersion)
|
||||||
|
|
||||||
|
putU32(a.region[a.base+arOffCapacity:], 1<<30)
|
||||||
|
if _, err := attachArena(a.region, a.base, a.cap); err == nil {
|
||||||
|
t.Fatal("capacity 不匹配应被拒绝")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestArena_InitRejectsTooSmall(t *testing.T) {
|
||||||
|
const base = 64
|
||||||
|
region := make([]byte, base+arenaMinSize()-8)
|
||||||
|
if _, err := initArena(region, base, arenaMinSize()-8); err == nil {
|
||||||
|
t.Fatal("过小的段应被拒绝")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestArena_DataOffsetsAligned(t *testing.T) {
|
||||||
|
a := newTestArena(t, 64*1024)
|
||||||
|
for i, size := range []uint32{1, 7, 8, 9, 100, 4096} {
|
||||||
|
ref, err := a.Alloc(OwnerHost, int(size), 0)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("第 %d 次 Alloc(size=%d): %v", i, size, err)
|
||||||
|
}
|
||||||
|
if ref.Offset%8 != 0 {
|
||||||
|
t.Fatalf("size=%d 的数据起点 %d 未 8 字节对齐", size, ref.Offset)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -4,6 +4,7 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
"os/exec"
|
"os/exec"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
pubsdk "gitcode.com/JianFeeeee/homeagent-sdk/sdk"
|
pubsdk "gitcode.com/JianFeeeee/homeagent-sdk/sdk"
|
||||||
@ -29,6 +30,18 @@ import (
|
|||||||
// EvtRingWritePushConcurrent 83 ns/op ← 并发不恶化
|
// EvtRingWritePushConcurrent 83 ns/op ← 并发不恶化
|
||||||
// StageInvokeSharedMemory 132 µs/op ← 含 3 次进程间往返
|
// StageInvokeSharedMemory 132 µs/op ← 含 3 次进程间往返
|
||||||
// SegmentWriteAllReadInto 3.7 µs/op ← 占 stage 的 2.8%
|
// SegmentWriteAllReadInto 3.7 µs/op ← 占 stage 的 2.8%
|
||||||
|
//
|
||||||
|
// Payload 承载方式对比(2026-09-10,同机;§13.3 取消按大小切内联后补测,
|
||||||
|
// -count=3 取中位):
|
||||||
|
//
|
||||||
|
// payload inline(RPC 报文) frame(共享内存帧) 差
|
||||||
|
// 16 B 26.4 µs/op 48.0 µs/op +21.6 µs
|
||||||
|
// 32 KiB 695.6 µs/op 391.7 µs/op −303.9 µs
|
||||||
|
//
|
||||||
|
// 取舍随 payload 大小翻转:小 payload 多付 ~22µs 的帧分配/读写固定开销,
|
||||||
|
// 大 payload 省掉整份 JSON 编解码与管道字节拷贝,快 44%。因为线上工具结果
|
||||||
|
// 动辄几十 KB,且 22µs 相对 LLM 往返 2-8 秒可忽略,所以统一走帧而不是
|
||||||
|
// 按大小分叉——分叉还多一条只在小 payload 上才跑的分支要维护。
|
||||||
|
|
||||||
// buildBenchPlugin 编译 testdata 里的测试插件(benchmark 版)。
|
// buildBenchPlugin 编译 testdata 里的测试插件(benchmark 版)。
|
||||||
func buildBenchPlugin(b *testing.B, srcName string) string {
|
func buildBenchPlugin(b *testing.B, srcName string) string {
|
||||||
@ -46,11 +59,33 @@ func buildBenchPlugin(b *testing.B, srcName string) string {
|
|||||||
return bin
|
return bin
|
||||||
}
|
}
|
||||||
|
|
||||||
// BenchmarkToolInvoke 测量内核 → 插件的工具调用往返。
|
// BenchmarkToolInvoke 对比工具调用 payload 的两种承载方式(§13.3「延迟对比」)。
|
||||||
//
|
//
|
||||||
// 链路:Call 写 stdin → 插件读循环 → handler → 写 stdout →
|
// inline — legacy 内联:参数 JSON 直接放进 RPC 报文
|
||||||
// 内核 readLoop → pending channel 唤醒。对照实验 11 的 19.6µs。
|
// frame — 生产路径:内核 Alloc 帧,payload 全在共享内存,RPC 只传偏移描述符
|
||||||
|
//
|
||||||
|
// §13.3 取消了按大小切内联的分支,所以 frame 才是线上真实开销;inline 仅留给
|
||||||
|
// 直连 RPC 的测试(process/bench 不建 Host,拿不到共享内存)。两者共用同一条
|
||||||
|
// RPC 通道,差值就是「把 payload 挪进共享内存」的净成本:小 payload 会因为
|
||||||
|
// 多几次分配/拷贝而略慢,大 payload 则省掉整份 JSON 编解码与管道字节拷贝。
|
||||||
|
//
|
||||||
|
// 两个尺寸都测,是因为这一改动的取舍正好随 payload 大小翻转——只看单点
|
||||||
|
// 很容易得出相反结论。对照实验 11 的工具调用 RPC p50 = 19.6µs。
|
||||||
func BenchmarkToolInvoke(b *testing.B) {
|
func BenchmarkToolInvoke(b *testing.B) {
|
||||||
|
for _, sz := range []struct {
|
||||||
|
name string
|
||||||
|
n int
|
||||||
|
}{
|
||||||
|
{"small", 16},
|
||||||
|
{"large", 32 << 10},
|
||||||
|
} {
|
||||||
|
b.Run("inline/"+sz.name, func(b *testing.B) { benchmarkToolInvokeInline(b, sz.n) })
|
||||||
|
b.Run("frame/"+sz.name, func(b *testing.B) { benchmarkToolInvokeFrame(b, sz.n) })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// benchmarkToolInvokeInline 是改造前的老路径:参数随 RPC 报文一起过管道。
|
||||||
|
func benchmarkToolInvokeInline(b *testing.B, payloadSize int) {
|
||||||
bin := buildBenchPlugin(b, "echoplugin.go")
|
bin := buildBenchPlugin(b, "echoplugin.go")
|
||||||
|
|
||||||
p, err := Spawn("echo", bin, Options{Handler: noopHandler})
|
p, err := Spawn("echo", bin, Options{Handler: noopHandler})
|
||||||
@ -59,7 +94,7 @@ func BenchmarkToolInvoke(b *testing.B) {
|
|||||||
}
|
}
|
||||||
defer p.Kill()
|
defer p.Kill()
|
||||||
|
|
||||||
args := map[string]interface{}{"text": "benchmark"}
|
args := map[string]interface{}{"text": strings.Repeat("a", payloadSize)}
|
||||||
|
|
||||||
b.ResetTimer()
|
b.ResetTimer()
|
||||||
for i := 0; i < b.N; i++ {
|
for i := 0; i < b.N; i++ {
|
||||||
@ -72,6 +107,42 @@ func BenchmarkToolInvoke(b *testing.B) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// benchmarkToolInvokeFrame 是 §13.3 之后的生产路径:内核 Alloc 帧 → 参数写帧
|
||||||
|
// 前段 → RPC 只传 {frame, args_len} → 插件从帧读参数、结果写回帧结果区 →
|
||||||
|
// 内核读回并归还整帧(调用帧模型,见 plan.md §13.3)。
|
||||||
|
func benchmarkToolInvokeFrame(b *testing.B, payloadSize int) {
|
||||||
|
bin := buildBenchPlugin(b, "stageplugin.go")
|
||||||
|
core := newFakeCore()
|
||||||
|
|
||||||
|
host, err := NewHost()
|
||||||
|
if err != nil {
|
||||||
|
b.Fatalf("NewHost: %v", err)
|
||||||
|
}
|
||||||
|
defer host.Close()
|
||||||
|
|
||||||
|
p := New("demo", bin, b.TempDir(), nil, host, nil)
|
||||||
|
if err := p.Start(core); err != nil {
|
||||||
|
b.Fatalf("Start: %v", err)
|
||||||
|
}
|
||||||
|
defer p.Close()
|
||||||
|
|
||||||
|
core.mu.Lock()
|
||||||
|
h, ok := core.tools["demo_big"]
|
||||||
|
core.mu.Unlock()
|
||||||
|
if !ok {
|
||||||
|
b.Fatal("插件应注册 demo_big 工具")
|
||||||
|
}
|
||||||
|
|
||||||
|
args := map[string]interface{}{"text": strings.Repeat("a", payloadSize)}
|
||||||
|
|
||||||
|
b.ResetTimer()
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
if _, err := h(args); err != nil {
|
||||||
|
b.Fatalf("第 %d 次调用失败: %v", i, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// BenchmarkStageLockArbitration 测量**内核侧锁仲裁本身**的成本。
|
// BenchmarkStageLockArbitration 测量**内核侧锁仲裁本身**的成本。
|
||||||
//
|
//
|
||||||
// ⚠️ 不要拿这个数字对照实验 3 的 19.40µs——两者测的不是同一个东西:
|
// ⚠️ 不要拿这个数字对照实验 3 的 19.40µs——两者测的不是同一个东西:
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user