mirror of
https://gitcode.com/JianFeeeee/HomeAgent.git
synced 2026-11-12 20:27:30 +00:00
test: 按模块导出接口重写黑盒判据(批次 12:plugin / qwen3vl / chineseclip)
3 个文件全部在 `<module>/test/`,源码目录仍零 `_test.go`。
新增公开输入→输出判据:
## internal/plugin —— SKILL 内容与加载
TestValidateSKILLContent
合法内容通过;空内容/无描述/非法工具名分别被拒
TestLoadSKILLDirectoryExposesMetadataAndTools
临时 SKILL.md → Name/Version/Description/Tools/IOConfig
TestLoadSKILLMissingFileReturnsError
## providers/qwen3vl —— BPE tokenizer 公开契约
TestTokenizerSpecialTokenRoundTrip
临时 tokenizer.json → VocabSize/SpecialID/Encode/DecodeOne
TestTokenizerMissingFileReturnsError
## providers/chineseclip —— WordPiece tokenizer 公开契约
TestTokenizerEncodeUsesSpecialTokensAndPadding
临时 vocab.txt → 固定长度 ids 与 attention mask
TestTokenizerRejectsInvalidVocabulary
缺特殊 token / 非正 maxLength → 报错
所有测试都用 `package test` 导入模块,通过导出 API 操作;不访问私有符号,
不读取源码/文档作为业务断言。
## 变异验证
plugin ValidateSKILLContent 放过无描述内容 → no_description 断言红
qwen3vl Encode 丢弃特殊 token → Encode=[] 断言红
chineseclip Encode 不补 padding → 固定长度判据红
## 验证
go build ./... PASS
go vet ./... PASS
gofmt -l 干净
python3 .github/scripts/check-ci-gates.py PASS
go test ./... -count=1 45 个测试包 / 100 包无测试 / 0 FAIL
3 个新包 -race 全绿
源码目录 _test.go 0
This commit is contained in:
56
providers/chineseclip/test/tokenizer_contract_test.go
Normal file
56
providers/chineseclip/test/tokenizer_contract_test.go
Normal file
@ -0,0 +1,56 @@
|
||||
package test
|
||||
|
||||
// chineseclip tokenizer 黑盒判据:词表文件 → 固定长度的 ids/mask。
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/JianFeeeee/HologramAgent/providers/chineseclip"
|
||||
)
|
||||
|
||||
func writeCLIPVocab(t *testing.T) string {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
vocab := "[PAD]\n[UNK]\n[CLS]\n[SEP]\n你\n好\n"
|
||||
if err := os.WriteFile(filepath.Join(dir, "vocab.txt"), []byte(vocab), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return dir
|
||||
}
|
||||
|
||||
func TestTokenizerEncodeUsesSpecialTokensAndPadding(t *testing.T) {
|
||||
tok, err := chineseclip.LoadTokenizer(writeCLIPVocab(t), 6)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadTokenizer: %v", err)
|
||||
}
|
||||
if tok.MaxLength() != 6 {
|
||||
t.Fatalf("MaxLength=%d", tok.MaxLength())
|
||||
}
|
||||
ids, mask := tok.Encode("你好")
|
||||
if len(ids) != 6 || len(mask) != 6 {
|
||||
t.Fatalf("长度不符 ids=%v mask=%v", ids, mask)
|
||||
}
|
||||
// 词表按文件顺序编号:[PAD]=0,[UNK]=1,[CLS]=2,[SEP]=3,你=4,好=5。
|
||||
wantIDs := []int64{2, 4, 5, 3, 0, 0}
|
||||
wantMask := []int64{1, 1, 1, 1, 0, 0}
|
||||
for i := range ids {
|
||||
if ids[i] != wantIDs[i] || mask[i] != wantMask[i] {
|
||||
t.Fatalf("Encode=%v/%v", ids, mask)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTokenizerRejectsInvalidVocabulary(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(dir, "vocab.txt"), []byte("[PAD]\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := chineseclip.LoadTokenizer(dir, 4); err == nil {
|
||||
t.Fatal("缺特殊 token 的词表却成功")
|
||||
}
|
||||
if _, err := chineseclip.LoadTokenizer(dir, 0); err == nil {
|
||||
t.Fatal("非正 maxLength 却成功")
|
||||
}
|
||||
}
|
||||
62
providers/qwen3vl/test/tokenizer_contract_test.go
Normal file
62
providers/qwen3vl/test/tokenizer_contract_test.go
Normal file
@ -0,0 +1,62 @@
|
||||
package test
|
||||
|
||||
// qwen3vl tokenizer 黑盒判据:从最小公开 tokenizer.json 加载,验证
|
||||
// 特殊 token 的 Encode/Decode/SpecialID 及输入错误。
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/JianFeeeee/HologramAgent/providers/qwen3vl"
|
||||
)
|
||||
|
||||
func writeQwenTokenizer(t *testing.T) string {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
doc := map[string]interface{}{
|
||||
"model": map[string]interface{}{"vocab": map[string]int{"<|im_start|>": 42, "<|endoftext|>": 99}, "merges": []interface{}{}},
|
||||
"added_tokens": []map[string]interface{}{
|
||||
{"id": 42, "content": "<|im_start|>", "special": true},
|
||||
{"id": 99, "content": "<|endoftext|>", "special": true},
|
||||
},
|
||||
}
|
||||
b, err := json.Marshal(doc)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "tokenizer.json"), b, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return dir
|
||||
}
|
||||
|
||||
func TestTokenizerSpecialTokenRoundTrip(t *testing.T) {
|
||||
tok, err := qwen3vl.LoadTokenizer(writeQwenTokenizer(t))
|
||||
if err != nil {
|
||||
t.Fatalf("LoadTokenizer: %v", err)
|
||||
}
|
||||
if tok.VocabSize() != 2 {
|
||||
t.Fatalf("VocabSize=%d", tok.VocabSize())
|
||||
}
|
||||
id, ok := tok.SpecialID("<|im_start|>")
|
||||
if !ok || id != 42 {
|
||||
t.Fatalf("SpecialID=%d,%v", id, ok)
|
||||
}
|
||||
if got := tok.Encode("<|im_start|>"); len(got) != 1 || got[0] != 42 {
|
||||
t.Fatalf("Encode=%v", got)
|
||||
}
|
||||
if got := tok.DecodeOne(42); got != "<|im_start|>" {
|
||||
t.Fatalf("DecodeOne=%q", got)
|
||||
}
|
||||
if got := tok.DecodeOne(12345); got != "" {
|
||||
t.Fatalf("未知 token 解码为 %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTokenizerMissingFileReturnsError(t *testing.T) {
|
||||
if _, err := qwen3vl.LoadTokenizer(t.TempDir()); err == nil {
|
||||
t.Fatal("缺少 tokenizer.json 却成功")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user